@m4ike1/ion-coding-agent 0.1.0

package/dist/core/webfetch/doc-store.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Session-scoped store for fetched web documents.
+ *
+ * Documents are addressed by `web://<n>` refs (incremental per session, stable
+ * for citations) and read through windowed extraction so large pages never
+ * flood context: `extract` echoes the total plus the served window so the
+ * model can page with `offsetChars`/`lengthChars`.
+ *
+ * Search results are stored as incomplete stubs (title/url/snippet only);
+ * `webextract` upgrades a stub to the full fetched document on first read.
+ */
+import type { WebDocument } from "./backend.ts";
+export type WebDocumentSource = "firecrawl" | "undici";
+export interface StoredWebDocument {
+    ref: string;
+    doc: WebDocument;
+    source: WebDocumentSource;
+    /** False for search-result stubs that have not been fetched yet. */
+    complete: boolean;
+}
+export interface ExtractWindow {
+    ref: string;
+    totalChars: number;
+    offsetChars: number;
+    lengthChars: number;
+    text: string;
+}
+export interface ExtractOptions {
+    offsetChars?: number;
+    lengthChars?: number;
+}
+export declare const WEBFETCH_DEFAULT_WINDOW_CHARS = 8000;
+export declare class WebDocumentStore {
+    private readonly entries;
+    private nextId;
+    /** Store a document and return its `web://<n>` ref. Search stubs pass `complete: false`. */
+    store(doc: WebDocument, source: WebDocumentSource, complete?: boolean): string;
+    get(ref: string): StoredWebDocument | undefined;
+    /** Replace the document behind an existing ref (stub upgrade). Marks it complete. */
+    update(ref: string, doc: WebDocument, source: WebDocumentSource): void;
+    get size(): number;
+    /** Windowed read of a stored document. Offsets clamp into range; the window is echoed back for paging. */
+    extract(ref: string, options?: ExtractOptions): ExtractWindow;
+}
+//# sourceMappingURL=doc-store.d.ts.map

package/dist/core/webfetch/doc-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"doc-store.d.ts","sourceRoot":"","sources":["../../../src/core/webfetch/doc-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG,QAAQ,CAAC;AAEvD,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,WAAW,CAAC;IACjB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,oEAAoE;IACpE,QAAQ,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,cAAc;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,6BAA6B,OAAQ,CAAC;AAEnD,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAChE,OAAO,CAAC,MAAM,CAAK;IAEnB,4FAA4F;IAC5F,KAAK,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,iBAAiB,EAAE,QAAQ,UAAO,GAAG,MAAM,CAK1E;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAE9C;IAED,qFAAqF;IACrF,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,iBAAiB,GAAG,IAAI,CAMrE;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,0GAA0G;IAC1G,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,aAAa,CAiBhE;CACD","sourcesContent":["/**\n * Session-scoped store for fetched web documents.\n *\n * Documents are addressed by `web://<n>` refs (incremental per session, stable\n * for citations) and read through windowed extraction so large pages never\n * flood context: `extract` echoes the total plus the served window so the\n * model can page with `offsetChars`/`lengthChars`.\n *\n * Search results are stored as incomplete stubs (title/url/snippet only);\n * `webextract` upgrades a stub to the full fetched document on first read.\n */\n\nimport type { WebDocument } from \"./backend.ts\";\n\nexport type WebDocumentSource = \"firecrawl\" | \"undici\";\n\nexport interface StoredWebDocument {\n\tref: string;\n\tdoc: WebDocument;\n\tsource: WebDocumentSource;\n\t/** False for search-result stubs that have not been fetched yet. */\n\tcomplete: boolean;\n}\n\nexport interface ExtractWindow {\n\tref: string;\n\ttotalChars: number;\n\toffsetChars: number;\n\tlengthChars: number;\n\ttext: string;\n}\n\nexport interface ExtractOptions {\n\toffsetChars?: number;\n\tlengthChars?: number;\n}\n\nexport const WEBFETCH_DEFAULT_WINDOW_CHARS = 8_000;\n\nexport class WebDocumentStore {\n\tprivate readonly entries = new Map<string, StoredWebDocument>();\n\tprivate nextId = 1;\n\n\t/** Store a document and return its `web://<n>` ref. Search stubs pass `complete: false`. */\n\tstore(doc: WebDocument, source: WebDocumentSource, complete = true): string {\n\t\tconst ref = `web://${this.nextId}`;\n\t\tthis.nextId += 1;\n\t\tthis.entries.set(ref, { ref, doc, source, complete });\n\t\treturn ref;\n\t}\n\n\tget(ref: string): StoredWebDocument | undefined {\n\t\treturn this.entries.get(ref);\n\t}\n\n\t/** Replace the document behind an existing ref (stub upgrade). Marks it complete. */\n\tupdate(ref: string, doc: WebDocument, source: WebDocumentSource): void {\n\t\tconst stored = this.entries.get(ref);\n\t\tif (!stored) {\n\t\t\tthrow new Error(`Unknown web reference: ${ref}`);\n\t\t}\n\t\tthis.entries.set(ref, { ref, doc, source, complete: true });\n\t}\n\n\tget size(): number {\n\t\treturn this.entries.size;\n\t}\n\n\t/** Windowed read of a stored document. Offsets clamp into range; the window is echoed back for paging. */\n\textract(ref: string, options: ExtractOptions = {}): ExtractWindow {\n\t\tconst stored = this.entries.get(ref);\n\t\tif (!stored) {\n\t\t\tthrow new Error(`Unknown web reference: ${ref}`);\n\t\t}\n\t\tconst totalChars = stored.doc.markdown.length;\n\t\tconst offsetChars = Math.min(totalChars, Math.max(0, Math.floor(options.offsetChars ?? 0)));\n\t\tconst maxLength = totalChars - offsetChars;\n\t\tconst requested = options.lengthChars ?? WEBFETCH_DEFAULT_WINDOW_CHARS;\n\t\tconst lengthChars = Math.min(maxLength, Math.max(0, Math.floor(requested)));\n\t\treturn {\n\t\t\tref,\n\t\t\ttotalChars,\n\t\t\toffsetChars,\n\t\t\tlengthChars,\n\t\t\ttext: stored.doc.markdown.slice(offsetChars, offsetChars + lengthChars),\n\t\t};\n\t}\n}\n"]}

package/dist/core/webfetch/doc-store.js

@@ -0,0 +1,57 @@
+/**
+ * Session-scoped store for fetched web documents.
+ *
+ * Documents are addressed by `web://<n>` refs (incremental per session, stable
+ * for citations) and read through windowed extraction so large pages never
+ * flood context: `extract` echoes the total plus the served window so the
+ * model can page with `offsetChars`/`lengthChars`.
+ *
+ * Search results are stored as incomplete stubs (title/url/snippet only);
+ * `webextract` upgrades a stub to the full fetched document on first read.
+ */
+export const WEBFETCH_DEFAULT_WINDOW_CHARS = 8_000;
+export class WebDocumentStore {
+    entries = new Map();
+    nextId = 1;
+    /** Store a document and return its `web://<n>` ref. Search stubs pass `complete: false`. */
+    store(doc, source, complete = true) {
+        const ref = `web://${this.nextId}`;
+        this.nextId += 1;
+        this.entries.set(ref, { ref, doc, source, complete });
+        return ref;
+    }
+    get(ref) {
+        return this.entries.get(ref);
+    }
+    /** Replace the document behind an existing ref (stub upgrade). Marks it complete. */
+    update(ref, doc, source) {
+        const stored = this.entries.get(ref);
+        if (!stored) {
+            throw new Error(`Unknown web reference: ${ref}`);
+        }
+        this.entries.set(ref, { ref, doc, source, complete: true });
+    }
+    get size() {
+        return this.entries.size;
+    }
+    /** Windowed read of a stored document. Offsets clamp into range; the window is echoed back for paging. */
+    extract(ref, options = {}) {
+        const stored = this.entries.get(ref);
+        if (!stored) {
+            throw new Error(`Unknown web reference: ${ref}`);
+        }
+        const totalChars = stored.doc.markdown.length;
+        const offsetChars = Math.min(totalChars, Math.max(0, Math.floor(options.offsetChars ?? 0)));
+        const maxLength = totalChars - offsetChars;
+        const requested = options.lengthChars ?? WEBFETCH_DEFAULT_WINDOW_CHARS;
+        const lengthChars = Math.min(maxLength, Math.max(0, Math.floor(requested)));
+        return {
+            ref,
+            totalChars,
+            offsetChars,
+            lengthChars,
+            text: stored.doc.markdown.slice(offsetChars, offsetChars + lengthChars),
+        };
+    }
+}
+//# sourceMappingURL=doc-store.js.map

package/dist/core/webfetch/doc-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"doc-store.js","sourceRoot":"","sources":["../../../src/core/webfetch/doc-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AA2BH,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAK,CAAC;AAEnD,MAAM,OAAO,gBAAgB;IACX,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;IACxD,MAAM,GAAG,CAAC,CAAC;IAEnB,4FAA4F;IAC5F,KAAK,CAAC,GAAgB,EAAE,MAAyB,EAAE,QAAQ,GAAG,IAAI,EAAU;QAC3E,MAAM,GAAG,GAAG,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QACjB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtD,OAAO,GAAG,CAAC;IAAA,CACX;IAED,GAAG,CAAC,GAAW,EAAiC;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAAA,CAC7B;IAED,qFAAqF;IACrF,MAAM,CAAC,GAAW,EAAE,GAAgB,EAAE,MAAyB,EAAQ;QACtE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAAA,CAC5D;IAED,IAAI,IAAI,GAAW;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAAA,CACzB;IAED,0GAA0G;IAC1G,OAAO,CAAC,GAAW,EAAE,OAAO,GAAmB,EAAE,EAAiB;QACjE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,SAAS,GAAG,UAAU,GAAG,WAAW,CAAC;QAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,IAAI,6BAA6B,CAAC;QACvE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5E,OAAO;YACN,GAAG;YACH,UAAU;YACV,WAAW;YACX,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,WAAW,GAAG,WAAW,CAAC;SACvE,CAAC;IAAA,CACF;CACD","sourcesContent":["/**\n * Session-scoped store for fetched web documents.\n *\n * Documents are addressed by `web://<n>` refs (incremental per session, stable\n * for citations) and read through windowed extraction so large pages never\n * flood context: `extract` echoes the total plus the served window so the\n * model can page with `offsetChars`/`lengthChars`.\n *\n * Search results are stored as incomplete stubs (title/url/snippet only);\n * `webextract` upgrades a stub to the full fetched document on first read.\n */\n\nimport type { WebDocument } from \"./backend.ts\";\n\nexport type WebDocumentSource = \"firecrawl\" | \"undici\";\n\nexport interface StoredWebDocument {\n\tref: string;\n\tdoc: WebDocument;\n\tsource: WebDocumentSource;\n\t/** False for search-result stubs that have not been fetched yet. */\n\tcomplete: boolean;\n}\n\nexport interface ExtractWindow {\n\tref: string;\n\ttotalChars: number;\n\toffsetChars: number;\n\tlengthChars: number;\n\ttext: string;\n}\n\nexport interface ExtractOptions {\n\toffsetChars?: number;\n\tlengthChars?: number;\n}\n\nexport const WEBFETCH_DEFAULT_WINDOW_CHARS = 8_000;\n\nexport class WebDocumentStore {\n\tprivate readonly entries = new Map<string, StoredWebDocument>();\n\tprivate nextId = 1;\n\n\t/** Store a document and return its `web://<n>` ref. Search stubs pass `complete: false`. */\n\tstore(doc: WebDocument, source: WebDocumentSource, complete = true): string {\n\t\tconst ref = `web://${this.nextId}`;\n\t\tthis.nextId += 1;\n\t\tthis.entries.set(ref, { ref, doc, source, complete });\n\t\treturn ref;\n\t}\n\n\tget(ref: string): StoredWebDocument | undefined {\n\t\treturn this.entries.get(ref);\n\t}\n\n\t/** Replace the document behind an existing ref (stub upgrade). Marks it complete. */\n\tupdate(ref: string, doc: WebDocument, source: WebDocumentSource): void {\n\t\tconst stored = this.entries.get(ref);\n\t\tif (!stored) {\n\t\t\tthrow new Error(`Unknown web reference: ${ref}`);\n\t\t}\n\t\tthis.entries.set(ref, { ref, doc, source, complete: true });\n\t}\n\n\tget size(): number {\n\t\treturn this.entries.size;\n\t}\n\n\t/** Windowed read of a stored document. Offsets clamp into range; the window is echoed back for paging. */\n\textract(ref: string, options: ExtractOptions = {}): ExtractWindow {\n\t\tconst stored = this.entries.get(ref);\n\t\tif (!stored) {\n\t\t\tthrow new Error(`Unknown web reference: ${ref}`);\n\t\t}\n\t\tconst totalChars = stored.doc.markdown.length;\n\t\tconst offsetChars = Math.min(totalChars, Math.max(0, Math.floor(options.offsetChars ?? 0)));\n\t\tconst maxLength = totalChars - offsetChars;\n\t\tconst requested = options.lengthChars ?? WEBFETCH_DEFAULT_WINDOW_CHARS;\n\t\tconst lengthChars = Math.min(maxLength, Math.max(0, Math.floor(requested)));\n\t\treturn {\n\t\t\tref,\n\t\t\ttotalChars,\n\t\t\toffsetChars,\n\t\t\tlengthChars,\n\t\t\ttext: stored.doc.markdown.slice(offsetChars, offsetChars + lengthChars),\n\t\t};\n\t}\n}\n"]}

package/dist/core/webfetch/guardrails.d.ts

@@ -0,0 +1,75 @@
+/**
+ * SSRF guardrails for webfetch (patterns copied from Jina Reader — not vendored).
+ *
+ * - Only `http:`/`https:` targets are allowed.
+ * - Literal-IP targets are checked against non-public CIDRs.
+ * - Hostnames are resolved via DNS and **every** returned address is re-checked,
+ *   including on every redirect hop (TOCTOU: re-validate on redirect).
+ * - Redirect chains are capped, response bytes are capped (with a
+ *   `Content-Length` pre-check plus a streaming cutoff), and each hop has a
+ *   per-request timeout.
+ */
+export declare const WEBFETCH_MAX_REDIRECTS = 6;
+export declare const WEBFETCH_MAX_BYTES = 2000000;
+export declare const WEBFETCH_REQUEST_TIMEOUT_MS = 30000;
+export declare const WEBFETCH_FIRECRAWL_MAX_LIMIT = 100;
+export type WebfetchErrorCode = "invalid-url" | "blocked-host" | "redirect-limit" | "byte-limit" | "timeout" | "empty" | "transport" | "unavailable";
+export declare class WebfetchError extends Error {
+    readonly code: WebfetchErrorCode;
+    constructor(message: string, code: WebfetchErrorCode);
+}
+/** True when `ip` is a literal IP address inside a blocked non-public range. Non-IP strings return false. */
+export declare function isBlockedIpAddress(ip: string): boolean;
+/** Parse and validate a web target URL. Only `http:`/`https:` are allowed. */
+export declare function normalizeWebUrl(rawUrl: string): URL;
+/** Reject literal-IP targets inside blocked ranges. Hostnames are checked via DNS separately. */
+export declare function assertLiteralTargetAllowed(url: URL): void;
+/** Reject when any resolved address is blocked. Every address is checked (fail closed). */
+export declare function assertResolvedAddressesAllowed(addresses: string[], hostname: string): void;
+export type DnsLookup = (hostname: string) => Promise<string[]>;
+/**
+ * Fail-closed host check: literal IPs are checked directly, hostnames are
+ * resolved and every returned address is re-checked. Callers must invoke this
+ * for the initial URL and again for each redirect target.
+ */
+export declare function assertHostAllowed(hostname: string, lookup?: DnsLookup): Promise<void>;
+/** Enforce the byte cap against a `Content-Length` header before streaming the body. */
+export declare function assertContentLengthAllowed(contentLength: string | null | undefined, maxBytes?: number): void;
+/** Enforce the byte cap while streaming. Throws once `totalBytes` exceeds `maxBytes`. */
+export declare function assertBytesWithinBudget(totalBytes: number, maxBytes?: number): void;
+/** Clamp a search limit into `[min, max]`, falling back to `fallback` for missing/invalid values. */
+export declare function clampLimit(limit: number | undefined, options: {
+    fallback: number;
+    min: number;
+    max: number;
+}): number;
+export interface GuardedBody {
+    statusCode: number;
+    headers: Record<string, string>;
+    body: AsyncIterable<Uint8Array>;
+}
+export type LowLevelRequestFn = (url: string, init: {
+    headers: Record<string, string>;
+    signal: AbortSignal;
+}) => Promise<GuardedBody>;
+export interface GuardedRequestOptions {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    maxBytes?: number;
+    maxRedirects?: number;
+    signal?: AbortSignal;
+    lookup?: DnsLookup;
+    request?: LowLevelRequestFn;
+}
+export interface GuardedResponse extends GuardedBody {
+    finalUrl: string;
+    redirectCount: number;
+}
+/**
+ * GET a URL under the shared guardrails: scheme allowlist, literal-IP and DNS
+ * checks on the initial URL and every redirect hop, redirect cap,
+ * `Content-Length` pre-check, and per-hop timeout. Streaming byte cutoff is
+ * the caller's job via `assertBytesWithinBudget`.
+ */
+export declare function guardedRequest(rawUrl: string, options?: GuardedRequestOptions): Promise<GuardedResponse>;
+//# sourceMappingURL=guardrails.d.ts.map

package/dist/core/webfetch/guardrails.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"guardrails.d.ts","sourceRoot":"","sources":["../../../src/core/webfetch/guardrails.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAMH,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,kBAAkB,UAAY,CAAC;AAC5C,eAAO,MAAM,2BAA2B,QAAS,CAAC;AAClD,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAEhD,MAAM,MAAM,iBAAiB,GAC1B,aAAa,GACb,cAAc,GACd,gBAAgB,GAChB,YAAY,GACZ,SAAS,GACT,OAAO,GACP,WAAW,GACX,aAAa,CAAC;AAEjB,qBAAa,aAAc,SAAQ,KAAK;IACvC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAC;IACjC,YAAY,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,EAInD;CACD;AAqBD,6GAA6G;AAC7G,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAItD;AAED,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAWnD;AAED,iGAAiG;AACjG,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI,CAKzD;AAED,2FAA2F;AAC3F,wBAAgB,8BAA8B,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAS1F;AAED,MAAM,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAOhE;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAE,SAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1G;AAED,wFAAwF;AACxF,wBAAgB,0BAA0B,CACzC,aAAa,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EACxC,QAAQ,GAAE,MAA2B,GACnC,IAAI,CASN;AAED,yFAAyF;AACzF,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,GAAE,MAA2B,GAAG,IAAI,CAIvG;AAED,qGAAqG;AACrG,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,OAAO,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAGrH;AAED,MAAM,WAAW,WAAW;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,iBAAiB,GAAG,CAC/B,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,MAAM,EAAE,WAAW,CAAA;CAAE,KAC1D,OAAO,CAAC,WAAW,CAAC,CAAC;AAiB1B,MAAM,WAAW,qBAAqB;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,eAAgB,SAAQ,WAAW;IACnD,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC,CAoDlH","sourcesContent":["/**\n * SSRF guardrails for webfetch (patterns copied from Jina Reader — not vendored).\n *\n * - Only `http:`/`https:` targets are allowed.\n * - Literal-IP targets are checked against non-public CIDRs.\n * - Hostnames are resolved via DNS and **every** returned address is re-checked,\n *   including on every redirect hop (TOCTOU: re-validate on redirect).\n * - Redirect chains are capped, response bytes are capped (with a\n *   `Content-Length` pre-check plus a streaming cutoff), and each hop has a\n *   per-request timeout.\n */\n\nimport { lookup as dnsLookupAll } from \"node:dns/promises\";\nimport { BlockList, isIP } from \"node:net\";\nimport { request as undiciRequest } from \"undici\";\n\nexport const WEBFETCH_MAX_REDIRECTS = 6;\nexport const WEBFETCH_MAX_BYTES = 2_000_000;\nexport const WEBFETCH_REQUEST_TIMEOUT_MS = 30_000;\nexport const WEBFETCH_FIRECRAWL_MAX_LIMIT = 100;\n\nexport type WebfetchErrorCode =\n\t| \"invalid-url\"\n\t| \"blocked-host\"\n\t| \"redirect-limit\"\n\t| \"byte-limit\"\n\t| \"timeout\"\n\t| \"empty\"\n\t| \"transport\"\n\t| \"unavailable\";\n\nexport class WebfetchError extends Error {\n\treadonly code: WebfetchErrorCode;\n\tconstructor(message: string, code: WebfetchErrorCode) {\n\t\tsuper(message);\n\t\tthis.name = \"WebfetchError\";\n\t\tthis.code = code;\n\t}\n}\n\nconst blockedRanges: Array<{ subnet: string; prefix: number; type: \"ipv4\" | \"ipv6\" }> = [\n\t{ subnet: \"10.0.0.0\", prefix: 8, type: \"ipv4\" },\n\t{ subnet: \"172.16.0.0\", prefix: 12, type: \"ipv4\" },\n\t{ subnet: \"192.168.0.0\", prefix: 16, type: \"ipv4\" },\n\t{ subnet: \"127.0.0.0\", prefix: 8, type: \"ipv4\" },\n\t{ subnet: \"169.254.0.0\", prefix: 16, type: \"ipv4\" },\n\t{ subnet: \"224.0.0.0\", prefix: 4, type: \"ipv4\" },\n\t{ subnet: \"100.64.0.0\", prefix: 10, type: \"ipv4\" },\n\t{ subnet: \"fc00::\", prefix: 7, type: \"ipv6\" },\n\t{ subnet: \"fe80::\", prefix: 10, type: \"ipv6\" },\n\t{ subnet: \"ff00::\", prefix: 8, type: \"ipv6\" },\n\t{ subnet: \"::1\", prefix: 128, type: \"ipv6\" },\n];\n\nconst blockedList = new BlockList();\nfor (const range of blockedRanges) {\n\tblockedList.addSubnet(range.subnet, range.prefix, range.type);\n}\n\n/** True when `ip` is a literal IP address inside a blocked non-public range. Non-IP strings return false. */\nexport function isBlockedIpAddress(ip: string): boolean {\n\tconst family = isIP(ip);\n\tif (family === 0) return false;\n\treturn blockedList.check(ip, family === 4 ? \"ipv4\" : \"ipv6\");\n}\n\n/** Parse and validate a web target URL. Only `http:`/`https:` are allowed. */\nexport function normalizeWebUrl(rawUrl: string): URL {\n\tlet url: URL;\n\ttry {\n\t\turl = new URL(rawUrl.trim());\n\t} catch {\n\t\tthrow new WebfetchError(`Invalid URL: ${rawUrl}`, \"invalid-url\");\n\t}\n\tif (url.protocol !== \"http:\" && url.protocol !== \"https:\") {\n\t\tthrow new WebfetchError(`Blocked URL scheme \"${url.protocol}\": only http and https are allowed`, \"blocked-host\");\n\t}\n\treturn url;\n}\n\n/** Reject literal-IP targets inside blocked ranges. Hostnames are checked via DNS separately. */\nexport function assertLiteralTargetAllowed(url: URL): void {\n\tconst bare = url.hostname.replace(/^\\[|\\]$/g, \"\");\n\tif (isIP(bare) !== 0 && isBlockedIpAddress(bare)) {\n\t\tthrow new WebfetchError(`Blocked request to private IP ${bare}`, \"blocked-host\");\n\t}\n}\n\n/** Reject when any resolved address is blocked. Every address is checked (fail closed). */\nexport function assertResolvedAddressesAllowed(addresses: string[], hostname: string): void {\n\tif (addresses.length === 0) {\n\t\tthrow new WebfetchError(`Could not resolve host \"${hostname}\"`, \"blocked-host\");\n\t}\n\tfor (const address of addresses) {\n\t\tif (isBlockedIpAddress(address)) {\n\t\t\tthrow new WebfetchError(`Blocked request to \"${hostname}\": resolves to private IP ${address}`, \"blocked-host\");\n\t\t}\n\t}\n}\n\nexport type DnsLookup = (hostname: string) => Promise<string[]>;\n\nasync function defaultLookup(hostname: string): Promise<string[]> {\n\tconst results = await dnsLookupAll(hostname, { all: true });\n\treturn results.map((result) => result.address);\n}\n\n/**\n * Fail-closed host check: literal IPs are checked directly, hostnames are\n * resolved and every returned address is re-checked. Callers must invoke this\n * for the initial URL and again for each redirect target.\n */\nexport async function assertHostAllowed(hostname: string, lookup: DnsLookup = defaultLookup): Promise<void> {\n\tconst bare = hostname.replace(/^\\[|\\]$/g, \"\");\n\tif (isIP(bare) !== 0) {\n\t\tif (isBlockedIpAddress(bare)) {\n\t\t\tthrow new WebfetchError(`Blocked request to private IP ${bare}`, \"blocked-host\");\n\t\t}\n\t\treturn;\n\t}\n\tlet addresses: string[];\n\ttry {\n\t\taddresses = await lookup(bare);\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\tthrow new WebfetchError(`Could not resolve host \"${bare}\": ${reason}`, \"blocked-host\");\n\t}\n\tassertResolvedAddressesAllowed(addresses, bare);\n}\n\n/** Enforce the byte cap against a `Content-Length` header before streaming the body. */\nexport function assertContentLengthAllowed(\n\tcontentLength: string | null | undefined,\n\tmaxBytes: number = WEBFETCH_MAX_BYTES,\n): void {\n\tif (contentLength === null || contentLength === undefined) return;\n\tconst declared = Number.parseInt(contentLength.split(\",\")[0]?.trim() ?? \"\", 10);\n\tif (Number.isFinite(declared) && declared > maxBytes) {\n\t\tthrow new WebfetchError(\n\t\t\t`Response body exceeds ${maxBytes} byte limit (${declared} bytes declared)`,\n\t\t\t\"byte-limit\",\n\t\t);\n\t}\n}\n\n/** Enforce the byte cap while streaming. Throws once `totalBytes` exceeds `maxBytes`. */\nexport function assertBytesWithinBudget(totalBytes: number, maxBytes: number = WEBFETCH_MAX_BYTES): void {\n\tif (totalBytes > maxBytes) {\n\t\tthrow new WebfetchError(`Response body exceeds ${maxBytes} byte limit`, \"byte-limit\");\n\t}\n}\n\n/** Clamp a search limit into `[min, max]`, falling back to `fallback` for missing/invalid values. */\nexport function clampLimit(limit: number | undefined, options: { fallback: number; min: number; max: number }): number {\n\tif (limit === undefined || !Number.isFinite(limit)) return options.fallback;\n\treturn Math.min(options.max, Math.max(options.min, Math.floor(limit)));\n}\n\nexport interface GuardedBody {\n\tstatusCode: number;\n\theaders: Record<string, string>;\n\tbody: AsyncIterable<Uint8Array>;\n}\n\nexport type LowLevelRequestFn = (\n\turl: string,\n\tinit: { headers: Record<string, string>; signal: AbortSignal },\n) => Promise<GuardedBody>;\n\nasync function defaultLowLevelRequest(\n\turl: string,\n\tinit: { headers: Record<string, string>; signal: AbortSignal },\n): Promise<GuardedBody> {\n\tconst response = await undiciRequest(url, { method: \"GET\", headers: init.headers, signal: init.signal });\n\tconst headers: Record<string, string> = {};\n\tfor (const [key, value] of Object.entries(response.headers)) {\n\t\tif (value === undefined) continue;\n\t\theaders[key.toLowerCase()] = Array.isArray(value) ? value.join(\", \") : String(value);\n\t}\n\treturn { statusCode: response.statusCode, headers, body: response.body as AsyncIterable<Uint8Array> };\n}\n\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\n\nexport interface GuardedRequestOptions {\n\theaders?: Record<string, string>;\n\ttimeoutMs?: number;\n\tmaxBytes?: number;\n\tmaxRedirects?: number;\n\tsignal?: AbortSignal;\n\tlookup?: DnsLookup;\n\trequest?: LowLevelRequestFn;\n}\n\nexport interface GuardedResponse extends GuardedBody {\n\tfinalUrl: string;\n\tredirectCount: number;\n}\n\n/**\n * GET a URL under the shared guardrails: scheme allowlist, literal-IP and DNS\n * checks on the initial URL and every redirect hop, redirect cap,\n * `Content-Length` pre-check, and per-hop timeout. Streaming byte cutoff is\n * the caller's job via `assertBytesWithinBudget`.\n */\nexport async function guardedRequest(rawUrl: string, options: GuardedRequestOptions = {}): Promise<GuardedResponse> {\n\tconst timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;\n\tconst maxBytes = options.maxBytes ?? WEBFETCH_MAX_BYTES;\n\tconst maxRedirects = options.maxRedirects ?? WEBFETCH_MAX_REDIRECTS;\n\tconst lookup = options.lookup ?? defaultLookup;\n\tconst request = options.request ?? defaultLowLevelRequest;\n\tconst headers = options.headers ?? {};\n\n\tlet current = normalizeWebUrl(rawUrl);\n\tassertLiteralTargetAllowed(current);\n\n\tfor (let redirectCount = 0; ; redirectCount++) {\n\t\tawait assertHostAllowed(current.hostname, lookup);\n\n\t\tconst timeoutSignal = AbortSignal.timeout(timeoutMs);\n\t\tconst signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;\n\t\tlet response: GuardedBody;\n\t\ttry {\n\t\t\tresponse = await request(current.href, { headers, signal });\n\t\t} catch (error) {\n\t\t\tif (timeoutSignal.aborted && !options.signal?.aborted) {\n\t\t\t\tthrow new WebfetchError(`Request to ${current.href} timed out after ${timeoutMs}ms`, \"timeout\");\n\t\t\t}\n\t\t\tif (options.signal?.aborted) {\n\t\t\t\tthrow new Error(\"Operation aborted\");\n\t\t\t}\n\t\t\tthrow error;\n\t\t}\n\n\t\tconst location = response.headers.location ?? response.headers.Location;\n\t\tif (location !== undefined && REDIRECT_STATUSES.has(response.statusCode)) {\n\t\t\tif (redirectCount >= maxRedirects) {\n\t\t\t\tthrow new WebfetchError(`Redirect chain exceeds ${maxRedirects} hops`, \"redirect-limit\");\n\t\t\t}\n\t\t\ttry {\n\t\t\t\tcurrent = new URL(location, current.href);\n\t\t\t} catch {\n\t\t\t\tthrow new WebfetchError(`Invalid redirect target: ${location}`, \"invalid-url\");\n\t\t\t}\n\t\t\tif (current.protocol !== \"http:\" && current.protocol !== \"https:\") {\n\t\t\t\tthrow new WebfetchError(\n\t\t\t\t\t`Blocked redirect to scheme \"${current.protocol}\": only http and https are allowed`,\n\t\t\t\t\t\"blocked-host\",\n\t\t\t\t);\n\t\t\t}\n\t\t\tassertLiteralTargetAllowed(current);\n\t\t\tcontinue;\n\t\t}\n\n\t\tassertContentLengthAllowed(response.headers[\"content-length\"], maxBytes);\n\t\treturn { ...response, finalUrl: current.href, redirectCount };\n\t}\n}\n"]}

package/dist/core/webfetch/guardrails.js

@@ -0,0 +1,195 @@
+/**
+ * SSRF guardrails for webfetch (patterns copied from Jina Reader — not vendored).
+ *
+ * - Only `http:`/`https:` targets are allowed.
+ * - Literal-IP targets are checked against non-public CIDRs.
+ * - Hostnames are resolved via DNS and **every** returned address is re-checked,
+ *   including on every redirect hop (TOCTOU: re-validate on redirect).
+ * - Redirect chains are capped, response bytes are capped (with a
+ *   `Content-Length` pre-check plus a streaming cutoff), and each hop has a
+ *   per-request timeout.
+ */
+import { lookup as dnsLookupAll } from "node:dns/promises";
+import { BlockList, isIP } from "node:net";
+import { request as undiciRequest } from "undici";
+export const WEBFETCH_MAX_REDIRECTS = 6;
+export const WEBFETCH_MAX_BYTES = 2_000_000;
+export const WEBFETCH_REQUEST_TIMEOUT_MS = 30_000;
+export const WEBFETCH_FIRECRAWL_MAX_LIMIT = 100;
+export class WebfetchError extends Error {
+    code;
+    constructor(message, code) {
+        super(message);
+        this.name = "WebfetchError";
+        this.code = code;
+    }
+}
+const blockedRanges = [
+    { subnet: "10.0.0.0", prefix: 8, type: "ipv4" },
+    { subnet: "172.16.0.0", prefix: 12, type: "ipv4" },
+    { subnet: "192.168.0.0", prefix: 16, type: "ipv4" },
+    { subnet: "127.0.0.0", prefix: 8, type: "ipv4" },
+    { subnet: "169.254.0.0", prefix: 16, type: "ipv4" },
+    { subnet: "224.0.0.0", prefix: 4, type: "ipv4" },
+    { subnet: "100.64.0.0", prefix: 10, type: "ipv4" },
+    { subnet: "fc00::", prefix: 7, type: "ipv6" },
+    { subnet: "fe80::", prefix: 10, type: "ipv6" },
+    { subnet: "ff00::", prefix: 8, type: "ipv6" },
+    { subnet: "::1", prefix: 128, type: "ipv6" },
+];
+const blockedList = new BlockList();
+for (const range of blockedRanges) {
+    blockedList.addSubnet(range.subnet, range.prefix, range.type);
+}
+/** True when `ip` is a literal IP address inside a blocked non-public range. Non-IP strings return false. */
+export function isBlockedIpAddress(ip) {
+    const family = isIP(ip);
+    if (family === 0)
+        return false;
+    return blockedList.check(ip, family === 4 ? "ipv4" : "ipv6");
+}
+/** Parse and validate a web target URL. Only `http:`/`https:` are allowed. */
+export function normalizeWebUrl(rawUrl) {
+    let url;
+    try {
+        url = new URL(rawUrl.trim());
+    }
+    catch {
+        throw new WebfetchError(`Invalid URL: ${rawUrl}`, "invalid-url");
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        throw new WebfetchError(`Blocked URL scheme "${url.protocol}": only http and https are allowed`, "blocked-host");
+    }
+    return url;
+}
+/** Reject literal-IP targets inside blocked ranges. Hostnames are checked via DNS separately. */
+export function assertLiteralTargetAllowed(url) {
+    const bare = url.hostname.replace(/^\[|\]$/g, "");
+    if (isIP(bare) !== 0 && isBlockedIpAddress(bare)) {
+        throw new WebfetchError(`Blocked request to private IP ${bare}`, "blocked-host");
+    }
+}
+/** Reject when any resolved address is blocked. Every address is checked (fail closed). */
+export function assertResolvedAddressesAllowed(addresses, hostname) {
+    if (addresses.length === 0) {
+        throw new WebfetchError(`Could not resolve host "${hostname}"`, "blocked-host");
+    }
+    for (const address of addresses) {
+        if (isBlockedIpAddress(address)) {
+            throw new WebfetchError(`Blocked request to "${hostname}": resolves to private IP ${address}`, "blocked-host");
+        }
+    }
+}
+async function defaultLookup(hostname) {
+    const results = await dnsLookupAll(hostname, { all: true });
+    return results.map((result) => result.address);
+}
+/**
+ * Fail-closed host check: literal IPs are checked directly, hostnames are
+ * resolved and every returned address is re-checked. Callers must invoke this
+ * for the initial URL and again for each redirect target.
+ */
+export async function assertHostAllowed(hostname, lookup = defaultLookup) {
+    const bare = hostname.replace(/^\[|\]$/g, "");
+    if (isIP(bare) !== 0) {
+        if (isBlockedIpAddress(bare)) {
+            throw new WebfetchError(`Blocked request to private IP ${bare}`, "blocked-host");
+        }
+        return;
+    }
+    let addresses;
+    try {
+        addresses = await lookup(bare);
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new WebfetchError(`Could not resolve host "${bare}": ${reason}`, "blocked-host");
+    }
+    assertResolvedAddressesAllowed(addresses, bare);
+}
+/** Enforce the byte cap against a `Content-Length` header before streaming the body. */
+export function assertContentLengthAllowed(contentLength, maxBytes = WEBFETCH_MAX_BYTES) {
+    if (contentLength === null || contentLength === undefined)
+        return;
+    const declared = Number.parseInt(contentLength.split(",")[0]?.trim() ?? "", 10);
+    if (Number.isFinite(declared) && declared > maxBytes) {
+        throw new WebfetchError(`Response body exceeds ${maxBytes} byte limit (${declared} bytes declared)`, "byte-limit");
+    }
+}
+/** Enforce the byte cap while streaming. Throws once `totalBytes` exceeds `maxBytes`. */
+export function assertBytesWithinBudget(totalBytes, maxBytes = WEBFETCH_MAX_BYTES) {
+    if (totalBytes > maxBytes) {
+        throw new WebfetchError(`Response body exceeds ${maxBytes} byte limit`, "byte-limit");
+    }
+}
+/** Clamp a search limit into `[min, max]`, falling back to `fallback` for missing/invalid values. */
+export function clampLimit(limit, options) {
+    if (limit === undefined || !Number.isFinite(limit))
+        return options.fallback;
+    return Math.min(options.max, Math.max(options.min, Math.floor(limit)));
+}
+async function defaultLowLevelRequest(url, init) {
+    const response = await undiciRequest(url, { method: "GET", headers: init.headers, signal: init.signal });
+    const headers = {};
+    for (const [key, value] of Object.entries(response.headers)) {
+        if (value === undefined)
+            continue;
+        headers[key.toLowerCase()] = Array.isArray(value) ? value.join(", ") : String(value);
+    }
+    return { statusCode: response.statusCode, headers, body: response.body };
+}
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+/**
+ * GET a URL under the shared guardrails: scheme allowlist, literal-IP and DNS
+ * checks on the initial URL and every redirect hop, redirect cap,
+ * `Content-Length` pre-check, and per-hop timeout. Streaming byte cutoff is
+ * the caller's job via `assertBytesWithinBudget`.
+ */
+export async function guardedRequest(rawUrl, options = {}) {
+    const timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;
+    const maxBytes = options.maxBytes ?? WEBFETCH_MAX_BYTES;
+    const maxRedirects = options.maxRedirects ?? WEBFETCH_MAX_REDIRECTS;
+    const lookup = options.lookup ?? defaultLookup;
+    const request = options.request ?? defaultLowLevelRequest;
+    const headers = options.headers ?? {};
+    let current = normalizeWebUrl(rawUrl);
+    assertLiteralTargetAllowed(current);
+    for (let redirectCount = 0;; redirectCount++) {
+        await assertHostAllowed(current.hostname, lookup);
+        const timeoutSignal = AbortSignal.timeout(timeoutMs);
+        const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
+        let response;
+        try {
+            response = await request(current.href, { headers, signal });
+        }
+        catch (error) {
+            if (timeoutSignal.aborted && !options.signal?.aborted) {
+                throw new WebfetchError(`Request to ${current.href} timed out after ${timeoutMs}ms`, "timeout");
+            }
+            if (options.signal?.aborted) {
+                throw new Error("Operation aborted");
+            }
+            throw error;
+        }
+        const location = response.headers.location ?? response.headers.Location;
+        if (location !== undefined && REDIRECT_STATUSES.has(response.statusCode)) {
+            if (redirectCount >= maxRedirects) {
+                throw new WebfetchError(`Redirect chain exceeds ${maxRedirects} hops`, "redirect-limit");
+            }
+            try {
+                current = new URL(location, current.href);
+            }
+            catch {
+                throw new WebfetchError(`Invalid redirect target: ${location}`, "invalid-url");
+            }
+            if (current.protocol !== "http:" && current.protocol !== "https:") {
+                throw new WebfetchError(`Blocked redirect to scheme "${current.protocol}": only http and https are allowed`, "blocked-host");
+            }
+            assertLiteralTargetAllowed(current);
+            continue;
+        }
+        assertContentLengthAllowed(response.headers["content-length"], maxBytes);
+        return { ...response, finalUrl: current.href, redirectCount };
+    }
+}
+//# sourceMappingURL=guardrails.js.map

package/dist/core/webfetch/guardrails.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"guardrails.js","sourceRoot":"","sources":["../../../src/core/webfetch/guardrails.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,QAAQ,CAAC;AAElD,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC;AAC5C,MAAM,CAAC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAClD,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAYhD,MAAM,OAAO,aAAc,SAAQ,KAAK;IAC9B,IAAI,CAAoB;IACjC,YAAY,OAAe,EAAE,IAAuB,EAAE;QACrD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;QAC5B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IAAA,CACjB;CACD;AAED,MAAM,aAAa,GAAqE;IACvF,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAC/C,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IAClD,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IACnD,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAChD,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IACnD,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAChD,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IAClD,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAC7C,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9C,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAC7C,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE;CAC5C,CAAC;AAEF,MAAM,WAAW,GAAG,IAAI,SAAS,EAAE,CAAC;AACpC,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;IACnC,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;AAC/D,CAAC;AAED,6GAA6G;AAC7G,MAAM,UAAU,kBAAkB,CAAC,EAAU,EAAW;IACvD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/B,OAAO,WAAW,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAAA,CAC7D;AAED,8EAA8E;AAC9E,MAAM,UAAU,eAAe,CAAC,MAAc,EAAO;IACpD,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACJ,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,aAAa,CAAC,gBAAgB,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3D,MAAM,IAAI,aAAa,CAAC,uBAAuB,GAAG,CAAC,QAAQ,oCAAoC,EAAE,cAAc,CAAC,CAAC;IAClH,CAAC;IACD,OAAO,GAAG,CAAC;AAAA,CACX;AAED,iGAAiG;AACjG,MAAM,UAAU,0BAA0B,CAAC,GAAQ,EAAQ;IAC1D,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAClD,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,aAAa,CAAC,iCAAiC,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC;IAClF,CAAC;AAAA,CACD;AAED,2FAA2F;AAC3F,MAAM,UAAU,8BAA8B,CAAC,SAAmB,EAAE,QAAgB,EAAQ;IAC3F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,aAAa,CAAC,2BAA2B,QAAQ,GAAG,EAAE,cAAc,CAAC,CAAC;IACjF,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,aAAa,CAAC,uBAAuB,QAAQ,6BAA6B,OAAO,EAAE,EAAE,cAAc,CAAC,CAAC;QAChH,CAAC;IACF,CAAC;AAAA,CACD;AAID,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAqB;IACjE,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAAA,CAC/C;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB,EAAE,MAAM,GAAc,aAAa,EAAiB;IAC3G,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACtB,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,aAAa,CAAC,iCAAiC,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC;QAClF,CAAC;QACD,OAAO;IACR,CAAC;IACD,IAAI,SAAmB,CAAC;IACxB,IAAI,CAAC;QACJ,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,IAAI,aAAa,CAAC,2BAA2B,IAAI,MAAM,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC;IACxF,CAAC;IACD,8BAA8B,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAAA,CAChD;AAED,wFAAwF;AACxF,MAAM,UAAU,0BAA0B,CACzC,aAAwC,EACxC,QAAQ,GAAW,kBAAkB,EAC9B;IACP,IAAI,aAAa,KAAK,IAAI,IAAI,aAAa,KAAK,SAAS;QAAE,OAAO;IAClE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAChF,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;QACtD,MAAM,IAAI,aAAa,CACtB,yBAAyB,QAAQ,gBAAgB,QAAQ,kBAAkB,EAC3E,YAAY,CACZ,CAAC;IACH,CAAC;AAAA,CACD;AAED,yFAAyF;AACzF,MAAM,UAAU,uBAAuB,CAAC,UAAkB,EAAE,QAAQ,GAAW,kBAAkB,EAAQ;IACxG,IAAI,UAAU,GAAG,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,aAAa,CAAC,yBAAyB,QAAQ,aAAa,EAAE,YAAY,CAAC,CAAC;IACvF,CAAC;AAAA,CACD;AAED,qGAAqG;AACrG,MAAM,UAAU,UAAU,CAAC,KAAyB,EAAE,OAAuD,EAAU;IACtH,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAAA,CACvE;AAaD,KAAK,UAAU,sBAAsB,CACpC,GAAW,EACX,IAA8D,EACvC;IACvB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACzG,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAClC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAiC,EAAE,CAAC;AAAA,CACtG;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAiB7D;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAc,EAAE,OAAO,GAA0B,EAAE,EAA4B;IACnH,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,2BAA2B,CAAC;IACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,kBAAkB,CAAC;IACxD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,sBAAsB,CAAC;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,sBAAsB,CAAC;IAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IAEtC,IAAI,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACtC,0BAA0B,CAAC,OAAO,CAAC,CAAC;IAEpC,KAAK,IAAI,aAAa,GAAG,CAAC,GAAI,aAAa,EAAE,EAAE,CAAC;QAC/C,MAAM,iBAAiB,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAElD,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;QACjG,IAAI,QAAqB,CAAC;QAC1B,IAAI,CAAC;YACJ,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,aAAa,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACvD,MAAM,IAAI,aAAa,CAAC,cAAc,OAAO,CAAC,IAAI,oBAAoB,SAAS,IAAI,EAAE,SAAS,CAAC,CAAC;YACjG,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YACtC,CAAC;YACD,MAAM,KAAK,CAAC;QACb,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC;QACxE,IAAI,QAAQ,KAAK,SAAS,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,IAAI,aAAa,IAAI,YAAY,EAAE,CAAC;gBACnC,MAAM,IAAI,aAAa,CAAC,0BAA0B,YAAY,OAAO,EAAE,gBAAgB,CAAC,CAAC;YAC1F,CAAC;YACD,IAAI,CAAC;gBACJ,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,IAAI,aAAa,CAAC,4BAA4B,QAAQ,EAAE,EAAE,aAAa,CAAC,CAAC;YAChF,CAAC;YACD,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACnE,MAAM,IAAI,aAAa,CACtB,+BAA+B,OAAO,CAAC,QAAQ,oCAAoC,EACnF,cAAc,CACd,CAAC;YACH,CAAC;YACD,0BAA0B,CAAC,OAAO,CAAC,CAAC;YACpC,SAAS;QACV,CAAC;QAED,0BAA0B,CAAC,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,QAAQ,CAAC,CAAC;QACzE,OAAO,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,aAAa,EAAE,CAAC;IAC/D,CAAC;AAAA,CACD","sourcesContent":["/**\n * SSRF guardrails for webfetch (patterns copied from Jina Reader — not vendored).\n *\n * - Only `http:`/`https:` targets are allowed.\n * - Literal-IP targets are checked against non-public CIDRs.\n * - Hostnames are resolved via DNS and **every** returned address is re-checked,\n *   including on every redirect hop (TOCTOU: re-validate on redirect).\n * - Redirect chains are capped, response bytes are capped (with a\n *   `Content-Length` pre-check plus a streaming cutoff), and each hop has a\n *   per-request timeout.\n */\n\nimport { lookup as dnsLookupAll } from \"node:dns/promises\";\nimport { BlockList, isIP } from \"node:net\";\nimport { request as undiciRequest } from \"undici\";\n\nexport const WEBFETCH_MAX_REDIRECTS = 6;\nexport const WEBFETCH_MAX_BYTES = 2_000_000;\nexport const WEBFETCH_REQUEST_TIMEOUT_MS = 30_000;\nexport const WEBFETCH_FIRECRAWL_MAX_LIMIT = 100;\n\nexport type WebfetchErrorCode =\n\t| \"invalid-url\"\n\t| \"blocked-host\"\n\t| \"redirect-limit\"\n\t| \"byte-limit\"\n\t| \"timeout\"\n\t| \"empty\"\n\t| \"transport\"\n\t| \"unavailable\";\n\nexport class WebfetchError extends Error {\n\treadonly code: WebfetchErrorCode;\n\tconstructor(message: string, code: WebfetchErrorCode) {\n\t\tsuper(message);\n\t\tthis.name = \"WebfetchError\";\n\t\tthis.code = code;\n\t}\n}\n\nconst blockedRanges: Array<{ subnet: string; prefix: number; type: \"ipv4\" | \"ipv6\" }> = [\n\t{ subnet: \"10.0.0.0\", prefix: 8, type: \"ipv4\" },\n\t{ subnet: \"172.16.0.0\", prefix: 12, type: \"ipv4\" },\n\t{ subnet: \"192.168.0.0\", prefix: 16, type: \"ipv4\" },\n\t{ subnet: \"127.0.0.0\", prefix: 8, type: \"ipv4\" },\n\t{ subnet: \"169.254.0.0\", prefix: 16, type: \"ipv4\" },\n\t{ subnet: \"224.0.0.0\", prefix: 4, type: \"ipv4\" },\n\t{ subnet: \"100.64.0.0\", prefix: 10, type: \"ipv4\" },\n\t{ subnet: \"fc00::\", prefix: 7, type: \"ipv6\" },\n\t{ subnet: \"fe80::\", prefix: 10, type: \"ipv6\" },\n\t{ subnet: \"ff00::\", prefix: 8, type: \"ipv6\" },\n\t{ subnet: \"::1\", prefix: 128, type: \"ipv6\" },\n];\n\nconst blockedList = new BlockList();\nfor (const range of blockedRanges) {\n\tblockedList.addSubnet(range.subnet, range.prefix, range.type);\n}\n\n/** True when `ip` is a literal IP address inside a blocked non-public range. Non-IP strings return false. */\nexport function isBlockedIpAddress(ip: string): boolean {\n\tconst family = isIP(ip);\n\tif (family === 0) return false;\n\treturn blockedList.check(ip, family === 4 ? \"ipv4\" : \"ipv6\");\n}\n\n/** Parse and validate a web target URL. Only `http:`/`https:` are allowed. */\nexport function normalizeWebUrl(rawUrl: string): URL {\n\tlet url: URL;\n\ttry {\n\t\turl = new URL(rawUrl.trim());\n\t} catch {\n\t\tthrow new WebfetchError(`Invalid URL: ${rawUrl}`, \"invalid-url\");\n\t}\n\tif (url.protocol !== \"http:\" && url.protocol !== \"https:\") {\n\t\tthrow new WebfetchError(`Blocked URL scheme \"${url.protocol}\": only http and https are allowed`, \"blocked-host\");\n\t}\n\treturn url;\n}\n\n/** Reject literal-IP targets inside blocked ranges. Hostnames are checked via DNS separately. */\nexport function assertLiteralTargetAllowed(url: URL): void {\n\tconst bare = url.hostname.replace(/^\\[|\\]$/g, \"\");\n\tif (isIP(bare) !== 0 && isBlockedIpAddress(bare)) {\n\t\tthrow new WebfetchError(`Blocked request to private IP ${bare}`, \"blocked-host\");\n\t}\n}\n\n/** Reject when any resolved address is blocked. Every address is checked (fail closed). */\nexport function assertResolvedAddressesAllowed(addresses: string[], hostname: string): void {\n\tif (addresses.length === 0) {\n\t\tthrow new WebfetchError(`Could not resolve host \"${hostname}\"`, \"blocked-host\");\n\t}\n\tfor (const address of addresses) {\n\t\tif (isBlockedIpAddress(address)) {\n\t\t\tthrow new WebfetchError(`Blocked request to \"${hostname}\": resolves to private IP ${address}`, \"blocked-host\");\n\t\t}\n\t}\n}\n\nexport type DnsLookup = (hostname: string) => Promise<string[]>;\n\nasync function defaultLookup(hostname: string): Promise<string[]> {\n\tconst results = await dnsLookupAll(hostname, { all: true });\n\treturn results.map((result) => result.address);\n}\n\n/**\n * Fail-closed host check: literal IPs are checked directly, hostnames are\n * resolved and every returned address is re-checked. Callers must invoke this\n * for the initial URL and again for each redirect target.\n */\nexport async function assertHostAllowed(hostname: string, lookup: DnsLookup = defaultLookup): Promise<void> {\n\tconst bare = hostname.replace(/^\\[|\\]$/g, \"\");\n\tif (isIP(bare) !== 0) {\n\t\tif (isBlockedIpAddress(bare)) {\n\t\t\tthrow new WebfetchError(`Blocked request to private IP ${bare}`, \"blocked-host\");\n\t\t}\n\t\treturn;\n\t}\n\tlet addresses: string[];\n\ttry {\n\t\taddresses = await lookup(bare);\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\tthrow new WebfetchError(`Could not resolve host \"${bare}\": ${reason}`, \"blocked-host\");\n\t}\n\tassertResolvedAddressesAllowed(addresses, bare);\n}\n\n/** Enforce the byte cap against a `Content-Length` header before streaming the body. */\nexport function assertContentLengthAllowed(\n\tcontentLength: string | null | undefined,\n\tmaxBytes: number = WEBFETCH_MAX_BYTES,\n): void {\n\tif (contentLength === null || contentLength === undefined) return;\n\tconst declared = Number.parseInt(contentLength.split(\",\")[0]?.trim() ?? \"\", 10);\n\tif (Number.isFinite(declared) && declared > maxBytes) {\n\t\tthrow new WebfetchError(\n\t\t\t`Response body exceeds ${maxBytes} byte limit (${declared} bytes declared)`,\n\t\t\t\"byte-limit\",\n\t\t);\n\t}\n}\n\n/** Enforce the byte cap while streaming. Throws once `totalBytes` exceeds `maxBytes`. */\nexport function assertBytesWithinBudget(totalBytes: number, maxBytes: number = WEBFETCH_MAX_BYTES): void {\n\tif (totalBytes > maxBytes) {\n\t\tthrow new WebfetchError(`Response body exceeds ${maxBytes} byte limit`, \"byte-limit\");\n\t}\n}\n\n/** Clamp a search limit into `[min, max]`, falling back to `fallback` for missing/invalid values. */\nexport function clampLimit(limit: number | undefined, options: { fallback: number; min: number; max: number }): number {\n\tif (limit === undefined || !Number.isFinite(limit)) return options.fallback;\n\treturn Math.min(options.max, Math.max(options.min, Math.floor(limit)));\n}\n\nexport interface GuardedBody {\n\tstatusCode: number;\n\theaders: Record<string, string>;\n\tbody: AsyncIterable<Uint8Array>;\n}\n\nexport type LowLevelRequestFn = (\n\turl: string,\n\tinit: { headers: Record<string, string>; signal: AbortSignal },\n) => Promise<GuardedBody>;\n\nasync function defaultLowLevelRequest(\n\turl: string,\n\tinit: { headers: Record<string, string>; signal: AbortSignal },\n): Promise<GuardedBody> {\n\tconst response = await undiciRequest(url, { method: \"GET\", headers: init.headers, signal: init.signal });\n\tconst headers: Record<string, string> = {};\n\tfor (const [key, value] of Object.entries(response.headers)) {\n\t\tif (value === undefined) continue;\n\t\theaders[key.toLowerCase()] = Array.isArray(value) ? value.join(\", \") : String(value);\n\t}\n\treturn { statusCode: response.statusCode, headers, body: response.body as AsyncIterable<Uint8Array> };\n}\n\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\n\nexport interface GuardedRequestOptions {\n\theaders?: Record<string, string>;\n\ttimeoutMs?: number;\n\tmaxBytes?: number;\n\tmaxRedirects?: number;\n\tsignal?: AbortSignal;\n\tlookup?: DnsLookup;\n\trequest?: LowLevelRequestFn;\n}\n\nexport interface GuardedResponse extends GuardedBody {\n\tfinalUrl: string;\n\tredirectCount: number;\n}\n\n/**\n * GET a URL under the shared guardrails: scheme allowlist, literal-IP and DNS\n * checks on the initial URL and every redirect hop, redirect cap,\n * `Content-Length` pre-check, and per-hop timeout. Streaming byte cutoff is\n * the caller's job via `assertBytesWithinBudget`.\n */\nexport async function guardedRequest(rawUrl: string, options: GuardedRequestOptions = {}): Promise<GuardedResponse> {\n\tconst timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;\n\tconst maxBytes = options.maxBytes ?? WEBFETCH_MAX_BYTES;\n\tconst maxRedirects = options.maxRedirects ?? WEBFETCH_MAX_REDIRECTS;\n\tconst lookup = options.lookup ?? defaultLookup;\n\tconst request = options.request ?? defaultLowLevelRequest;\n\tconst headers = options.headers ?? {};\n\n\tlet current = normalizeWebUrl(rawUrl);\n\tassertLiteralTargetAllowed(current);\n\n\tfor (let redirectCount = 0; ; redirectCount++) {\n\t\tawait assertHostAllowed(current.hostname, lookup);\n\n\t\tconst timeoutSignal = AbortSignal.timeout(timeoutMs);\n\t\tconst signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;\n\t\tlet response: GuardedBody;\n\t\ttry {\n\t\t\tresponse = await request(current.href, { headers, signal });\n\t\t} catch (error) {\n\t\t\tif (timeoutSignal.aborted && !options.signal?.aborted) {\n\t\t\t\tthrow new WebfetchError(`Request to ${current.href} timed out after ${timeoutMs}ms`, \"timeout\");\n\t\t\t}\n\t\t\tif (options.signal?.aborted) {\n\t\t\t\tthrow new Error(\"Operation aborted\");\n\t\t\t}\n\t\t\tthrow error;\n\t\t}\n\n\t\tconst location = response.headers.location ?? response.headers.Location;\n\t\tif (location !== undefined && REDIRECT_STATUSES.has(response.statusCode)) {\n\t\t\tif (redirectCount >= maxRedirects) {\n\t\t\t\tthrow new WebfetchError(`Redirect chain exceeds ${maxRedirects} hops`, \"redirect-limit\");\n\t\t\t}\n\t\t\ttry {\n\t\t\t\tcurrent = new URL(location, current.href);\n\t\t\t} catch {\n\t\t\t\tthrow new WebfetchError(`Invalid redirect target: ${location}`, \"invalid-url\");\n\t\t\t}\n\t\t\tif (current.protocol !== \"http:\" && current.protocol !== \"https:\") {\n\t\t\t\tthrow new WebfetchError(\n\t\t\t\t\t`Blocked redirect to scheme \"${current.protocol}\": only http and https are allowed`,\n\t\t\t\t\t\"blocked-host\",\n\t\t\t\t);\n\t\t\t}\n\t\t\tassertLiteralTargetAllowed(current);\n\t\t\tcontinue;\n\t\t}\n\n\t\tassertContentLengthAllowed(response.headers[\"content-length\"], maxBytes);\n\t\treturn { ...response, finalUrl: current.href, redirectCount };\n\t}\n}\n"]}

package/dist/core/webfetch/robots.d.ts

@@ -0,0 +1,46 @@
+/**
+ * robots.txt parsing for the polite crawler.
+ *
+ * Follows RFC 9309 where it matters: the group whose user-agent matches ours
+ * (or `*` when none does) supplies the rules, the longest matching pattern wins,
+ * and an Allow beats a Disallow of the same length. A missing or unreadable
+ * robots.txt means "allowed", which is what the RFC requires and what a crawler
+ * must assume when the file is absent.
+ *
+ * The file is fetched through the shared guardrails, so a hostile robots.txt
+ * cannot redirect the crawler somewhere it was already barred from going.
+ */
+import { type DnsLookup, type LowLevelRequestFn } from "./guardrails.ts";
+export declare const ROBOTS_USER_AGENT = "ion";
+export interface RobotsRule {
+    allow: boolean;
+    pattern: string;
+    regex: RegExp;
+    length: number;
+}
+export interface RobotsPolicy {
+    rules: RobotsRule[];
+    /** Seconds requested by Crawl-delay, when the file declares one. */
+    crawlDelaySeconds?: number;
+    sitemaps: string[];
+}
+/** Parse a robots.txt body into a policy for {@link ROBOTS_USER_AGENT}. */
+export declare function parseRobotsTxt(text: string, userAgent?: string): RobotsPolicy;
+/** Longest matching pattern wins; an Allow wins a tie. No rules means allowed. */
+export declare function isPathAllowed(policy: RobotsPolicy, pathWithQuery: string): boolean;
+export declare function isUrlAllowed(policy: RobotsPolicy, url: URL): boolean;
+export interface RobotsFetchOptions {
+    userAgent?: string;
+    signal?: AbortSignal;
+    lookup?: DnsLookup;
+    request?: LowLevelRequestFn;
+    maxBytes?: number;
+}
+/**
+ * Fetch and parse a site's robots.txt. Returns an allow-everything policy when
+ * the file is missing, unreadable, or not text: a crawler must not refuse to
+ * read a site merely because the site failed to describe its own rules, and it
+ * must not guess rules it was not given.
+ */
+export declare function fetchRobotsPolicy(originUrl: URL, options?: RobotsFetchOptions): Promise<RobotsPolicy>;
+//# sourceMappingURL=robots.d.ts.map

package/dist/core/webfetch/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../../src/core/webfetch/robots.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAEN,KAAK,SAAS,EAEd,KAAK,iBAAiB,EAGtB,MAAM,iBAAiB,CAAC;AAEzB,eAAO,MAAM,iBAAiB,QAAQ,CAAC;AAEvC,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACnB;AAiBD,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAA0B,GAAG,YAAY,CAgEhG;AAED,kFAAkF;AAClF,wBAAgB,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CASlF;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAEpE;AAED,MAAM,WAAW,kBAAkB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,YAAY,CAAC,CA+B/G","sourcesContent":["/**\n * robots.txt parsing for the polite crawler.\n *\n * Follows RFC 9309 where it matters: the group whose user-agent matches ours\n * (or `*` when none does) supplies the rules, the longest matching pattern wins,\n * and an Allow beats a Disallow of the same length. A missing or unreadable\n * robots.txt means \"allowed\", which is what the RFC requires and what a crawler\n * must assume when the file is absent.\n *\n * The file is fetched through the shared guardrails, so a hostile robots.txt\n * cannot redirect the crawler somewhere it was already barred from going.\n */\n\nimport {\n\tassertBytesWithinBudget,\n\ttype DnsLookup,\n\tguardedRequest,\n\ttype LowLevelRequestFn,\n\tWEBFETCH_MAX_BYTES,\n\tWEBFETCH_REQUEST_TIMEOUT_MS,\n} from \"./guardrails.ts\";\n\nexport const ROBOTS_USER_AGENT = \"ion\";\n\nexport interface RobotsRule {\n\tallow: boolean;\n\tpattern: string;\n\tregex: RegExp;\n\tlength: number;\n}\n\nexport interface RobotsPolicy {\n\trules: RobotsRule[];\n\t/** Seconds requested by Crawl-delay, when the file declares one. */\n\tcrawlDelaySeconds?: number;\n\tsitemaps: string[];\n}\n\nfunction patternToRegex(pattern: string): { regex: RegExp; anchored: boolean } | undefined {\n\tif (pattern.length === 0) return undefined;\n\tconst anchored = pattern.endsWith(\"$\");\n\tconst body = anchored ? pattern.slice(0, -1) : pattern;\n\tlet source = \"\";\n\tfor (const character of body) {\n\t\tif (character === \"*\") {\n\t\t\tsource += \".*\";\n\t\t} else {\n\t\t\tsource += character.replace(/[.+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\t\t}\n\t}\n\treturn { regex: new RegExp(`^${source}${anchored ? \"$\" : \"\"}`), anchored };\n}\n\n/** Parse a robots.txt body into a policy for {@link ROBOTS_USER_AGENT}. */\nexport function parseRobotsTxt(text: string, userAgent: string = ROBOTS_USER_AGENT): RobotsPolicy {\n\tconst agent = userAgent.toLowerCase();\n\tconst groups: { agents: string[]; lines: string[] }[] = [];\n\tlet current: { agents: string[]; lines: string[] } | undefined;\n\tlet expectingAgents = false;\n\n\tfor (const rawLine of text.split(/\\r?\\n/)) {\n\t\tconst line = rawLine.replace(/#.*$/, \"\").trim();\n\t\tif (line.length === 0) continue;\n\t\tconst separator = line.indexOf(\":\");\n\t\tif (separator === -1) continue;\n\t\tconst field = line.slice(0, separator).trim().toLowerCase();\n\t\tconst value = line.slice(separator + 1).trim();\n\n\t\tif (field === \"user-agent\") {\n\t\t\tif (!current || !expectingAgents) {\n\t\t\t\tcurrent = { agents: [], lines: [] };\n\t\t\t\tgroups.push(current);\n\t\t\t\texpectingAgents = true;\n\t\t\t}\n\t\t\tcurrent.agents.push(value.toLowerCase());\n\t\t\tcontinue;\n\t\t}\n\n\t\t// Any non-user-agent field ends the agent block, per RFC 9309.\n\t\texpectingAgents = false;\n\t\tif (!current) continue;\n\t\tcurrent.lines.push(`${field}:${value}`);\n\t}\n\n\tconst matches = (group: { agents: string[] }) =>\n\t\tgroup.agents.some((candidate) => candidate !== \"*\" && agent.includes(candidate));\n\tconst specific = groups.find(matches);\n\tconst wildcard = groups.find((group) => group.agents.includes(\"*\"));\n\tconst selected = specific ?? wildcard;\n\n\tconst policy: RobotsPolicy = { rules: [], sitemaps: [] };\n\tif (!selected) return policy;\n\n\tfor (const entry of selected.lines) {\n\t\tconst separator = entry.indexOf(\":\");\n\t\tconst field = entry.slice(0, separator);\n\t\tconst value = entry.slice(separator + 1);\n\t\tif (field === \"sitemap\") {\n\t\t\tif (value.length > 0) policy.sitemaps.push(value);\n\t\t\tcontinue;\n\t\t}\n\t\tif (field === \"crawl-delay\") {\n\t\t\tconst seconds = Number.parseFloat(value);\n\t\t\tif (Number.isFinite(seconds) && seconds >= 0) policy.crawlDelaySeconds = seconds;\n\t\t\tcontinue;\n\t\t}\n\t\tif (field !== \"allow\" && field !== \"disallow\") continue;\n\t\tconst compiled = patternToRegex(value);\n\t\t// `Disallow:` with an empty value means allow everything, so it adds no rule.\n\t\tif (!compiled) continue;\n\t\tpolicy.rules.push({\n\t\t\tallow: field === \"allow\",\n\t\t\tpattern: value,\n\t\t\tregex: compiled.regex,\n\t\t\tlength: value.length,\n\t\t});\n\t}\n\treturn policy;\n}\n\n/** Longest matching pattern wins; an Allow wins a tie. No rules means allowed. */\nexport function isPathAllowed(policy: RobotsPolicy, pathWithQuery: string): boolean {\n\tlet best: RobotsRule | undefined;\n\tfor (const rule of policy.rules) {\n\t\tif (!rule.regex.test(pathWithQuery)) continue;\n\t\tif (!best || rule.length > best.length || (rule.length === best.length && rule.allow && !best.allow)) {\n\t\t\tbest = rule;\n\t\t}\n\t}\n\treturn best ? best.allow : true;\n}\n\nexport function isUrlAllowed(policy: RobotsPolicy, url: URL): boolean {\n\treturn isPathAllowed(policy, `${url.pathname}${url.search}`);\n}\n\nexport interface RobotsFetchOptions {\n\tuserAgent?: string;\n\tsignal?: AbortSignal;\n\tlookup?: DnsLookup;\n\trequest?: LowLevelRequestFn;\n\tmaxBytes?: number;\n}\n\n/**\n * Fetch and parse a site's robots.txt. Returns an allow-everything policy when\n * the file is missing, unreadable, or not text: a crawler must not refuse to\n * read a site merely because the site failed to describe its own rules, and it\n * must not guess rules it was not given.\n */\nexport async function fetchRobotsPolicy(originUrl: URL, options: RobotsFetchOptions = {}): Promise<RobotsPolicy> {\n\tconst allowAll: RobotsPolicy = { rules: [], sitemaps: [] };\n\tconst robotsUrl = new URL(\"/robots.txt\", originUrl.origin);\n\ttry {\n\t\tconst response = await guardedRequest(robotsUrl.href, {\n\t\t\theaders: { \"User-Agent\": options.userAgent ?? ROBOTS_USER_AGENT },\n\t\t\tsignal: options.signal,\n\t\t\tlookup: options.lookup,\n\t\t\trequest: options.request,\n\t\t\tmaxBytes: options.maxBytes ?? WEBFETCH_MAX_BYTES,\n\t\t\ttimeoutMs: WEBFETCH_REQUEST_TIMEOUT_MS,\n\t\t});\n\t\t// 4xx means no published rules; only a served body is authoritative.\n\t\tif (response.statusCode >= 400) return allowAll;\n\t\tconst chunks: Uint8Array[] = [];\n\t\tlet totalBytes = 0;\n\t\tfor await (const chunk of response.body) {\n\t\t\ttotalBytes += chunk.length;\n\t\t\tassertBytesWithinBudget(totalBytes, options.maxBytes ?? WEBFETCH_MAX_BYTES);\n\t\t\tchunks.push(chunk);\n\t\t}\n\t\tconst merged = new Uint8Array(totalBytes);\n\t\tlet offset = 0;\n\t\tfor (const chunk of chunks) {\n\t\t\tmerged.set(chunk, offset);\n\t\t\toffset += chunk.length;\n\t\t}\n\t\treturn parseRobotsTxt(new TextDecoder().decode(merged), options.userAgent ?? ROBOTS_USER_AGENT);\n\t} catch {\n\t\treturn allowAll;\n\t}\n}\n"]}