@x12i/api-hub-safe-fetch 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,11 @@
-# Temporary Holding Version
+# @x12i/api-hub-safe-fetch
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+SSRF-safe HTTP fetch for the x12i API hub
+
+ESM package for Node.js 24 or newer. Import the public name:
+
+```js
+import * as safeFetch from '@x12i/api-hub-safe-fetch';
+```
+
+The whole set is also available from [`@x12i/api-hub`](https://www.npmjs.com/package/@x12i/api-hub).

package/dist/address.d.ts

@@ -0,0 +1,2 @@
+/** True only for globally routable unicast addresses. Anything that is not an IP literal is not public. */
+export declare function isPublicAddress(ip: string): boolean;

package/dist/address.js

@@ -0,0 +1,79 @@
+import { BlockList, isIP } from 'node:net';
+const blocked = new BlockList();
+const V4 = [
+    ['0.0.0.0', 8], // "this" network
+    ['10.0.0.0', 8],
+    ['100.64.0.0', 10], // CGNAT
+    ['127.0.0.0', 8],
+    ['169.254.0.0', 16], // link-local, cloud metadata
+    ['172.16.0.0', 12],
+    ['192.0.0.0', 24], // IETF protocol assignments
+    ['192.0.2.0', 24], // TEST-NET-1
+    ['192.88.99.0', 24], // 6to4 relay anycast
+    ['192.168.0.0', 16],
+    ['198.18.0.0', 15], // benchmarking
+    ['198.51.100.0', 24], // TEST-NET-2
+    ['203.0.113.0', 24], // TEST-NET-3
+    ['224.0.0.0', 4], // multicast
+    ['240.0.0.0', 4], // reserved, broadcast
+];
+const V6 = [
+    ['::', 128], // unspecified
+    ['::1', 128], // loopback
+    ['::', 96], // IPv4-compatible (deprecated)
+    ['64:ff9b::', 96], // NAT64
+    ['64:ff9b:1::', 48], // local-use NAT64
+    ['100::', 64], // discard
+    ['2001::', 32], // Teredo
+    ['2001:2::', 48], // benchmarking
+    ['2001:db8::', 32], // documentation
+    ['2001:10::', 28], // ORCHID
+    ['2002::', 16], // 6to4 (can embed any IPv4 address)
+    ['3fff::', 20], // documentation
+    ['5f00::', 16], // SRv6 SIDs
+    ['fc00::', 7], // unique local
+    ['fe80::', 10], // link-local
+    ['fec0::', 10], // site-local (deprecated)
+    ['ff00::', 8], // multicast
+];
+for (const [net, prefix] of V4)
+    blocked.addSubnet(net, prefix, 'ipv4');
+for (const [net, prefix] of V6)
+    blocked.addSubnet(net, prefix, 'ipv6');
+/** Eight 16-bit groups for any valid IPv6 literal, including embedded dotted IPv4 and zone ids. */
+function expandV6(ip) {
+    let text = ip.toLowerCase().replace(/%.*$/, '');
+    const dotted = /(\d+\.\d+\.\d+\.\d+)$/.exec(text);
+    if (dotted) {
+        const [a, b, c, d] = dotted[1].split('.').map(Number);
+        text = `${text.slice(0, -dotted[1].length)}${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`;
+    }
+    const [head, tail] = text.split('::');
+    const left = head ? head.split(':') : [];
+    const right = tail ? tail.split(':') : [];
+    const fill = tail === undefined ? [] : Array(8 - left.length - right.length).fill('0');
+    return [...left, ...fill, ...right].map((h) => parseInt(h || '0', 16));
+}
+function embeddedV4(ip) {
+    const g = expandV6(ip);
+    const mapped = g.slice(0, 5).every((x) => x === 0) && g[5] === 0xffff;
+    if (!mapped)
+        return undefined;
+    const hi = g[6];
+    const lo = g[7];
+    return `${hi >> 8}.${hi & 255}.${lo >> 8}.${lo & 255}`;
+}
+/** True only for globally routable unicast addresses. Anything that is not an IP literal is not public. */
+export function isPublicAddress(ip) {
+    const family = isIP(ip);
+    if (family === 4)
+        return !blocked.check(ip, 'ipv4');
+    if (family === 6) {
+        const v4 = embeddedV4(ip);
+        if (v4 !== undefined)
+            return isIP(v4) === 4 && !blocked.check(v4, 'ipv4');
+        return !blocked.check(ip, 'ipv6');
+    }
+    return false;
+}
+//# sourceMappingURL=address.js.map

package/dist/address.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"address.js","sourceRoot":"","sources":["../../../../../packages/safe-fetch/src/address.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,OAAO,GAAG,IAAI,SAAS,EAAE,CAAC;AAEhC,MAAM,EAAE,GAAuB;IAC7B,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,iBAAiB;IACjC,CAAC,UAAU,EAAE,CAAC,CAAC;IACf,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,QAAQ;IAC5B,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,aAAa,EAAE,EAAE,CAAC,EAAE,6BAA6B;IAClD,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE,4BAA4B;IAC/C,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE,aAAa;IAChC,CAAC,aAAa,EAAE,EAAE,CAAC,EAAE,qBAAqB;IAC1C,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,eAAe;IACnC,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,aAAa;IACnC,CAAC,aAAa,EAAE,EAAE,CAAC,EAAE,aAAa;IAClC,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,YAAY;IAC9B,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,sBAAsB;CACzC,CAAC;AAEF,MAAM,EAAE,GAAuB;IAC7B,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,cAAc;IAC3B,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,WAAW;IACzB,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,+BAA+B;IAC3C,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE,QAAQ;IAC3B,CAAC,aAAa,EAAE,EAAE,CAAC,EAAE,kBAAkB;IACvC,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,UAAU;IACzB,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,SAAS;IACzB,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,eAAe;IACjC,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,gBAAgB;IACpC,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS;IAC5B,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,oCAAoC;IACpD,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,gBAAgB;IAChC,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,YAAY;IAC5B,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,eAAe;IAC9B,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,aAAa;IAC7B,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,0BAA0B;IAC1C,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,YAAY;CAC5B,CAAC;AAEF,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE;IAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACvE,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE;IAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAEvE,mGAAmG;AACnG,SAAS,QAAQ,CAAC,EAAU;IAC1B,IAAI,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAqC,CAAC;QAC3F,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;IAC7G,CAAC;IACD,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiC,CAAC;IACtE,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAS,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/F,OAAO,CAAC,GAAG,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,UAAU,CAAC,EAAU;IAC5B,MAAM,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC;IACtE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACjB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACjB,OAAO,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,EAAE,CAAC;AACzD,CAAC;AAED,2GAA2G;AAC3G,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC1E,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC","sourcesContent":["import { BlockList, isIP } from 'node:net';\n\nconst blocked = new BlockList();\n\nconst V4: [string, number][] = [\n  ['0.0.0.0', 8], // \"this\" network\n  ['10.0.0.0', 8],\n  ['100.64.0.0', 10], // CGNAT\n  ['127.0.0.0', 8],\n  ['169.254.0.0', 16], // link-local, cloud metadata\n  ['172.16.0.0', 12],\n  ['192.0.0.0', 24], // IETF protocol assignments\n  ['192.0.2.0', 24], // TEST-NET-1\n  ['192.88.99.0', 24], // 6to4 relay anycast\n  ['192.168.0.0', 16],\n  ['198.18.0.0', 15], // benchmarking\n  ['198.51.100.0', 24], // TEST-NET-2\n  ['203.0.113.0', 24], // TEST-NET-3\n  ['224.0.0.0', 4], // multicast\n  ['240.0.0.0', 4], // reserved, broadcast\n];\n\nconst V6: [string, number][] = [\n  ['::', 128], // unspecified\n  ['::1', 128], // loopback\n  ['::', 96], // IPv4-compatible (deprecated)\n  ['64:ff9b::', 96], // NAT64\n  ['64:ff9b:1::', 48], // local-use NAT64\n  ['100::', 64], // discard\n  ['2001::', 32], // Teredo\n  ['2001:2::', 48], // benchmarking\n  ['2001:db8::', 32], // documentation\n  ['2001:10::', 28], // ORCHID\n  ['2002::', 16], // 6to4 (can embed any IPv4 address)\n  ['3fff::', 20], // documentation\n  ['5f00::', 16], // SRv6 SIDs\n  ['fc00::', 7], // unique local\n  ['fe80::', 10], // link-local\n  ['fec0::', 10], // site-local (deprecated)\n  ['ff00::', 8], // multicast\n];\n\nfor (const [net, prefix] of V4) blocked.addSubnet(net, prefix, 'ipv4');\nfor (const [net, prefix] of V6) blocked.addSubnet(net, prefix, 'ipv6');\n\n/** Eight 16-bit groups for any valid IPv6 literal, including embedded dotted IPv4 and zone ids. */\nfunction expandV6(ip: string): number[] {\n  let text = ip.toLowerCase().replace(/%.*$/, '');\n  const dotted = /(\\d+\\.\\d+\\.\\d+\\.\\d+)$/.exec(text);\n  if (dotted) {\n    const [a, b, c, d] = dotted[1]!.split('.').map(Number) as [number, number, number, number];\n    text = `${text.slice(0, -dotted[1]!.length)}${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`;\n  }\n  const [head, tail] = text.split('::') as [string, string | undefined];\n  const left = head ? head.split(':') : [];\n  const right = tail ? tail.split(':') : [];\n  const fill = tail === undefined ? [] : Array<string>(8 - left.length - right.length).fill('0');\n  return [...left, ...fill, ...right].map((h) => parseInt(h || '0', 16));\n}\n\nfunction embeddedV4(ip: string): string | undefined {\n  const g = expandV6(ip);\n  const mapped = g.slice(0, 5).every((x) => x === 0) && g[5] === 0xffff;\n  if (!mapped) return undefined;\n  const hi = g[6]!;\n  const lo = g[7]!;\n  return `${hi >> 8}.${hi & 255}.${lo >> 8}.${lo & 255}`;\n}\n\n/** True only for globally routable unicast addresses. Anything that is not an IP literal is not public. */\nexport function isPublicAddress(ip: string): boolean {\n  const family = isIP(ip);\n  if (family === 4) return !blocked.check(ip, 'ipv4');\n  if (family === 6) {\n    const v4 = embeddedV4(ip);\n    if (v4 !== undefined) return isIP(v4) === 4 && !blocked.check(v4, 'ipv4');\n    return !blocked.check(ip, 'ipv6');\n  }\n  return false;\n}\n"]}

package/dist/fake.d.ts

@@ -0,0 +1,18 @@
+import { HubError, type SafeFetch, type SafeFetchOptions } from '@x12i/api-hub-core';
+export interface FakeRoute {
+    status?: number;
+    headers?: Record<string, string>;
+    body?: string | Uint8Array;
+    /** Shorthand for a JSON body with `content-type: application/json`. */
+    json?: unknown;
+    /** Simulated failure instead of a response. */
+    error?: HubError;
+}
+export type FakeRoutes = Record<string, FakeRoute | ((options: SafeFetchOptions) => FakeRoute)>;
+/** Recorded-response fetcher for capability tests. Follows redirects like the real one; unknown URLs fail. */
+export declare function fakeFetch(routes: FakeRoutes): SafeFetch & {
+    calls: {
+        url: string;
+        options: SafeFetchOptions;
+    }[];
+};

package/dist/fake.js

@@ -0,0 +1,65 @@
+import { HubError } from '@x12i/api-hub-core';
+import { withoutCredentials } from "./fetch.js";
+const REDIRECTS = new Set([301, 302, 303, 307, 308]);
+/** Recorded-response fetcher for capability tests. Follows redirects like the real one; unknown URLs fail. */
+export function fakeFetch(routes) {
+    const calls = [];
+    const fn = async (rawUrl, options = {}) => {
+        let url;
+        try {
+            url = new URL(rawUrl).toString();
+        }
+        catch {
+            throw new HubError('invalid_input', 'Not a valid URL');
+        }
+        const maxRedirects = options.maxRedirects ?? 5;
+        const redirects = [];
+        for (;;) {
+            calls.push({ url, options });
+            const entry = routes[url];
+            if (!entry)
+                throw new HubError('upstream_unavailable', 'Could not fetch the URL', { reason: 'ENOTFOUND' });
+            const route = typeof entry === 'function' ? entry(options) : entry;
+            if (route.error)
+                throw route.error;
+            const status = route.status ?? 200;
+            const headers = Object.fromEntries(Object.entries(route.headers ?? {}).map(([k, v]) => [k.toLowerCase(), v]));
+            if (route.json !== undefined)
+                headers['content-type'] ??= 'application/json';
+            if (REDIRECTS.has(status) && headers.location && maxRedirects > 0) {
+                if (redirects.length >= maxRedirects)
+                    throw new HubError('limit_exceeded', `More than ${maxRedirects} redirects`);
+                redirects.push(url);
+                const next = new URL(headers.location, url);
+                if (next.origin !== new URL(url).origin)
+                    options = { ...options, headers: withoutCredentials(options.headers) };
+                url = next.toString();
+                continue;
+            }
+            const raw = route.json !== undefined ? JSON.stringify(route.json) : route.body;
+            let bytes = typeof raw === 'string' ? new TextEncoder().encode(raw) : (raw ?? new Uint8Array());
+            if (options.method === 'HEAD')
+                bytes = new Uint8Array();
+            const maxBytes = options.maxBytes ?? 5 * 1024 * 1024;
+            const truncated = bytes.length > maxBytes;
+            if (truncated && !options.truncate)
+                throw new HubError('limit_exceeded', `Response is larger than ${maxBytes} bytes`);
+            if (truncated)
+                bytes = bytes.slice(0, maxBytes);
+            return {
+                requestedUrl: rawUrl,
+                url,
+                status,
+                headers,
+                redirects,
+                remoteAddress: '203.0.113.10',
+                body: bytes,
+                truncated,
+                text: () => Buffer.from(bytes).toString('utf8'),
+                json: () => JSON.parse(Buffer.from(bytes).toString('utf8')),
+            };
+        }
+    };
+    return Object.assign(fn, { calls });
+}
+//# sourceMappingURL=fake.js.map

package/dist/fake.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fake.js","sourceRoot":"","sources":["../../../../../packages/safe-fetch/src/fake.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAA4D,MAAM,WAAW,CAAC;AAC/F,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAchD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAErD,8GAA8G;AAC9G,MAAM,UAAU,SAAS,CAAC,MAAkB;IAC1C,MAAM,KAAK,GAAiD,EAAE,CAAC;IAC/D,MAAM,EAAE,GAAG,KAAK,EAAE,MAAc,EAAE,UAA4B,EAAE,EAAyB,EAAE;QACzF,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,QAAQ,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,SAAS,CAAC;YACR,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,QAAQ,CAAC,sBAAsB,EAAE,yBAAyB,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YAC3G,MAAM,KAAK,GAAG,OAAO,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YACnE,IAAI,KAAK,CAAC,KAAK;gBAAE,MAAM,KAAK,CAAC,KAAK,CAAC;YACnC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9G,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,CAAC,cAAc,CAAC,KAAK,kBAAkB,CAAC;YAC7E,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,QAAQ,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;gBAClE,IAAI,SAAS,CAAC,MAAM,IAAI,YAAY;oBAAE,MAAM,IAAI,QAAQ,CAAC,gBAAgB,EAAE,aAAa,YAAY,YAAY,CAAC,CAAC;gBAClH,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC5C,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM;oBAAE,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChH,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;YAC/E,IAAI,KAAK,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC;YAChG,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;gBAAE,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;YACrD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;YAC1C,IAAI,SAAS,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,MAAM,IAAI,QAAQ,CAAC,gBAAgB,EAAE,2BAA2B,QAAQ,QAAQ,CAAC,CAAC;YACtH,IAAI,SAAS;gBAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YAChD,OAAO;gBACL,YAAY,EAAE,MAAM;gBACpB,GAAG;gBACH,MAAM;gBACN,OAAO;gBACP,SAAS;gBACT,aAAa,EAAE,cAAc;gBAC7B,IAAI,EAAE,KAAK;gBACX,SAAS;gBACT,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC/C,IAAI,EAAE,GAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAM;aACpE,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;AACtC,CAAC","sourcesContent":["import { HubError, type SafeFetch, type SafeFetchOptions, type SafeResponse } from '@x12i/api-hub-core';\nimport { withoutCredentials } from './fetch.ts';\n\nexport interface FakeRoute {\n  status?: number;\n  headers?: Record<string, string>;\n  body?: string | Uint8Array;\n  /** Shorthand for a JSON body with `content-type: application/json`. */\n  json?: unknown;\n  /** Simulated failure instead of a response. */\n  error?: HubError;\n}\n\nexport type FakeRoutes = Record<string, FakeRoute | ((options: SafeFetchOptions) => FakeRoute)>;\n\nconst REDIRECTS = new Set([301, 302, 303, 307, 308]);\n\n/** Recorded-response fetcher for capability tests. Follows redirects like the real one; unknown URLs fail. */\nexport function fakeFetch(routes: FakeRoutes): SafeFetch & { calls: { url: string; options: SafeFetchOptions }[] } {\n  const calls: { url: string; options: SafeFetchOptions }[] = [];\n  const fn = async (rawUrl: string, options: SafeFetchOptions = {}): Promise<SafeResponse> => {\n    let url: string;\n    try {\n      url = new URL(rawUrl).toString();\n    } catch {\n      throw new HubError('invalid_input', 'Not a valid URL');\n    }\n    const maxRedirects = options.maxRedirects ?? 5;\n    const redirects: string[] = [];\n    for (;;) {\n      calls.push({ url, options });\n      const entry = routes[url];\n      if (!entry) throw new HubError('upstream_unavailable', 'Could not fetch the URL', { reason: 'ENOTFOUND' });\n      const route = typeof entry === 'function' ? entry(options) : entry;\n      if (route.error) throw route.error;\n      const status = route.status ?? 200;\n      const headers = Object.fromEntries(Object.entries(route.headers ?? {}).map(([k, v]) => [k.toLowerCase(), v]));\n      if (route.json !== undefined) headers['content-type'] ??= 'application/json';\n      if (REDIRECTS.has(status) && headers.location && maxRedirects > 0) {\n        if (redirects.length >= maxRedirects) throw new HubError('limit_exceeded', `More than ${maxRedirects} redirects`);\n        redirects.push(url);\n        const next = new URL(headers.location, url);\n        if (next.origin !== new URL(url).origin) options = { ...options, headers: withoutCredentials(options.headers) };\n        url = next.toString();\n        continue;\n      }\n      const raw = route.json !== undefined ? JSON.stringify(route.json) : route.body;\n      let bytes = typeof raw === 'string' ? new TextEncoder().encode(raw) : (raw ?? new Uint8Array());\n      if (options.method === 'HEAD') bytes = new Uint8Array();\n      const maxBytes = options.maxBytes ?? 5 * 1024 * 1024;\n      const truncated = bytes.length > maxBytes;\n      if (truncated && !options.truncate) throw new HubError('limit_exceeded', `Response is larger than ${maxBytes} bytes`);\n      if (truncated) bytes = bytes.slice(0, maxBytes);\n      return {\n        requestedUrl: rawUrl,\n        url,\n        status,\n        headers,\n        redirects,\n        remoteAddress: '203.0.113.10',\n        body: bytes,\n        truncated,\n        text: () => Buffer.from(bytes).toString('utf8'),\n        json: <T>() => JSON.parse(Buffer.from(bytes).toString('utf8')) as T,\n      };\n    }\n  };\n  return Object.assign(fn, { calls });\n}\n"]}

package/dist/fetch.d.ts

@@ -0,0 +1,22 @@
+import { type SafeFetch } from '@x12i/api-hub-core';
+export interface ResolvedAddress {
+    address: string;
+    family: 4 | 6;
+}
+export type Resolver = (hostname: string) => Promise<ResolvedAddress[]>;
+export interface SafeFetchConfig {
+    /** Default byte cap on the decoded body. */
+    maxBytes?: number;
+    timeoutMs?: number;
+    maxRedirects?: number;
+    allowedPorts?: number[];
+    userAgent?: string;
+    /** Product token matched against robots.txt groups. */
+    robotsToken?: string;
+    resolve?: Resolver;
+    /** Decides whether an address may be contacted. Defaults to public unicast only. */
+    isAllowedAddress?: (address: string, hostname: string) => boolean;
+}
+/** Credentials are never forwarded to a different origin on redirect. */
+export declare function withoutCredentials(headers: Record<string, string> | undefined): Record<string, string> | undefined;
+export declare function createSafeFetch(config?: SafeFetchConfig): SafeFetch;

package/dist/fetch.js

@@ -0,0 +1,259 @@
+import { lookup as dnsLookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+import { Readable } from 'node:stream';
+import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';
+import { Agent, request } from 'undici';
+import { HubError } from '@x12i/api-hub-core';
+import { isPublicAddress } from "./address.js";
+import { ALLOW_ALL, DISALLOW_ALL, parseRobots } from "./robots.js";
+const DEFAULTS = {
+    maxBytes: 5 * 1024 * 1024,
+    timeoutMs: 15_000,
+    maxRedirects: 5,
+    allowedPorts: [80, 443, 8080, 8443],
+    userAgent: 'x12i-hub/0.1 (+https://x12i.com/bot)',
+    robotsToken: 'x12i-hub',
+};
+const defaultResolve = async (hostname) => (await dnsLookup(hostname, { all: true, verbatim: true })).map((a) => ({ address: a.address, family: a.family }));
+const REDIRECTS = new Set([301, 302, 303, 307, 308]);
+function invalid(message, details) {
+    throw new HubError('invalid_input', message, details);
+}
+function parseTarget(raw, allowedPorts) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        return invalid('Not a valid URL');
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        invalid('Only http and https URLs are allowed');
+    if (url.username || url.password)
+        invalid('URLs with credentials are not allowed');
+    const port = url.port ? Number(url.port) : url.protocol === 'https:' ? 443 : 80;
+    if (!allowedPorts.includes(port))
+        invalid(`Port ${port} is not allowed`);
+    url.hash = '';
+    return url;
+}
+const CREDENTIAL_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']);
+/** Credentials are never forwarded to a different origin on redirect. */
+export function withoutCredentials(headers) {
+    if (!headers)
+        return headers;
+    return Object.fromEntries(Object.entries(headers).filter(([k]) => !CREDENTIAL_HEADERS.has(k.toLowerCase())));
+}
+function headerRecord(headers) {
+    const out = {};
+    for (const [k, v] of Object.entries(headers)) {
+        if (v === undefined)
+            continue;
+        const key = k.toLowerCase();
+        // Set-Cookie values contain commas (Expires dates), so repeated cookies are newline-separated.
+        out[key] = Array.isArray(v) ? v.join(key === 'set-cookie' ? '\n' : ', ') : v;
+    }
+    return out;
+}
+function decoder(encoding) {
+    switch ((encoding ?? '').trim().toLowerCase()) {
+        case 'gzip':
+        case 'x-gzip':
+            return createGunzip();
+        case 'deflate':
+            return createInflate();
+        case 'br':
+            return createBrotliDecompress();
+        default:
+            return undefined;
+    }
+}
+async function readCapped(body, maxBytes, signal) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of body) {
+        if (signal.aborted)
+            throw signal.reason ?? new Error('aborted');
+        const buf = chunk;
+        if (size + buf.length > maxBytes) {
+            chunks.push(buf.subarray(0, maxBytes - size));
+            body.destroy();
+            return { bytes: Buffer.concat(chunks), truncated: true };
+        }
+        chunks.push(buf);
+        size += buf.length;
+    }
+    return { bytes: Buffer.concat(chunks), truncated: false };
+}
+export function createSafeFetch(config = {}) {
+    const cfg = { ...DEFAULTS, ...config };
+    const resolve = config.resolve ?? defaultResolve;
+    const allowed = config.isAllowedAddress ?? ((address) => isPublicAddress(address));
+    const robotsCache = new Map();
+    async function pin(url) {
+        const hostname = url.hostname.replace(/^\[|\]$/g, '');
+        const family = isIP(hostname);
+        let addresses;
+        if (family)
+            addresses = [{ address: hostname, family: family }];
+        else {
+            try {
+                addresses = await resolve(hostname);
+            }
+            catch {
+                return invalid(`Could not resolve host ${hostname}`);
+            }
+        }
+        if (addresses.length === 0)
+            invalid(`Could not resolve host ${hostname}`);
+        for (const a of addresses) {
+            if (!allowed(a.address, hostname))
+                invalid('The URL points to a non-public address', { host: hostname });
+        }
+        return addresses[0];
+    }
+    async function once(url, options, signal) {
+        const target = await pin(url);
+        const agent = new Agent({
+            connect: {
+                timeout: Math.min(cfg.timeoutMs, 10_000),
+                lookup: (_hostname, opts, cb) => {
+                    if (opts.all)
+                        cb(null, [{ address: target.address, family: target.family }]);
+                    else
+                        cb(null, target.address, target.family);
+                },
+            },
+        });
+        try {
+            const res = await request(url, {
+                dispatcher: agent,
+                method: options.method ?? 'GET',
+                headers: {
+                    'user-agent': options.userAgent ?? cfg.userAgent,
+                    accept: '*/*',
+                    'accept-encoding': 'gzip, deflate, br',
+                    ...(options.headers ?? {}),
+                },
+                body: options.body,
+                signal,
+            });
+            return { res, target, agent };
+        }
+        catch (err) {
+            await agent.close().catch(() => undefined);
+            throw err;
+        }
+    }
+    async function robotsFor(origin, signal, userAgent) {
+        const key = origin.origin;
+        const cached = robotsCache.get(key);
+        if (cached && cached.expires > Date.now())
+            return cached.policy;
+        let policy = ALLOW_ALL;
+        try {
+            const { res, agent } = await once(new URL('/robots.txt', origin), { userAgent }, signal);
+            try {
+                if (res.statusCode >= 200 && res.statusCode < 300) {
+                    const { bytes } = await readCapped(res.body, 500 * 1024, signal);
+                    policy = parseRobots(Buffer.from(bytes).toString('utf8'), cfg.robotsToken);
+                }
+                else if (res.statusCode >= 500) {
+                    policy = DISALLOW_ALL;
+                    await res.body.dump();
+                }
+                else {
+                    await res.body.dump();
+                }
+            }
+            finally {
+                await agent.close().catch(() => undefined);
+            }
+        }
+        catch (err) {
+            if (err instanceof HubError)
+                throw err;
+            policy = DISALLOW_ALL;
+        }
+        robotsCache.set(key, { policy, expires: Date.now() + 60 * 60 * 1000 });
+        return policy;
+    }
+    return async function safeFetch(rawUrl, options = {}) {
+        const maxBytes = options.maxBytes ?? cfg.maxBytes;
+        const maxRedirects = options.maxRedirects ?? cfg.maxRedirects;
+        const timeout = AbortSignal.timeout(options.timeoutMs ?? cfg.timeoutMs);
+        const signal = options.signal ? AbortSignal.any([options.signal, timeout]) : timeout;
+        const redirects = [];
+        let url = parseTarget(rawUrl, cfg.allowedPorts);
+        let method = options.method ?? 'GET';
+        let body = options.body;
+        let requestHeaders = options.headers;
+        try {
+            for (;;) {
+                if (options.respectRobots) {
+                    const policy = await robotsFor(url, signal, options.userAgent ?? cfg.userAgent);
+                    if (!policy.isAllowed(`${url.pathname}${url.search}`)) {
+                        throw new HubError('not_permitted', 'robots.txt does not allow fetching this URL');
+                    }
+                }
+                const { res, target, agent } = await once(url, { ...options, method, body, headers: requestHeaders }, signal);
+                try {
+                    const headers = headerRecord(res.headers);
+                    if (REDIRECTS.has(res.statusCode) && headers.location && maxRedirects > 0) {
+                        await res.body.dump();
+                        if (redirects.length >= maxRedirects) {
+                            throw new HubError('limit_exceeded', `More than ${maxRedirects} redirects`);
+                        }
+                        redirects.push(url.toString());
+                        const previousOrigin = url.origin;
+                        url = parseTarget(new URL(headers.location, url).toString(), cfg.allowedPorts);
+                        if (url.origin !== previousOrigin)
+                            requestHeaders = withoutCredentials(requestHeaders);
+                        if (res.statusCode === 303 || ((res.statusCode === 301 || res.statusCode === 302) && method === 'POST')) {
+                            method = 'GET';
+                            body = undefined;
+                        }
+                        continue;
+                    }
+                    const raw = res.body;
+                    const decode = method === 'HEAD' ? undefined : decoder(headers['content-encoding']);
+                    const stream = decode ? raw.pipe(decode) : raw;
+                    const { bytes, truncated } = await readCapped(stream, maxBytes, signal);
+                    if (truncated)
+                        raw.destroy();
+                    if (truncated && !options.truncate) {
+                        throw new HubError('limit_exceeded', `Response is larger than ${maxBytes} bytes`);
+                    }
+                    const finalUrl = url.toString();
+                    return {
+                        requestedUrl: rawUrl,
+                        url: finalUrl,
+                        status: res.statusCode,
+                        headers,
+                        redirects,
+                        remoteAddress: target.address,
+                        body: bytes,
+                        truncated,
+                        text: () => Buffer.from(bytes).toString('utf8'),
+                        json: () => JSON.parse(Buffer.from(bytes).toString('utf8')),
+                    };
+                }
+                finally {
+                    // Each request has its own pinned agent; destroying it drops any unread body instead of downloading it.
+                    await agent.destroy().catch(() => undefined);
+                }
+            }
+        }
+        catch (err) {
+            if (err instanceof HubError)
+                throw err;
+            if (timeout.aborted)
+                throw new HubError('timeout', 'The remote server took too long');
+            if (options.signal?.aborted)
+                throw new HubError('timeout', 'The request was cancelled');
+            const code = err.code;
+            throw new HubError('upstream_unavailable', 'Could not fetch the URL', code ? { reason: code } : undefined);
+        }
+    };
+}
+//# sourceMappingURL=fetch.js.map

package/dist/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../../../packages/safe-fetch/src/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAChF,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AACxC,OAAO,EAAE,QAAQ,EAA4D,MAAM,WAAW,CAAC;AAC/F,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAqB,MAAM,aAAa,CAAC;AAuBtF,MAAM,QAAQ,GAAG;IACf,QAAQ,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;IACzB,SAAS,EAAE,MAAM;IACjB,YAAY,EAAE,CAAC;IACf,YAAY,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC;IACnC,SAAS,EAAE,sCAAsC;IACjD,WAAW,EAAE,UAAU;CACxB,CAAC;AAEF,MAAM,cAAc,GAAa,KAAK,EAAE,QAAQ,EAAE,EAAE,CAClD,CAAC,MAAM,SAAS,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAe,EAAE,CAAC,CAAC,CAAC;AAE7H,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAErD,SAAS,OAAO,CAAC,OAAe,EAAE,OAAiB;IACjD,MAAM,IAAI,QAAQ,CAAC,eAAe,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,YAAsB;IACtD,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,CAAC,sCAAsC,CAAC,CAAC;IAC3G,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ;QAAE,OAAO,CAAC,uCAAuC,CAAC,CAAC;IACnF,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC,CAAC;IACzE,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;IACd,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,eAAe,EAAE,QAAQ,EAAE,qBAAqB,CAAC,CAAC,CAAC;AAEvF,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAC,OAA2C;IAC5E,IAAI,CAAC,OAAO;QAAE,OAAO,OAAO,CAAC;IAC7B,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/G,CAAC;AAED,SAAS,YAAY,CAAC,OAAsD;IAC1E,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QAC9B,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5B,+FAA+F;QAC/F,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,OAAO,CAAC,QAA4B;IAC3C,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QAC9C,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,YAAY,EAAE,CAAC;QACxB,KAAK,SAAS;YACZ,OAAO,aAAa,EAAE,CAAC;QACzB,KAAK,IAAI;YACP,OAAO,sBAAsB,EAAE,CAAC;QAClC;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAc,EAAE,QAAgB,EAAE,MAAmB;IAC7E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,MAAM,CAAC,OAAO;YAAE,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;QAChE,MAAM,GAAG,GAAG,KAAe,CAAC;QAC5B,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC;YAC9C,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC3D,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAA0B,EAAE;IAC1D,MAAM,GAAG,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,cAAc,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC,OAAe,EAAE,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3F,MAAM,WAAW,GAAG,IAAI,GAAG,EAAqD,CAAC;IAEjF,KAAK,UAAU,GAAG,CAAC,GAAQ;QACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,IAAI,SAA4B,CAAC;QACjC,IAAI,MAAM;YAAE,SAAS,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAe,EAAE,CAAC,CAAC;aACpE,CAAC;YACJ,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,OAAO,CAAC,0BAA0B,QAAQ,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,0BAA0B,QAAQ,EAAE,CAAC,CAAC;QAC1E,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC;gBAAE,OAAO,CAAC,wCAAwC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3G,CAAC;QACD,OAAO,SAAS,CAAC,CAAC,CAAE,CAAC;IACvB,CAAC;IAED,KAAK,UAAU,IAAI,CAAC,GAAQ,EAAE,OAAyB,EAAE,MAAmB;QAC1E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;YACtB,OAAO,EAAE;gBACP,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC;gBACxC,MAAM,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;oBAC9B,IAAK,IAA0B,CAAC,GAAG;wBAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;;wBAC9F,EAAsD,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBACpG,CAAC;aACF;SACF,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;gBAC7B,UAAU,EAAE,KAAK;gBACjB,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;gBAC/B,OAAO,EAAE;oBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS;oBAChD,MAAM,EAAE,KAAK;oBACb,iBAAiB,EAAE,mBAAmB;oBACtC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;iBAC3B;gBACD,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,MAAM;aACP,CAAC,CAAC;YACH,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC3C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,UAAU,SAAS,CAAC,MAAW,EAAE,MAAmB,EAAE,SAAiB;QAC1E,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE;YAAE,OAAO,MAAM,CAAC,MAAM,CAAC;QAChE,IAAI,MAAM,GAAiB,SAAS,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,CAAC;YACzF,IAAI,CAAC;gBACH,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;oBAClD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,IAA2B,EAAE,GAAG,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;oBACxF,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;gBAC7E,CAAC;qBAAM,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;oBACjC,MAAM,GAAG,YAAY,CAAC;oBACtB,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,QAAQ;gBAAE,MAAM,GAAG,CAAC;YACvC,MAAM,GAAG,YAAY,CAAC;QACxB,CAAC;QACD,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QACvE,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,UAA4B,EAAE;QAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC,YAAY,CAAC;QAC9D,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACrF,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QAChD,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;QACrC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACxB,IAAI,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;QAErC,IAAI,CAAC;YACH,SAAS,CAAC;gBACR,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;oBAC1B,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;oBAChF,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;wBACtD,MAAM,IAAI,QAAQ,CAAC,eAAe,EAAE,6CAA6C,CAAC,CAAC;oBACrF,CAAC;gBACH,CAAC;gBACD,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,MAAM,CAAC,CAAC;gBAC9G,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC1C,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,QAAQ,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;wBAC1E,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;wBACtB,IAAI,SAAS,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;4BACrC,MAAM,IAAI,QAAQ,CAAC,gBAAgB,EAAE,aAAa,YAAY,YAAY,CAAC,CAAC;wBAC9E,CAAC;wBACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAC/B,MAAM,cAAc,GAAG,GAAG,CAAC,MAAM,CAAC;wBAClC,GAAG,GAAG,WAAW,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;wBAC/E,IAAI,GAAG,CAAC,MAAM,KAAK,cAAc;4BAAE,cAAc,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;wBACvF,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,IAAI,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;4BACxG,MAAM,GAAG,KAAK,CAAC;4BACf,IAAI,GAAG,SAAS,CAAC;wBACnB,CAAC;wBACD,SAAS;oBACX,CAAC;oBAED,MAAM,GAAG,GAAG,GAAG,CAAC,IAA2B,CAAC;oBAC5C,MAAM,MAAM,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;oBACpF,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;oBAC/C,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;oBACxE,IAAI,SAAS;wBAAE,GAAG,CAAC,OAAO,EAAE,CAAC;oBAC7B,IAAI,SAAS,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;wBACnC,MAAM,IAAI,QAAQ,CAAC,gBAAgB,EAAE,2BAA2B,QAAQ,QAAQ,CAAC,CAAC;oBACpF,CAAC;oBACD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;oBAChC,OAAO;wBACL,YAAY,EAAE,MAAM;wBACpB,GAAG,EAAE,QAAQ;wBACb,MAAM,EAAE,GAAG,CAAC,UAAU;wBACtB,OAAO;wBACP,SAAS;wBACT,aAAa,EAAE,MAAM,CAAC,OAAO;wBAC7B,IAAI,EAAE,KAAK;wBACX,SAAS;wBACT,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC/C,IAAI,EAAE,GAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAM;qBACpE,CAAC;gBACJ,CAAC;wBAAS,CAAC;oBACT,wGAAwG;oBACxG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,QAAQ;gBAAE,MAAM,GAAG,CAAC;YACvC,IAAI,OAAO,CAAC,OAAO;gBAAE,MAAM,IAAI,QAAQ,CAAC,SAAS,EAAE,iCAAiC,CAAC,CAAC;YACtF,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,QAAQ,CAAC,SAAS,EAAE,2BAA2B,CAAC,CAAC;YACxF,MAAM,IAAI,GAAI,GAAyB,CAAC,IAAI,CAAC;YAC7C,MAAM,IAAI,QAAQ,CAAC,sBAAsB,EAAE,yBAAyB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7G,CAAC;IACH,CAAC,CAAC;AACJ,CAAC","sourcesContent":["import { lookup as dnsLookup } from 'node:dns/promises';\nimport { isIP } from 'node:net';\nimport { Readable } from 'node:stream';\nimport { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';\nimport { Agent, request } from 'undici';\nimport { HubError, type SafeFetch, type SafeFetchOptions, type SafeResponse } from '@x12i/api-hub-core';\nimport { isPublicAddress } from './address.ts';\nimport { ALLOW_ALL, DISALLOW_ALL, parseRobots, type RobotsPolicy } from './robots.ts';\n\nexport interface ResolvedAddress {\n  address: string;\n  family: 4 | 6;\n}\n\nexport type Resolver = (hostname: string) => Promise<ResolvedAddress[]>;\n\nexport interface SafeFetchConfig {\n  /** Default byte cap on the decoded body. */\n  maxBytes?: number;\n  timeoutMs?: number;\n  maxRedirects?: number;\n  allowedPorts?: number[];\n  userAgent?: string;\n  /** Product token matched against robots.txt groups. */\n  robotsToken?: string;\n  resolve?: Resolver;\n  /** Decides whether an address may be contacted. Defaults to public unicast only. */\n  isAllowedAddress?: (address: string, hostname: string) => boolean;\n}\n\nconst DEFAULTS = {\n  maxBytes: 5 * 1024 * 1024,\n  timeoutMs: 15_000,\n  maxRedirects: 5,\n  allowedPorts: [80, 443, 8080, 8443],\n  userAgent: 'x12i-hub/0.1 (+https://x12i.com/bot)',\n  robotsToken: 'x12i-hub',\n};\n\nconst defaultResolve: Resolver = async (hostname) =>\n  (await dnsLookup(hostname, { all: true, verbatim: true })).map((a) => ({ address: a.address, family: a.family as 4 | 6 }));\n\nconst REDIRECTS = new Set([301, 302, 303, 307, 308]);\n\nfunction invalid(message: string, details?: unknown): never {\n  throw new HubError('invalid_input', message, details);\n}\n\nfunction parseTarget(raw: string, allowedPorts: number[]): URL {\n  let url: URL;\n  try {\n    url = new URL(raw);\n  } catch {\n    return invalid('Not a valid URL');\n  }\n  if (url.protocol !== 'http:' && url.protocol !== 'https:') invalid('Only http and https URLs are allowed');\n  if (url.username || url.password) invalid('URLs with credentials are not allowed');\n  const port = url.port ? Number(url.port) : url.protocol === 'https:' ? 443 : 80;\n  if (!allowedPorts.includes(port)) invalid(`Port ${port} is not allowed`);\n  url.hash = '';\n  return url;\n}\n\nconst CREDENTIAL_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']);\n\n/** Credentials are never forwarded to a different origin on redirect. */\nexport function withoutCredentials(headers: Record<string, string> | undefined): Record<string, string> | undefined {\n  if (!headers) return headers;\n  return Object.fromEntries(Object.entries(headers).filter(([k]) => !CREDENTIAL_HEADERS.has(k.toLowerCase())));\n}\n\nfunction headerRecord(headers: Record<string, string | string[] | undefined>): Record<string, string> {\n  const out: Record<string, string> = {};\n  for (const [k, v] of Object.entries(headers)) {\n    if (v === undefined) continue;\n    const key = k.toLowerCase();\n    // Set-Cookie values contain commas (Expires dates), so repeated cookies are newline-separated.\n    out[key] = Array.isArray(v) ? v.join(key === 'set-cookie' ? '\\n' : ', ') : v;\n  }\n  return out;\n}\n\nfunction decoder(encoding: string | undefined) {\n  switch ((encoding ?? '').trim().toLowerCase()) {\n    case 'gzip':\n    case 'x-gzip':\n      return createGunzip();\n    case 'deflate':\n      return createInflate();\n    case 'br':\n      return createBrotliDecompress();\n    default:\n      return undefined;\n  }\n}\n\nasync function readCapped(body: Readable, maxBytes: number, signal: AbortSignal): Promise<{ bytes: Uint8Array; truncated: boolean }> {\n  const chunks: Buffer[] = [];\n  let size = 0;\n  for await (const chunk of body) {\n    if (signal.aborted) throw signal.reason ?? new Error('aborted');\n    const buf = chunk as Buffer;\n    if (size + buf.length > maxBytes) {\n      chunks.push(buf.subarray(0, maxBytes - size));\n      body.destroy();\n      return { bytes: Buffer.concat(chunks), truncated: true };\n    }\n    chunks.push(buf);\n    size += buf.length;\n  }\n  return { bytes: Buffer.concat(chunks), truncated: false };\n}\n\nexport function createSafeFetch(config: SafeFetchConfig = {}): SafeFetch {\n  const cfg = { ...DEFAULTS, ...config };\n  const resolve = config.resolve ?? defaultResolve;\n  const allowed = config.isAllowedAddress ?? ((address: string) => isPublicAddress(address));\n  const robotsCache = new Map<string, { policy: RobotsPolicy; expires: number }>();\n\n  async function pin(url: URL): Promise<ResolvedAddress> {\n    const hostname = url.hostname.replace(/^\\[|\\]$/g, '');\n    const family = isIP(hostname);\n    let addresses: ResolvedAddress[];\n    if (family) addresses = [{ address: hostname, family: family as 4 | 6 }];\n    else {\n      try {\n        addresses = await resolve(hostname);\n      } catch {\n        return invalid(`Could not resolve host ${hostname}`);\n      }\n    }\n    if (addresses.length === 0) invalid(`Could not resolve host ${hostname}`);\n    for (const a of addresses) {\n      if (!allowed(a.address, hostname)) invalid('The URL points to a non-public address', { host: hostname });\n    }\n    return addresses[0]!;\n  }\n\n  async function once(url: URL, options: SafeFetchOptions, signal: AbortSignal) {\n    const target = await pin(url);\n    const agent = new Agent({\n      connect: {\n        timeout: Math.min(cfg.timeoutMs, 10_000),\n        lookup: (_hostname, opts, cb) => {\n          if ((opts as { all?: boolean }).all) cb(null, [{ address: target.address, family: target.family }]);\n          else (cb as (e: Error | null, a: string, f: number) => void)(null, target.address, target.family);\n        },\n      },\n    });\n    try {\n      const res = await request(url, {\n        dispatcher: agent,\n        method: options.method ?? 'GET',\n        headers: {\n          'user-agent': options.userAgent ?? cfg.userAgent,\n          accept: '*/*',\n          'accept-encoding': 'gzip, deflate, br',\n          ...(options.headers ?? {}),\n        },\n        body: options.body,\n        signal,\n      });\n      return { res, target, agent };\n    } catch (err) {\n      await agent.close().catch(() => undefined);\n      throw err;\n    }\n  }\n\n  async function robotsFor(origin: URL, signal: AbortSignal, userAgent: string): Promise<RobotsPolicy> {\n    const key = origin.origin;\n    const cached = robotsCache.get(key);\n    if (cached && cached.expires > Date.now()) return cached.policy;\n    let policy: RobotsPolicy = ALLOW_ALL;\n    try {\n      const { res, agent } = await once(new URL('/robots.txt', origin), { userAgent }, signal);\n      try {\n        if (res.statusCode >= 200 && res.statusCode < 300) {\n          const { bytes } = await readCapped(res.body as unknown as Readable, 500 * 1024, signal);\n          policy = parseRobots(Buffer.from(bytes).toString('utf8'), cfg.robotsToken);\n        } else if (res.statusCode >= 500) {\n          policy = DISALLOW_ALL;\n          await res.body.dump();\n        } else {\n          await res.body.dump();\n        }\n      } finally {\n        await agent.close().catch(() => undefined);\n      }\n    } catch (err) {\n      if (err instanceof HubError) throw err;\n      policy = DISALLOW_ALL;\n    }\n    robotsCache.set(key, { policy, expires: Date.now() + 60 * 60 * 1000 });\n    return policy;\n  }\n\n  return async function safeFetch(rawUrl: string, options: SafeFetchOptions = {}): Promise<SafeResponse> {\n    const maxBytes = options.maxBytes ?? cfg.maxBytes;\n    const maxRedirects = options.maxRedirects ?? cfg.maxRedirects;\n    const timeout = AbortSignal.timeout(options.timeoutMs ?? cfg.timeoutMs);\n    const signal = options.signal ? AbortSignal.any([options.signal, timeout]) : timeout;\n    const redirects: string[] = [];\n    let url = parseTarget(rawUrl, cfg.allowedPorts);\n    let method = options.method ?? 'GET';\n    let body = options.body;\n    let requestHeaders = options.headers;\n\n    try {\n      for (;;) {\n        if (options.respectRobots) {\n          const policy = await robotsFor(url, signal, options.userAgent ?? cfg.userAgent);\n          if (!policy.isAllowed(`${url.pathname}${url.search}`)) {\n            throw new HubError('not_permitted', 'robots.txt does not allow fetching this URL');\n          }\n        }\n        const { res, target, agent } = await once(url, { ...options, method, body, headers: requestHeaders }, signal);\n        try {\n          const headers = headerRecord(res.headers);\n          if (REDIRECTS.has(res.statusCode) && headers.location && maxRedirects > 0) {\n            await res.body.dump();\n            if (redirects.length >= maxRedirects) {\n              throw new HubError('limit_exceeded', `More than ${maxRedirects} redirects`);\n            }\n            redirects.push(url.toString());\n            const previousOrigin = url.origin;\n            url = parseTarget(new URL(headers.location, url).toString(), cfg.allowedPorts);\n            if (url.origin !== previousOrigin) requestHeaders = withoutCredentials(requestHeaders);\n            if (res.statusCode === 303 || ((res.statusCode === 301 || res.statusCode === 302) && method === 'POST')) {\n              method = 'GET';\n              body = undefined;\n            }\n            continue;\n          }\n\n          const raw = res.body as unknown as Readable;\n          const decode = method === 'HEAD' ? undefined : decoder(headers['content-encoding']);\n          const stream = decode ? raw.pipe(decode) : raw;\n          const { bytes, truncated } = await readCapped(stream, maxBytes, signal);\n          if (truncated) raw.destroy();\n          if (truncated && !options.truncate) {\n            throw new HubError('limit_exceeded', `Response is larger than ${maxBytes} bytes`);\n          }\n          const finalUrl = url.toString();\n          return {\n            requestedUrl: rawUrl,\n            url: finalUrl,\n            status: res.statusCode,\n            headers,\n            redirects,\n            remoteAddress: target.address,\n            body: bytes,\n            truncated,\n            text: () => Buffer.from(bytes).toString('utf8'),\n            json: <T>() => JSON.parse(Buffer.from(bytes).toString('utf8')) as T,\n          };\n        } finally {\n          // Each request has its own pinned agent; destroying it drops any unread body instead of downloading it.\n          await agent.destroy().catch(() => undefined);\n        }\n      }\n    } catch (err) {\n      if (err instanceof HubError) throw err;\n      if (timeout.aborted) throw new HubError('timeout', 'The remote server took too long');\n      if (options.signal?.aborted) throw new HubError('timeout', 'The request was cancelled');\n      const code = (err as { code?: string }).code;\n      throw new HubError('upstream_unavailable', 'Could not fetch the URL', code ? { reason: code } : undefined);\n    }\n  };\n}\n"]}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { isPublicAddress } from './address.js';
+export { matchesPattern, parseRobots, parseRobotsFile, policyFor, type RobotsFile, type RobotsGroup, type RobotsMatch, type RobotsPolicy, type RobotsRule, } from './robots.js';
+export { fakeFetch, type FakeRoute, type FakeRoutes } from './fake.js';
+export { createSafeFetch, type Resolver, type ResolvedAddress, type SafeFetchConfig } from './fetch.js';

package/dist/index.js

@@ -0,0 +1,5 @@
+export { isPublicAddress } from "./address.js";
+export { matchesPattern, parseRobots, parseRobotsFile, policyFor, } from "./robots.js";
+export { fakeFetch } from "./fake.js";
+export { createSafeFetch } from "./fetch.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../packages/safe-fetch/src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EACL,cAAc,EACd,WAAW,EACX,eAAe,EACf,SAAS,GAMV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,SAAS,EAAmC,MAAM,WAAW,CAAC;AACvE,OAAO,EAAE,eAAe,EAA6D,MAAM,YAAY,CAAC","sourcesContent":["export { isPublicAddress } from './address.ts';\nexport {\n  matchesPattern,\n  parseRobots,\n  parseRobotsFile,\n  policyFor,\n  type RobotsFile,\n  type RobotsGroup,\n  type RobotsMatch,\n  type RobotsPolicy,\n  type RobotsRule,\n} from './robots.ts';\nexport { fakeFetch, type FakeRoute, type FakeRoutes } from './fake.ts';\nexport { createSafeFetch, type Resolver, type ResolvedAddress, type SafeFetchConfig } from './fetch.ts';\n"]}

package/dist/robots.d.ts

@@ -0,0 +1,35 @@
+export interface RobotsRule {
+    allow: boolean;
+    pattern: string;
+}
+export interface RobotsGroup {
+    agents: string[];
+    rules: RobotsRule[];
+    crawlDelay: number | null;
+}
+export interface RobotsFile {
+    groups: RobotsGroup[];
+    sitemaps: string[];
+}
+export interface RobotsMatch {
+    allowed: boolean;
+    /** The rule that decided, or null when no rule matched (allowed by default). */
+    rule: RobotsRule | null;
+}
+export interface RobotsPolicy {
+    isAllowed(pathWithQuery: string): boolean;
+    match(pathWithQuery: string): RobotsMatch;
+    /** User-agent lines of the group(s) that apply; empty when none does. */
+    agents: string[];
+    crawlDelay: number | null;
+}
+export declare const ALLOW_ALL: RobotsPolicy;
+export declare const DISALLOW_ALL: RobotsPolicy;
+export declare function parseRobotsFile(text: string): RobotsFile;
+/**
+ * Wildcard match without regular expressions, so hostile patterns cannot cause backtracking.
+ * `*` matches any run of characters; a trailing `$` anchors the end.
+ */
+export declare function matchesPattern(pattern: string, path: string): boolean;
+export declare function policyFor(file: RobotsFile, userAgentToken: string): RobotsPolicy;
+export declare function parseRobots(text: string, userAgentToken: string): RobotsPolicy;

package/dist/robots.js

@@ -0,0 +1,104 @@
+export const ALLOW_ALL = {
+    isAllowed: () => true,
+    match: () => ({ allowed: true, rule: null }),
+    agents: [],
+    crawlDelay: null,
+};
+export const DISALLOW_ALL = {
+    isAllowed: () => false,
+    match: () => ({ allowed: false, rule: { allow: false, pattern: '/' } }),
+    agents: [],
+    crawlDelay: null,
+};
+const MAX_LINES = 20_000;
+const MAX_PATTERN = 512;
+export function parseRobotsFile(text) {
+    const groups = [];
+    const sitemaps = [];
+    let current;
+    let lastWasAgent = false;
+    for (const rawLine of text.split(/\r?\n/, MAX_LINES)) {
+        const line = rawLine.replace(/#.*$/, '').trim();
+        const sep = line.indexOf(':');
+        if (sep <= 0)
+            continue;
+        const key = line.slice(0, sep).trim().toLowerCase();
+        const value = line.slice(sep + 1).trim();
+        if (key === 'user-agent') {
+            if (!current || !lastWasAgent) {
+                current = { agents: [], rules: [], crawlDelay: null };
+                groups.push(current);
+            }
+            current.agents.push(value.toLowerCase());
+            lastWasAgent = true;
+            continue;
+        }
+        lastWasAgent = false;
+        if (key === 'sitemap') {
+            if (value)
+                sitemaps.push(value);
+        }
+        else if ((key === 'allow' || key === 'disallow') && current) {
+            if (key === 'disallow' && value === '')
+                continue;
+            current.rules.push({ allow: key === 'allow', pattern: value.slice(0, MAX_PATTERN) });
+        }
+        else if (key === 'crawl-delay' && current) {
+            const delay = Number(value);
+            if (Number.isFinite(delay) && delay >= 0)
+                current.crawlDelay = delay;
+        }
+    }
+    return { groups, sitemaps };
+}
+/**
+ * Wildcard match without regular expressions, so hostile patterns cannot cause backtracking.
+ * `*` matches any run of characters; a trailing `$` anchors the end.
+ */
+export function matchesPattern(pattern, path) {
+    const anchored = pattern.endsWith('$');
+    const parts = (anchored ? pattern.slice(0, -1) : pattern).split('*');
+    if (!path.startsWith(parts[0]))
+        return false;
+    let pos = parts[0].length;
+    for (let i = 1; i < parts.length; i++) {
+        const part = parts[i];
+        if (i === parts.length - 1 && anchored)
+            return path.length - part.length >= pos && path.endsWith(part);
+        const found = path.indexOf(part, pos);
+        if (found === -1)
+            return false;
+        pos = found + part.length;
+    }
+    return anchored ? pos === path.length : true;
+}
+export function policyFor(file, userAgentToken) {
+    // RFC 9309 matches the product token (the name before any "/version"), not a substring of it.
+    const token = (/^[a-z0-9_-]+/i.exec(userAgentToken.trim())?.[0] ?? '').toLowerCase();
+    const specific = file.groups.filter((g) => g.agents.some((a) => a !== '*' && a === token));
+    const chosen = specific.length > 0 ? specific : file.groups.filter((g) => g.agents.includes('*'));
+    const rules = chosen.flatMap((g) => g.rules);
+    const delays = chosen.map((g) => g.crawlDelay).filter((d) => d !== null);
+    function match(pathWithQuery) {
+        let best = null;
+        for (const rule of rules) {
+            if (!matchesPattern(rule.pattern, pathWithQuery))
+                continue;
+            // RFC 9309: the most specific (longest) rule wins; ties go to allow.
+            if (!best || rule.pattern.length > best.pattern.length || (rule.pattern.length === best.pattern.length && rule.allow)) {
+                best = rule;
+            }
+        }
+        return { allowed: best?.allow ?? true, rule: best };
+    }
+    return {
+        match,
+        isAllowed: (path) => match(path).allowed,
+        agents: [...new Set(chosen.flatMap((g) => g.agents))],
+        crawlDelay: delays.length ? Math.max(...delays) : null,
+    };
+}
+export function parseRobots(text, userAgentToken) {
+    return policyFor(parseRobotsFile(text), userAgentToken);
+}
+//# sourceMappingURL=robots.js.map

package/dist/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../../../../packages/safe-fetch/src/robots.ts"],"names":[],"mappings":"AA8BA,MAAM,CAAC,MAAM,SAAS,GAAiB;IACrC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI;IACrB,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5C,MAAM,EAAE,EAAE;IACV,UAAU,EAAE,IAAI;CACjB,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAiB;IACxC,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK;IACtB,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;IACvE,MAAM,EAAE,EAAE;IACV,UAAU,EAAE,IAAI;CACjB,CAAC;AAEF,MAAM,SAAS,GAAG,MAAM,CAAC;AACzB,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,OAAgC,CAAC;IACrC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,GAAG,IAAI,CAAC;YAAE,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzC,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;gBAC9B,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YACzC,YAAY,GAAG,IAAI,CAAC;YACpB,SAAS;QACX,CAAC;QACD,YAAY,GAAG,KAAK,CAAC;QACrB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,KAAK;gBAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,CAAC,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC;YAC9D,IAAI,GAAG,KAAK,UAAU,IAAI,KAAK,KAAK,EAAE;gBAAE,SAAS;YACjD,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,KAAK,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;aAAM,IAAI,GAAG,KAAK,aAAa,IAAI,OAAO,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC;gBAAE,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC;QACvE,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe,EAAE,IAAY;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ;YAAE,OAAO,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvG,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/B,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,OAAO,QAAQ,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAgB,EAAE,cAAsB;IAChE,8FAA8F;IAC9F,MAAM,KAAK,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAClG,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAEtF,SAAS,KAAK,CAAC,aAAqB;QAClC,IAAI,IAAI,GAAsB,IAAI,CAAC;QACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC;gBAAE,SAAS;YAC3D,qEAAqE;YACrE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtH,IAAI,GAAG,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACtD,CAAC;IAED,OAAO;QACL,KAAK;QACL,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO;QACxC,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;KACvD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,cAAsB;IAC9D,OAAO,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC;AAC1D,CAAC","sourcesContent":["export interface RobotsRule {\n  allow: boolean;\n  pattern: string;\n}\n\nexport interface RobotsGroup {\n  agents: string[];\n  rules: RobotsRule[];\n  crawlDelay: number | null;\n}\n\nexport interface RobotsFile {\n  groups: RobotsGroup[];\n  sitemaps: string[];\n}\n\nexport interface RobotsMatch {\n  allowed: boolean;\n  /** The rule that decided, or null when no rule matched (allowed by default). */\n  rule: RobotsRule | null;\n}\n\nexport interface RobotsPolicy {\n  isAllowed(pathWithQuery: string): boolean;\n  match(pathWithQuery: string): RobotsMatch;\n  /** User-agent lines of the group(s) that apply; empty when none does. */\n  agents: string[];\n  crawlDelay: number | null;\n}\n\nexport const ALLOW_ALL: RobotsPolicy = {\n  isAllowed: () => true,\n  match: () => ({ allowed: true, rule: null }),\n  agents: [],\n  crawlDelay: null,\n};\nexport const DISALLOW_ALL: RobotsPolicy = {\n  isAllowed: () => false,\n  match: () => ({ allowed: false, rule: { allow: false, pattern: '/' } }),\n  agents: [],\n  crawlDelay: null,\n};\n\nconst MAX_LINES = 20_000;\nconst MAX_PATTERN = 512;\n\nexport function parseRobotsFile(text: string): RobotsFile {\n  const groups: RobotsGroup[] = [];\n  const sitemaps: string[] = [];\n  let current: RobotsGroup | undefined;\n  let lastWasAgent = false;\n  for (const rawLine of text.split(/\\r?\\n/, MAX_LINES)) {\n    const line = rawLine.replace(/#.*$/, '').trim();\n    const sep = line.indexOf(':');\n    if (sep <= 0) continue;\n    const key = line.slice(0, sep).trim().toLowerCase();\n    const value = line.slice(sep + 1).trim();\n    if (key === 'user-agent') {\n      if (!current || !lastWasAgent) {\n        current = { agents: [], rules: [], crawlDelay: null };\n        groups.push(current);\n      }\n      current.agents.push(value.toLowerCase());\n      lastWasAgent = true;\n      continue;\n    }\n    lastWasAgent = false;\n    if (key === 'sitemap') {\n      if (value) sitemaps.push(value);\n    } else if ((key === 'allow' || key === 'disallow') && current) {\n      if (key === 'disallow' && value === '') continue;\n      current.rules.push({ allow: key === 'allow', pattern: value.slice(0, MAX_PATTERN) });\n    } else if (key === 'crawl-delay' && current) {\n      const delay = Number(value);\n      if (Number.isFinite(delay) && delay >= 0) current.crawlDelay = delay;\n    }\n  }\n  return { groups, sitemaps };\n}\n\n/**\n * Wildcard match without regular expressions, so hostile patterns cannot cause backtracking.\n * `*` matches any run of characters; a trailing `$` anchors the end.\n */\nexport function matchesPattern(pattern: string, path: string): boolean {\n  const anchored = pattern.endsWith('$');\n  const parts = (anchored ? pattern.slice(0, -1) : pattern).split('*');\n  if (!path.startsWith(parts[0]!)) return false;\n  let pos = parts[0]!.length;\n  for (let i = 1; i < parts.length; i++) {\n    const part = parts[i]!;\n    if (i === parts.length - 1 && anchored) return path.length - part.length >= pos && path.endsWith(part);\n    const found = path.indexOf(part, pos);\n    if (found === -1) return false;\n    pos = found + part.length;\n  }\n  return anchored ? pos === path.length : true;\n}\n\nexport function policyFor(file: RobotsFile, userAgentToken: string): RobotsPolicy {\n  // RFC 9309 matches the product token (the name before any \"/version\"), not a substring of it.\n  const token = (/^[a-z0-9_-]+/i.exec(userAgentToken.trim())?.[0] ?? '').toLowerCase();\n  const specific = file.groups.filter((g) => g.agents.some((a) => a !== '*' && a === token));\n  const chosen = specific.length > 0 ? specific : file.groups.filter((g) => g.agents.includes('*'));\n  const rules = chosen.flatMap((g) => g.rules);\n  const delays = chosen.map((g) => g.crawlDelay).filter((d): d is number => d !== null);\n\n  function match(pathWithQuery: string): RobotsMatch {\n    let best: RobotsRule | null = null;\n    for (const rule of rules) {\n      if (!matchesPattern(rule.pattern, pathWithQuery)) continue;\n      // RFC 9309: the most specific (longest) rule wins; ties go to allow.\n      if (!best || rule.pattern.length > best.pattern.length || (rule.pattern.length === best.pattern.length && rule.allow)) {\n        best = rule;\n      }\n    }\n    return { allowed: best?.allow ?? true, rule: best };\n  }\n\n  return {\n    match,\n    isAllowed: (path) => match(path).allowed,\n    agents: [...new Set(chosen.flatMap((g) => g.agents))],\n    crawlDelay: delays.length ? Math.max(...delays) : null,\n  };\n}\n\nexport function parseRobots(text: string, userAgentToken: string): RobotsPolicy {\n  return policyFor(parseRobotsFile(text), userAgentToken);\n}\n"]}

package/package.json

@@ -1,6 +1,37 @@
 {
   "name": "@x12i/api-hub-safe-fetch",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "SSRF-safe HTTP fetch for the x12i API hub",
+  "type": "module",
+  "engines": {
+    "node": ">=24"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/x12i/apis-hub.git",
+    "directory": "packages/safe-fetch"
+  },
+  "bugs": {
+    "url": "https://github.com/x12i/apis-hub/issues"
+  },
+  "homepage": "https://github.com/x12i/apis-hub",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@x12i/api-hub-core": "0.1.1",
+    "undici": "8.12.3"
+  }
+}