@hraness/sponge 0.6.7 → 0.6.9

package/dist/src/browser/setup.js

@@ -0,0 +1,103 @@
+import { existsSync } from 'node:fs';
+import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { askBridge, bridgeSocket, extensionId, hostName } from './bridge.js';
+/** Where each Chromium browser on macOS looks for user-level native messaging hosts. */
+const browsers = {
+    Chrome: 'Google/Chrome',
+    'Chrome Beta': 'Google/Chrome Beta',
+    'Chrome Canary': 'Google/Chrome Canary',
+    Chromium: 'Chromium',
+    Brave: 'BraveSoftware/Brave-Browser',
+    Edge: 'Microsoft Edge',
+    Arc: 'Arc/User Data',
+    Vivaldi: 'Vivaldi',
+};
+const manifestName = `${hostName}.json`;
+/** The installed browsers' host directories, plus any the reader named. */
+function hostDirectories(options) {
+    const support = join(options.home ?? homedir(), 'Library', 'Application Support');
+    return [
+        ...Object.entries(browsers)
+            .filter(([, data]) => existsSync(join(support, data)))
+            .map(([browser, data]) => ({ browser, dir: join(support, data, 'NativeMessagingHosts') })),
+        ...(options.hostsDirs ?? []).map((dir) => ({ browser: 'custom', dir })),
+    ];
+}
+const wrapperPath = (home) => join(home, '.local', 'state', 'sponge', 'browser-host');
+/** The unpacked extension that ships with this Sponge. */
+function extensionDirectory() {
+    return fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? '../../extension/' : '../../../extension/', import.meta.url));
+}
+const quote = (value) => `'${value.replace(/'/g, `'\\''`)}'`;
+/**
+ * Registers the native host with each installed browser for one library. The host is a wrapper script, since a
+ * manifest names one executable and takes no arguments; it runs this Node and this Sponge's host.
+ */
+export async function setupBrowser(libraryRoot, options = {}) {
+    const home = options.home ?? homedir(), wrapper = wrapperPath(home), host = fileURLToPath(new URL('./host.js', import.meta.url));
+    await mkdir(dirname(wrapper), { recursive: true, mode: 0o700 });
+    await writeFile(wrapper, `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(host)} --library ${quote(libraryRoot)} --socket ${quote(bridgeSocket(libraryRoot, home))} "$@"\n`, { mode: 0o700 });
+    await chmod(wrapper, 0o700);
+    const manifest = JSON.stringify({
+        name: hostName,
+        description: 'Sponge: save pages with your own sign-ins',
+        path: wrapper,
+        type: 'stdio',
+        allowed_origins: [`chrome-extension://${extensionId}/`],
+    }, null, 2);
+    const registered = [];
+    for (const { browser, dir } of hostDirectories(options)) {
+        await mkdir(dir, { recursive: true });
+        await writeFile(join(dir, manifestName), manifest + '\n', { mode: 0o644 });
+        registered.push(browser);
+    }
+    return { registered, extension: extensionDirectory(), extensionId };
+}
+/** Which browsers have the host registered, and whether an extension is connected for this library now. */
+export async function browserStatus(libraryRoot, options = {}) {
+    const home = options.home ?? homedir(), wrapper = wrapperPath(home), registered = [];
+    for (const { browser, dir } of hostDirectories(options)) {
+        try {
+            if (JSON.parse(await readFile(join(dir, manifestName), 'utf8')).path === wrapper)
+                registered.push(browser);
+        }
+        catch {
+            // Not registered with this browser.
+        }
+    }
+    let bound = false;
+    try {
+        bound = (await readFile(wrapper, 'utf8')).includes(quote(bridgeSocket(libraryRoot, home)));
+    }
+    catch {
+        bound = false;
+    }
+    const answer = await askBridge(bridgeSocket(libraryRoot, home), { op: 'status' }, { timeoutMs: 3000, maxBytes: 4096 });
+    return {
+        registered,
+        bound,
+        connected: answer.ok,
+        ...(answer.ok && 'background' in answer ? { extension: answer.extension, background: answer.background } : {}),
+    };
+}
+/** Removes the registrations that point at Sponge's host; another program's manifest is left alone. */
+export async function removeBrowser(options = {}) {
+    const home = options.home ?? homedir(), wrapper = wrapperPath(home), removed = [];
+    for (const { browser, dir } of hostDirectories(options)) {
+        const file = join(dir, manifestName);
+        try {
+            if (JSON.parse(await readFile(file, 'utf8')).path !== wrapper)
+                continue;
+        }
+        catch {
+            continue;
+        }
+        await rm(file, { force: true });
+        removed.push(browser);
+    }
+    await rm(wrapper, { force: true });
+    return { removed };
+}

package/dist/src/capture/network.d.ts

@@ -0,0 +1,56 @@
+export type FetchFailureCode = 'invalid-url' | 'private-network' | 'dns' | 'timeout' | 'http' | 'redirect' | 'too-large' | 'network';
+export declare class FetchFailure extends Error {
+    readonly code: FetchFailureCode;
+    readonly status: number | null;
+    constructor(code: FetchFailureCode, message: string, options?: ErrorOptions & {
+        status?: number;
+    });
+}
+export interface SafeFetchOptions {
+    timeoutMs: number;
+    maxBytes: number;
+    allowPrivateNetwork: boolean;
+    userAgent: string;
+    referer?: string;
+    accept?: string;
+    retries?: number;
+    maxRedirects?: number;
+}
+export interface SafeFetchResult {
+    bytes: Uint8Array;
+    finalUrl: URL;
+    status: number;
+    contentType: string | null;
+}
+interface ResolvedAddress {
+    address: string;
+    family: 4 | 6;
+}
+export interface PinnedRequest {
+    url: URL;
+    address: ResolvedAddress;
+    headers: Headers;
+    signal: AbortSignal;
+}
+export interface PinnedResponse {
+    status: number;
+    headers: Headers;
+    body: AsyncIterable<unknown> | null;
+    cancel: () => void;
+}
+export interface SafeFetchDependencies {
+    resolveHostname: (hostname: string) => Promise<readonly ResolvedAddress[]>;
+    transport: (request: PinnedRequest) => Promise<PinnedResponse>;
+    /** Test seam. Production callers use the system interface snapshot. */
+    localAddresses: () => readonly string[];
+}
+/** True for loopback, link-local, private, reserved, multicast and unspecified addresses. */
+export declare function isPrivateAddress(address: string): boolean;
+export declare function isPrivateHostname(hostname: string): boolean;
+/** Refuse a private, reserved or host-assigned target unless explicitly allowed. */
+export declare function assertPublicUrl(url: URL, allowPrivateNetwork: boolean, timeoutMs?: number): Promise<void>;
+/** A bounded fetcher: every hop is resolved, validated and pinned; redirects are re-checked; bodies are capped. */
+export declare function createSafeFetch(dependencies?: Partial<SafeFetchDependencies>): (url: URL, options: SafeFetchOptions) => Promise<SafeFetchResult>;
+export declare const safeFetch: (url: URL, options: SafeFetchOptions) => Promise<SafeFetchResult>;
+export declare function decodeBytes(bytes: Uint8Array, contentType: string | null): string;
+export {};

package/dist/src/capture/network.js

@@ -0,0 +1,453 @@
+// Ported from Wordcell's clip/network.ts (MIT, see native/NOTICES-capture.txt). Cookie forwarding, connection
+// pooling and manual redirects are dropped: Sponge captures only public pages over a pinned, bounded fetch.
+import { lookup } from 'node:dns/promises';
+import { request as requestHttp } from 'node:http';
+import { request as requestHttps } from 'node:https';
+import { isIP } from 'node:net';
+import { networkInterfaces } from 'node:os';
+import { setTimeout as sleep } from 'node:timers/promises';
+export class FetchFailure extends Error {
+    code;
+    status;
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = 'FetchFailure';
+        this.code = code;
+        this.status = options?.status ?? null;
+    }
+}
+const privateSuffixes = ['.localhost', '.local', '.internal', '.home', '.lan'];
+const retryableStatuses = new Set([408, 425, 429, 500, 502, 503, 504]);
+const normalizeHostname = (hostname) => (hostname.startsWith('[') && hostname.endsWith(']') ? hostname.slice(1, -1) : hostname)
+    .toLowerCase()
+    .replace(/\.$/, '');
+function parseIpv4(address) {
+    const pieces = address.split('.');
+    if (pieces.length !== 4)
+        return null;
+    const numbers = pieces.map(Number);
+    return numbers.every((piece) => Number.isInteger(piece) && piece >= 0 && piece <= 255) ? numbers : null;
+}
+function parseIpv6(address) {
+    let value = address;
+    const separator = value.lastIndexOf(':');
+    const tail = separator >= 0 ? value.slice(separator + 1) : value;
+    if (tail.includes('.')) {
+        const ipv4 = parseIpv4(tail);
+        if (ipv4 === null || separator < 0)
+            return null;
+        value = `${value.slice(0, separator)}:${((ipv4[0] << 8) | ipv4[1]).toString(16)}:${((ipv4[2] << 8) | ipv4[3]).toString(16)}`;
+    }
+    const compression = value.indexOf('::');
+    if (compression !== -1 && compression !== value.lastIndexOf('::'))
+        return null;
+    const leftText = compression === -1 ? value : value.slice(0, compression);
+    const rightText = compression === -1 ? '' : value.slice(compression + 2);
+    const left = leftText === '' ? [] : leftText.split(':');
+    const right = rightText === '' ? [] : rightText.split(':');
+    if ([...left, ...right].some((part) => !/^[0-9a-f]{1,4}$/i.test(part)))
+        return null;
+    const missing = 8 - left.length - right.length;
+    if ((compression === -1 && missing !== 0) || (compression !== -1 && missing < 1))
+        return null;
+    return [
+        ...left.map((part) => Number.parseInt(part, 16)),
+        ...Array.from({ length: missing }, () => 0),
+        ...right.map((part) => Number.parseInt(part, 16)),
+    ];
+}
+function withoutScope(address) {
+    const normalized = normalizeHostname(address);
+    const scope = normalized.indexOf('%');
+    return scope === -1 ? normalized : normalized.slice(0, scope);
+}
+/** The IPv4 address an IPv4-mapped or IPv4-compatible IPv6 address reaches, if any. */
+function embeddedIpv4(groups) {
+    const compatible = groups.slice(0, 6).every((group) => group === 0);
+    const mapped = groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff;
+    if (!compatible && !mapped)
+        return null;
+    const high = groups[6], low = groups[7];
+    return `${high >>> 8}.${high & 0xff}.${low >>> 8}.${low & 0xff}`;
+}
+/** Syntax-independent keys for one IP literal; an embedded IPv4 address also gets its IPv4 key. */
+function addressKeys(address) {
+    const normalized = withoutScope(address);
+    const version = isIP(normalized);
+    if (version === 4) {
+        const pieces = parseIpv4(normalized);
+        return pieces === null ? [] : [`4:${pieces.join('.')}`];
+    }
+    if (version !== 6)
+        return [];
+    const groups = parseIpv6(normalized);
+    if (groups === null)
+        return [];
+    const keys = [`6:${groups.map((group) => group.toString(16).padStart(4, '0')).join(':')}`];
+    const ipv4 = embeddedIpv4(groups);
+    if (ipv4 !== null)
+        keys.push(`4:${ipv4}`);
+    return keys;
+}
+function systemLocalAddresses() {
+    try {
+        return Object.values(networkInterfaces()).flatMap((records) => (records ?? []).map((record) => record.address));
+    }
+    catch (error) {
+        throw new FetchFailure('network', 'could not inspect local network interfaces', { cause: error });
+    }
+}
+function localKeys(provider) {
+    let addresses;
+    try {
+        addresses = provider();
+    }
+    catch (error) {
+        if (error instanceof FetchFailure)
+            throw error;
+        throw new FetchFailure('network', 'could not inspect local network interfaces', { cause: error });
+    }
+    return new Set(addresses.flatMap(addressKeys));
+}
+const isLocalAddress = (address, keys) => addressKeys(address).some((key) => keys.has(key));
+/** True for loopback, link-local, private, reserved, multicast and unspecified addresses. */
+export function isPrivateAddress(address) {
+    const normalized = withoutScope(address);
+    const version = isIP(normalized);
+    if (version === 4) {
+        const pieces = parseIpv4(normalized);
+        if (pieces === null)
+            return true;
+        const [a, b, c] = pieces;
+        return (a === 0 ||
+            a === 10 ||
+            a === 127 ||
+            (a === 100 && b >= 64 && b <= 127) ||
+            (a === 169 && b === 254) ||
+            (a === 172 && b >= 16 && b <= 31) ||
+            (a === 192 && (b === 0 || b === 168)) ||
+            (a === 198 && (b === 18 || b === 19)) ||
+            (a === 198 && b === 51 && c === 100) ||
+            (a === 203 && b === 0 && c === 113) ||
+            a >= 224);
+    }
+    if (version === 6) {
+        const groups = parseIpv6(normalized);
+        if (groups === null)
+            return true;
+        const ipv4 = embeddedIpv4(groups);
+        if (ipv4 !== null)
+            return isPrivateAddress(ipv4);
+        const [first, second] = groups;
+        return ((first & 0xfe00) === 0xfc00 ||
+            (first & 0xffc0) === 0xfe80 ||
+            (first & 0xffc0) === 0xfec0 ||
+            (first & 0xff00) === 0xff00 ||
+            first === 0x0064 ||
+            first === 0x0100 ||
+            (first === 0x2001 && second <= 0x01ff) ||
+            (first === 0x2001 && second === 0x0db8) ||
+            first === 0x2002 ||
+            first === 0x3ffe ||
+            (first === 0x3fff && (second & 0xf000) === 0) ||
+            first === 0x5f00);
+    }
+    return true;
+}
+export function isPrivateHostname(hostname) {
+    const normalized = normalizeHostname(hostname);
+    if (normalized === 'localhost' || normalized === 'localhost.localdomain')
+        return true;
+    if (privateSuffixes.some((suffix) => normalized.endsWith(suffix)))
+        return true;
+    return isIP(normalized) !== 0 && isPrivateAddress(normalized);
+}
+async function systemResolve(hostname) {
+    const answers = await lookup(hostname, { all: true, verbatim: true });
+    return answers.flatMap((answer) => answer.family === 4 || answer.family === 6 ? [{ address: answer.address, family: answer.family }] : []);
+}
+async function beforeDeadline(promise, deadline, message) {
+    const remaining = deadline - Date.now();
+    if (remaining <= 0)
+        throw new FetchFailure('timeout', message);
+    let timer;
+    try {
+        return await Promise.race([
+            promise,
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new FetchFailure('timeout', message)), remaining);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+async function resolveTarget(url, allowPrivateNetwork, dependencies, deadline, timeoutMs) {
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        throw new FetchFailure('invalid-url', `unsupported URL protocol: ${url.protocol}`);
+    if (url.username !== '' || url.password !== '')
+        throw new FetchFailure('invalid-url', 'credential-bearing URLs are not accepted');
+    const hostname = normalizeHostname(url.hostname);
+    if (!allowPrivateNetwork && isPrivateHostname(hostname))
+        throw new FetchFailure('private-network', `private-network URL: ${url.origin}`);
+    const local = allowPrivateNetwork ? new Set() : localKeys(dependencies.localAddresses);
+    const literal = isIP(hostname);
+    if (literal === 4 || literal === 6) {
+        if (isLocalAddress(hostname, local))
+            throw new FetchFailure('private-network', `address ${hostname} is assigned to a local interface`);
+        return [{ address: hostname, family: literal }];
+    }
+    let answers;
+    try {
+        answers = await beforeDeadline(dependencies.resolveHostname(hostname), deadline, `request timed out after ${timeoutMs}ms while resolving ${hostname}`);
+    }
+    catch (error) {
+        if (error instanceof FetchFailure)
+            throw error;
+        throw new FetchFailure('dns', `could not resolve ${hostname}`, { cause: error });
+    }
+    if (answers.length === 0)
+        throw new FetchFailure('dns', `could not resolve ${hostname}`);
+    // DNS can take most of the deadline; an interface that appeared meanwhile must not become a local hop.
+    if (!allowPrivateNetwork)
+        for (const key of localKeys(dependencies.localAddresses))
+            local.add(key);
+    const unique = new Map();
+    for (const answer of answers) {
+        const address = normalizeHostname(answer.address);
+        const family = isIP(address);
+        if ((family !== 4 && family !== 6) || family !== answer.family)
+            throw new FetchFailure('dns', `${hostname} returned an invalid DNS answer`);
+        if (!allowPrivateNetwork && (isPrivateAddress(address) || isLocalAddress(address, local)))
+            throw new FetchFailure('private-network', `${hostname} resolves to a private or local address`);
+        unique.set(`${family}:${address}`, { address, family });
+    }
+    return [...unique.values()];
+}
+/** Refuse a private, reserved or host-assigned target unless explicitly allowed. */
+export async function assertPublicUrl(url, allowPrivateNetwork, timeoutMs = 30_000) {
+    await resolveTarget(url, allowPrivateNetwork, { resolveHostname: systemResolve, localAddresses: systemLocalAddresses }, Date.now() + timeoutMs, timeoutMs);
+}
+/** A lookup callback that can only answer with the address already validated. */
+const pinnedLookup = ({ address, family }) => (_hostname, options, callback) => queueMicrotask(() => options.all === true ? callback(null, [{ address, family }]) : callback(null, address, family));
+function responseHeaders(headers) {
+    const result = new Headers();
+    for (const [name, value] of Object.entries(headers)) {
+        if (typeof value === 'string')
+            result.append(name, value);
+        else if (Array.isArray(value))
+            for (const item of value)
+                result.append(name, item);
+    }
+    return result;
+}
+/** One HTTP(S) request through one validated address. No DNS, redirects, retries or body accumulation. */
+function requestPinned(request) {
+    const hostname = normalizeHostname(request.url.hostname);
+    if (isIP(request.address.address) !== request.address.family)
+        return Promise.reject(new Error('pinned address does not match its family'));
+    const literal = isIP(hostname);
+    if (literal !== 0 &&
+        (literal !== request.address.family || addressKeys(hostname)[0] !== addressKeys(request.address.address)[0]))
+        return Promise.reject(new Error('IP-literal hostname does not match the pinned address'));
+    const headers = {};
+    request.headers.forEach((value, name) => (headers[name] = value));
+    const options = {
+        protocol: request.url.protocol,
+        hostname,
+        method: 'GET',
+        path: `${request.url.pathname}${request.url.search}`,
+        headers,
+        lookup: pinnedLookup(request.address),
+        family: request.address.family,
+        signal: request.signal,
+        agent: false,
+        ...(request.url.port === '' ? {} : { port: request.url.port }),
+    };
+    return new Promise((resolve, reject) => {
+        const onResponse = (response) => {
+            if (response.statusCode === undefined) {
+                response.destroy();
+                reject(new Error('HTTP response omitted a status code'));
+                return;
+            }
+            resolve({
+                status: response.statusCode,
+                headers: responseHeaders(response.headers),
+                body: response,
+                cancel: () => response.destroy(),
+            });
+        };
+        const client = request.url.protocol === 'https:'
+            ? requestHttps({ ...options, ...(literal === 0 ? { servername: hostname } : {}) }, onResponse)
+            : requestHttp(options, onResponse);
+        // A handshake error can be followed by a socket teardown error; keep the listener for the request lifetime.
+        client.on('error', reject);
+        client.end();
+    });
+}
+function retryDelay(response, attempt) {
+    const header = response?.headers.get('retry-after')?.trim();
+    if (header) {
+        const seconds = Number(header);
+        if (Number.isFinite(seconds) && seconds >= 0)
+            return Math.min(seconds * 1000, 5000);
+        const date = Date.parse(header);
+        if (!Number.isNaN(date))
+            return Math.min(Math.max(date - Date.now(), 0), 5000);
+    }
+    return Math.min(250 * 2 ** attempt, 2000);
+}
+async function readBounded(response, maxBytes) {
+    const declared = Number(response.headers.get('content-length') ?? NaN);
+    if (Number.isFinite(declared) && declared > maxBytes) {
+        response.cancel();
+        throw new FetchFailure('too-large', `response declares ${declared} bytes; limit is ${maxBytes}`);
+    }
+    if (response.body === null)
+        return new Uint8Array();
+    const chunks = [];
+    let total = 0;
+    for await (const chunk of response.body) {
+        if (!(chunk instanceof Uint8Array)) {
+            response.cancel();
+            throw new FetchFailure('network', 'response body yielded a non-byte chunk');
+        }
+        total += chunk.byteLength;
+        if (total > maxBytes) {
+            response.cancel();
+            throw new FetchFailure('too-large', `response exceeded ${maxBytes} bytes`);
+        }
+        chunks.push(chunk);
+    }
+    return Buffer.concat(chunks, total);
+}
+function requestHeaders(current, options) {
+    const headers = new Headers({
+        Accept: options.accept ?? 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5',
+        'Accept-Encoding': 'identity',
+        'User-Agent': options.userAgent,
+    });
+    if (options.referer !== undefined) {
+        try {
+            const referer = new URL(options.referer);
+            headers.set('Referer', referer.origin === current.origin ? referer.href : `${referer.origin}/`);
+        }
+        catch {
+            // An invalid optional referrer is dropped rather than forwarded.
+        }
+    }
+    return headers;
+}
+/** A bounded fetcher: every hop is resolved, validated and pinned; redirects are re-checked; bodies are capped. */
+export function createSafeFetch(dependencies = {}) {
+    const resolve = {
+        resolveHostname: dependencies.resolveHostname ?? systemResolve,
+        localAddresses: dependencies.localAddresses ?? systemLocalAddresses,
+    };
+    const transport = dependencies.transport ?? requestPinned;
+    return async (url, options) => {
+        const maxRedirects = options.maxRedirects ?? 8, retries = options.retries ?? 2, deadline = Date.now() + options.timeoutMs;
+        let current = new URL(url);
+        for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
+            const timedOut = `request timed out after ${options.timeoutMs}ms: ${current.href}`;
+            const addresses = await resolveTarget(current, options.allowPrivateNetwork, resolve, deadline, options.timeoutMs);
+            let response = null, lastError, controller = null, timer;
+            for (let attempt = 0; attempt <= retries; attempt += 1) {
+                const remaining = deadline - Date.now();
+                if (remaining <= 0)
+                    throw new FetchFailure('timeout', timedOut);
+                const attemptController = new AbortController();
+                const attemptTimer = setTimeout(() => attemptController.abort(), remaining);
+                try {
+                    response = await beforeDeadline(transport({
+                        url: new URL(current),
+                        address: addresses[attempt % addresses.length],
+                        headers: requestHeaders(current, options),
+                        signal: attemptController.signal,
+                    }), deadline, timedOut);
+                    if (!retryableStatuses.has(response.status) || attempt === retries) {
+                        controller = attemptController;
+                        timer = attemptTimer;
+                        break;
+                    }
+                    attemptController.abort();
+                    response.cancel();
+                    clearTimeout(attemptTimer);
+                }
+                catch (error) {
+                    lastError = error;
+                    // The deadline race and the abort timer are separate callbacks; abort so a pending transport frees its socket.
+                    attemptController.abort();
+                    clearTimeout(attemptTimer);
+                    if (error instanceof FetchFailure && error.code === 'timeout')
+                        throw error;
+                    if (attempt === retries) {
+                        if (attemptController.signal.aborted && Date.now() >= deadline)
+                            throw new FetchFailure('timeout', timedOut, { cause: error });
+                        if (error instanceof FetchFailure)
+                            throw error;
+                        throw new FetchFailure('network', `request failed: ${current.href}`, { cause: error });
+                    }
+                }
+                const delay = Math.min(retryDelay(response, attempt), Math.max(0, deadline - Date.now()));
+                if (delay > 0)
+                    await sleep(delay);
+            }
+            if (response === null)
+                throw new FetchFailure('network', `request failed: ${current.href}`, { cause: lastError });
+            if (response.status >= 300 && response.status < 400) {
+                try {
+                    const location = response.headers.get('location');
+                    if (location === null)
+                        throw new FetchFailure('redirect', `HTTP ${response.status} omitted Location`);
+                    if (redirects === maxRedirects)
+                        throw new FetchFailure('redirect', `more than ${maxRedirects} redirects`);
+                    try {
+                        current = new URL(location, current);
+                    }
+                    catch (error) {
+                        throw new FetchFailure('redirect', 'invalid redirect target', { cause: error });
+                    }
+                    continue;
+                }
+                finally {
+                    controller?.abort();
+                    response.cancel();
+                    clearTimeout(timer);
+                }
+            }
+            if (response.status < 200 || response.status >= 300) {
+                clearTimeout(timer);
+                controller?.abort();
+                response.cancel();
+                throw new FetchFailure('http', `HTTP ${response.status} for ${current.href}`, { status: response.status });
+            }
+            const bodyTimedOut = `response body timed out after ${options.timeoutMs}ms: ${current.href}`;
+            try {
+                const bytes = await beforeDeadline(readBounded(response, options.maxBytes), deadline, bodyTimedOut);
+                return { bytes, finalUrl: current, status: response.status, contentType: response.headers.get('content-type') };
+            }
+            catch (error) {
+                if (controller?.signal.aborted || (error instanceof FetchFailure && error.code === 'timeout')) {
+                    response.cancel();
+                    throw new FetchFailure('timeout', bodyTimedOut, { cause: error });
+                }
+                if (error instanceof FetchFailure)
+                    throw error;
+                throw new FetchFailure('network', `response body failed: ${current.href}`, { cause: error });
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        }
+        throw new FetchFailure('redirect', `more than ${maxRedirects} redirects`);
+    };
+}
+export const safeFetch = createSafeFetch();
+export function decodeBytes(bytes, contentType) {
+    const charset = /charset\s*=\s*["']?([^;"'\s]+)/i.exec(contentType ?? '')?.[1]?.toLowerCase();
+    const encoding = charset === 'iso-8859-1' || charset === 'windows-1252' ? 'windows-1252' : 'utf-8';
+    return new TextDecoder(encoding, { fatal: false }).decode(bytes);
+}

package/dist/src/capture/types.d.ts

@@ -0,0 +1,26 @@
+export type CaptureScope = 'auto' | 'page' | 'thread' | 'comments';
+export type AcquisitionMethod = 'http' | 'hacker-news-api' | 'bluesky-api' | 'reddit-json' | 'browser-cdp' | 'browser-fetch' | 'browser-render';
+export type AcquiredPage = {
+    readonly body: string;
+    readonly contentType: string | null;
+    readonly finalUrl: URL;
+    readonly method: AcquisitionMethod;
+    readonly warnings: readonly string[];
+    readonly browserTitle?: string;
+    readonly contentTruncated?: boolean;
+    readonly renderedText?: string;
+    readonly renderedTextTruncated?: boolean;
+    readonly renderedTextByteLimit?: number;
+};
+/** The bounded knobs one capture runs under; every lane reads the same values. */
+export type CaptureOptions = {
+    readonly url: URL;
+    readonly scope: CaptureScope;
+    readonly timeoutMs: number;
+    readonly maxItems: number;
+    readonly maxDepth: number;
+    readonly maxHtmlBytes: number;
+    readonly allowPrivateNetwork: boolean;
+    readonly userAgent: string;
+};
+export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";

package/dist/src/capture/types.js

@@ -0,0 +1 @@
+export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';