@db3.ai/app 0.1.0-beta.1 → 0.1.0-beta.4

package/dist/network/guardedFetch.js

@@ -0,0 +1,133 @@
+import { Agent } from 'undici';
+import { BlockedUrlError } from './BlockedUrlError.js';
+import { assertPublicUrl } from './assertPublicUrl.js';
+import { publicAddressLookup } from './publicAddressLookup.js';
+const DEFAULT_MAX_REDIRECTS = 4;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const CROSS_ORIGIN_SENSITIVE_HEADERS = ['authorization', 'cookie', 'proxy-authorization'];
+let publicDispatcher;
+let privateDispatcher;
+/**
+ * Fetches a user-supplied URL without allowing it to reach private networks.
+ *
+ * Each hop is validated before it is requested, and the connection itself uses
+ * {@link publicAddressLookup}, so the address DNS returns at connect time is
+ * the address that was checked. Redirects are followed manually: every target
+ * is validated, `303` (and `301`/`302` after `POST`) switch to `GET` without a
+ * body as `fetch` does, and credentials (`Authorization`, `Cookie`) are removed
+ * when a redirect leaves the original origin.
+ *
+ * `allowPrivate` permits private destinations without weakening TLS certificate
+ * verification. Local HTTPS receivers must present a certificate trusted by Node.
+ *
+ * The request is sent with the runtime's global `fetch`, so test doubles that
+ * replace `globalThis.fetch` continue to work.
+ *
+ * @param rawUrl - Absolute HTTP(S) URL to request.
+ * @param options - Fetch init, redirect handling and destination policy.
+ * @returns Final response and the URL that produced it.
+ * @throws {BlockedUrlError} When any hop targets a refused destination or the
+ * redirect limit is exceeded.
+ *
+ * @example
+ * const { response, finalUrl } = await guardedFetch('https://example.com/', {
+ * 	init: { headers: { accept: 'text/html' }, signal: AbortSignal.timeout(10_000) },
+ * });
+ */
+export async function guardedFetch(rawUrl, options = {}) {
+    const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
+    let current = rawUrl;
+    let init = { ...options.init };
+    for (let hop = 0;; hop++) {
+        await assertPublicUrl(current, options);
+        const response = await fetchHop(current, init, options.allowPrivate === true);
+        const location = response.headers.get('location');
+        if (!REDIRECT_STATUSES.has(response.status) || !location || options.redirect === 'manual') {
+            return { response, finalUrl: current };
+        }
+        await response.body?.cancel();
+        if (hop >= maxRedirects)
+            throw new BlockedUrlError('Too many redirects.');
+        const next = new URL(location, current).toString();
+        init = redirectInit(init, response.status, new URL(current).origin !== new URL(next).origin);
+        current = next;
+    }
+}
+/**
+ * Sends one request hop through the dispatcher matching the destination policy.
+ *
+ * @param url - Already validated hop URL.
+ * @param init - Fetch options for this hop.
+ * @param allowPrivate - Whether private destinations are allowed.
+ * @returns Unfollowed hop response.
+ * @throws {BlockedUrlError} When the connect-time lookup refuses the address.
+ */
+async function fetchHop(url, init, allowPrivate) {
+    try {
+        // The caller checks each URL/redirect; this dispatcher validates the actual
+        // socket address, including DNS changes between validation and connection.
+        return await fetch(url, {
+            ...init,
+            redirect: 'manual',
+            dispatcher: hopDispatcher(allowPrivate),
+        });
+    }
+    catch (error) {
+        throw blockedCause(error) ?? error;
+    }
+}
+/**
+ * Chooses the connection dispatcher for one hop.
+ *
+ * @param allowPrivate - Whether private destinations are allowed.
+ * @returns Shared dispatcher for the hop's policy.
+ */
+function hopDispatcher(allowPrivate) {
+    if (!allowPrivate) {
+        return publicDispatcher ??= new Agent({ connect: { lookup: publicAddressLookup, rejectUnauthorized: true } });
+    }
+    return privateDispatcher ??= new Agent({ connect: { rejectUnauthorized: true } });
+}
+/**
+ * Builds the fetch options for the next redirect hop.
+ *
+ * @param init - Options used for the previous hop.
+ * @param status - Redirect status code.
+ * @param crossOrigin - Whether the redirect leaves the previous origin.
+ * @returns Options for the next hop.
+ */
+function redirectInit(init, status, crossOrigin) {
+    const method = (init.method ?? 'GET').toUpperCase();
+    const switchesToGet = status === 303
+        ? method !== 'GET' && method !== 'HEAD'
+        : (status === 301 || status === 302) && method === 'POST';
+    const headers = new Headers(init.headers);
+    if (switchesToGet) {
+        headers.delete('content-type');
+        headers.delete('content-length');
+    }
+    if (crossOrigin) {
+        for (const name of CROSS_ORIGIN_SENSITIVE_HEADERS)
+            headers.delete(name);
+    }
+    return {
+        ...init,
+        headers,
+        ...(switchesToGet ? { method: 'GET', body: undefined } : {}),
+    };
+}
+/**
+ * Finds a guard rejection wrapped inside a fetch network error.
+ *
+ * @param error - Error thrown by `fetch`.
+ * @returns The wrapped `BlockedUrlError`, or null.
+ */
+function blockedCause(error) {
+    let current = error;
+    for (let depth = 0; depth < 5 && current; depth++) {
+        if (current instanceof BlockedUrlError)
+            return current;
+        current = current.cause;
+    }
+    return null;
+}

package/dist/network/index.d.ts

@@ -0,0 +1,6 @@
+export * from './contracts/index.js';
+export * from './BlockedUrlError.js';
+export { assertPublicUrl } from './assertPublicUrl.js';
+export * from './guardedFetch.js';
+export * from './privateNetworkAddresses.js';
+export * from './publicAddressLookup.js';

package/dist/network/index.js

@@ -0,0 +1,6 @@
+export * from './contracts/index.js';
+export * from './BlockedUrlError.js';
+export { assertPublicUrl } from './assertPublicUrl.js';
+export * from './guardedFetch.js';
+export * from './privateNetworkAddresses.js';
+export * from './publicAddressLookup.js';

package/dist/network/playwright/GuardedBrowserProxy.d.ts

@@ -0,0 +1,32 @@
+import { BlockedUrlError } from '../BlockedUrlError.js';
+/** Internal connection policy; the browser adapter owns request attribution. */
+interface ProxyOptions {
+    allowPrivate?: boolean;
+    requestTimeoutMs?: number;
+    onBlocked: (url: string, error: BlockedUrlError) => void;
+}
+/**
+ * Context-owned loopback proxy enforcing the destination of every browser socket.
+ *
+ * HTTP bodies stream without buffering. HTTPS remains end-to-end TLS through a
+ * validated CONNECT tunnel, preserving browser certificates, cookies and origins.
+ * Redirects create normal browser requests and cannot escape the proxy's policy.
+ */
+export declare class GuardedBrowserProxy {
+    #private;
+    /** Creates an unstarted, isolated browser proxy. */
+    constructor(options: ProxyOptions);
+    /**
+     * Starts the proxy on an ephemeral loopback port.
+     * @returns Chromium proxy options; the bypass rule includes loopback traffic.
+     */
+    start(): Promise<{
+        server: string;
+        bypass: string;
+        username: string;
+        password: string;
+    }>;
+    /** Stops accepting requests and synchronously tears down every owned socket. */
+    close(): void;
+}
+export {};

package/dist/network/playwright/GuardedBrowserProxy.js

@@ -0,0 +1,172 @@
+import { randomBytes } from 'node:crypto';
+import { createServer, request as httpRequest } from 'node:http';
+import { connect } from 'node:net';
+import { assertPublicUrl } from '../assertPublicUrl.js';
+import { BlockedUrlError } from '../BlockedUrlError.js';
+import { normalizedUrlHostname } from '../privateNetworkAddresses.js';
+import { publicAddressLookup } from '../publicAddressLookup.js';
+/**
+ * Context-owned loopback proxy enforcing the destination of every browser socket.
+ *
+ * HTTP bodies stream without buffering. HTTPS remains end-to-end TLS through a
+ * validated CONNECT tunnel, preserving browser certificates, cookies and origins.
+ * Redirects create normal browser requests and cannot escape the proxy's policy.
+ */
+export class GuardedBrowserProxy {
+    #options;
+    #sockets = new Set();
+    #server = createServer();
+    #username = 'db3-browser';
+    #password = randomBytes(32).toString('hex');
+    #closed = false;
+    /** Creates an unstarted, isolated browser proxy. */
+    constructor(options) {
+        this.#options = options;
+        this.#server.on('connection', socket => this.#track(socket));
+        this.#server.on('request', (request, response) => { void this.#forward(request, response); });
+        this.#server.on('connect', (request, socket, head) => { void this.#tunnel(request, socket, head); });
+        this.#server.on('clientError', (_error, socket) => socket.destroy());
+    }
+    /**
+     * Starts the proxy on an ephemeral loopback port.
+     * @returns Chromium proxy options; the bypass rule includes loopback traffic.
+     */
+    async start() {
+        await new Promise((resolve, reject) => {
+            this.#server.once('error', reject);
+            this.#server.listen(0, '127.0.0.1', () => {
+                this.#server.off('error', reject);
+                resolve();
+            });
+        });
+        const address = this.#server.address();
+        if (!address || typeof address === 'string')
+            throw new Error('Browser proxy did not open a TCP port.');
+        return { server: `http://127.0.0.1:${address.port}`, bypass: '<-loopback>', username: this.#username, password: this.#password };
+    }
+    /** Stops accepting requests and synchronously tears down every owned socket. */
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        for (const socket of this.#sockets)
+            socket.destroy();
+        this.#server.close();
+    }
+    /**
+     * Registers socket cleanup and an inactivity timeout without buffering data.
+     * @param socket - Accepted browser socket or outgoing destination connection.
+     */
+    #track(socket) {
+        if (this.#closed) {
+            socket.destroy();
+            return;
+        }
+        this.#sockets.add(socket);
+        socket.once('close', () => this.#sockets.delete(socket));
+        socket.on('error', () => socket.destroy());
+        if ('setTimeout' in socket)
+            socket.setTimeout(this.#options.requestTimeoutMs ?? 30_000, () => socket.destroy());
+    }
+    /** Checks the per-context proxy credential supplied by Chromium. */
+    #authorized(request) {
+        return request.headers['proxy-authorization'] === `Basic ${Buffer.from(`${this.#username}:${this.#password}`).toString('base64')}`;
+    }
+    /**
+     * Streams a normal HTTP proxy request through a validated connection.
+     * @param request - Browser's absolute-form request.
+     * @param response - Response streamed back to the browser.
+     */
+    async #forward(request, response) {
+        if (!this.#authorized(request)) {
+            response.writeHead(407, { 'proxy-authenticate': 'Basic realm="db3-browser"' }).end();
+            return;
+        }
+        const url = request.url ?? '';
+        try {
+            await assertPublicUrl(url, this.#options);
+            const destination = new URL(url);
+            if (destination.protocol !== 'http:')
+                throw new BlockedUrlError('HTTPS browser requests must use a CONNECT tunnel.');
+            if (this.#closed || response.destroyed)
+                return;
+            const headers = { ...request.headers, host: destination.host };
+            delete headers['proxy-authorization'];
+            delete headers['proxy-connection'];
+            const outgoing = httpRequest(destination, {
+                method: request.method,
+                headers,
+                agent: false,
+                lookup: this.#options.allowPrivate ? undefined : publicAddressLookup,
+            }, upstream => {
+                response.writeHead(upstream.statusCode ?? 502, upstream.headers);
+                upstream.on('error', () => response.destroy());
+                upstream.pipe(response);
+            });
+            outgoing.on('socket', socket => this.#track(socket));
+            outgoing.on('error', error => {
+                this.#reportBlocked(url, error);
+                response.destroy();
+            });
+            request.on('aborted', () => outgoing.destroy());
+            request.on('error', () => outgoing.destroy());
+            response.on('close', () => outgoing.destroy());
+            request.pipe(outgoing);
+        }
+        catch (error) {
+            this.#reportBlocked(url, error);
+            response.destroy();
+        }
+    }
+    /**
+     * Opens a TLS tunnel only after validating its actual destination connection.
+     * @param request - Chromium CONNECT authority.
+     * @param browser - Browser side of the opaque tunnel.
+     * @param head - Bytes received with the CONNECT request.
+     */
+    async #tunnel(request, browser, head) {
+        if (!this.#authorized(request)) {
+            browser.end('HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm="db3-browser"\r\nContent-Length: 0\r\n\r\n');
+            return;
+        }
+        const url = `https://${request.url ?? ''}/`;
+        try {
+            await assertPublicUrl(url, this.#options);
+            if (this.#closed || browser.destroyed)
+                return;
+            const destination = new URL(url);
+            const upstream = connect({
+                host: normalizedUrlHostname(destination.hostname),
+                port: Number(destination.port || 443),
+                lookup: this.#options.allowPrivate ? undefined : publicAddressLookup,
+            });
+            this.#track(upstream);
+            browser.once('close', () => upstream.destroy());
+            upstream.once('close', () => browser.destroy());
+            upstream.on('error', error => {
+                this.#reportBlocked(url, error);
+                browser.destroy();
+            });
+            upstream.once('connect', () => {
+                browser.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+                if (head.length)
+                    upstream.write(head);
+                upstream.pipe(browser);
+                browser.pipe(upstream);
+            });
+        }
+        catch (error) {
+            this.#reportBlocked(url, error);
+            browser.destroy();
+        }
+    }
+    /** Records policy refusals without allowing observer failures to escape I/O. */
+    #reportBlocked(url, error) {
+        if (!(error instanceof BlockedUrlError))
+            return;
+        try {
+            this.#options.onBlocked(url, error);
+        }
+        catch { /* Observers cannot change network enforcement. */ }
+    }
+}

package/dist/network/playwright/contracts/GuardedBrowserContextOptions.d.ts

@@ -0,0 +1,37 @@
+import type { Request } from 'playwright';
+import type { BlockedUrlError } from '../../BlockedUrlError.js';
+import type { PublicUrlOptions } from '../../contracts/index.js';
+/**
+ * Details passed to the observer when the guard refuses a browser request.
+ */
+export interface BlockedBrowserRequest {
+    /** URL the page tried to load. */
+    url: string;
+    /** Guard rejection with a caller-safe message. */
+    error: BlockedUrlError;
+    /**
+     * Playwright request that was aborted.
+     *
+     * Use `request.isNavigationRequest()` and `request.frame()` to tell whether
+     * the page's own navigation was refused rather than an embedded resource.
+     */
+    request: Request;
+}
+/**
+ * Network policy for a browser context whose traffic is routed through the guard.
+ */
+export interface GuardedBrowserContextOptions extends PublicUrlOptions {
+    /**
+     * Playwright resource types aborted without a network request, for example
+     * `['image', 'font', 'media']` when only the rendered DOM is needed.
+     */
+    blockedResourceTypes?: readonly string[];
+    /** Socket inactivity timeout in milliseconds. Defaults to 30 seconds. */
+    requestTimeoutMs?: number;
+    /**
+     * Observer called for each request refused by the destination policy.
+     *
+     * The request is aborted whether or not an observer is supplied.
+     */
+    onBlocked?: (blocked: BlockedBrowserRequest) => void;
+}

package/dist/network/playwright/contracts/GuardedBrowserContextOptions.js

@@ -0,0 +1 @@
+export {};

package/dist/network/playwright/contracts/index.d.ts

@@ -0,0 +1 @@
+export * from './GuardedBrowserContextOptions.js';

package/dist/network/playwright/contracts/index.js

@@ -0,0 +1 @@
+export * from './GuardedBrowserContextOptions.js';

package/dist/network/playwright/index.d.ts

@@ -0,0 +1,2 @@
+export * from './contracts/index.js';
+export * from './newGuardedBrowserContext.js';

package/dist/network/playwright/index.js

@@ -0,0 +1,2 @@
+export * from './contracts/index.js';
+export * from './newGuardedBrowserContext.js';

package/dist/network/playwright/newGuardedBrowserContext.d.ts

@@ -0,0 +1,21 @@
+import type { Browser, BrowserContext, BrowserContextOptions } from 'playwright';
+import type * as playwrightNetwork from './contracts/index.js';
+/**
+ * Creates a Chromium context with connection-level private-network protection.
+ *
+ * All HTTP requests and HTTPS tunnels use a context-owned authenticated proxy.
+ * Its socket lookup validates DNS at connect time, including redirect hops that
+ * Playwright does not route again. Chromium retains native redirects, cookies,
+ * response streaming and TLS verification. Closing the context closes its proxy.
+ * Service workers and WebSockets are disabled; callers must also disable WebRTC
+ * non-proxied UDP when launching Chromium for untrusted pages.
+ *
+ * @param browser - Launched Chromium browser.
+ * @param contextOptions - Browser settings; upstream proxy overrides are refused.
+ * @param options - Destination policy, resource filters and refusal observer.
+ * @returns Isolated browser context, owned and closed by the caller.
+ * @example
+ * const context = await newGuardedBrowserContext(browser);
+ * try { await (await context.newPage()).goto(url); } finally { await context.close(); }
+ */
+export declare function newGuardedBrowserContext(browser: Browser, contextOptions?: BrowserContextOptions, options?: playwrightNetwork.GuardedBrowserContextOptions): Promise<BrowserContext>;

package/dist/network/playwright/newGuardedBrowserContext.js

@@ -0,0 +1,86 @@
+import { BlockedUrlError } from '../BlockedUrlError.js';
+import { assertPublicUrl } from '../assertPublicUrl.js';
+import { GuardedBrowserProxy } from './GuardedBrowserProxy.js';
+/**
+ * Creates a Chromium context with connection-level private-network protection.
+ *
+ * All HTTP requests and HTTPS tunnels use a context-owned authenticated proxy.
+ * Its socket lookup validates DNS at connect time, including redirect hops that
+ * Playwright does not route again. Chromium retains native redirects, cookies,
+ * response streaming and TLS verification. Closing the context closes its proxy.
+ * Service workers and WebSockets are disabled; callers must also disable WebRTC
+ * non-proxied UDP when launching Chromium for untrusted pages.
+ *
+ * @param browser - Launched Chromium browser.
+ * @param contextOptions - Browser settings; upstream proxy overrides are refused.
+ * @param options - Destination policy, resource filters and refusal observer.
+ * @returns Isolated browser context, owned and closed by the caller.
+ * @example
+ * const context = await newGuardedBrowserContext(browser);
+ * try { await (await context.newPage()).goto(url); } finally { await context.close(); }
+ */
+export async function newGuardedBrowserContext(browser, contextOptions = {}, options = {}) {
+    if (browser.browserType().name() !== 'chromium')
+        throw new Error('Guarded browser contexts require Chromium.');
+    if (contextOptions.proxy)
+        throw new Error('Guarded browser contexts own their proxy configuration.');
+    const requests = new Set();
+    const reported = new WeakSet();
+    const blockedOrigins = new Map();
+    const blockedTypes = new Set(options.blockedResourceTypes ?? []);
+    /** Attributes a connection rejection without exposing observer errors to I/O. */
+    function report(request, error) {
+        if (reported.has(request))
+            return;
+        reported.add(request);
+        try {
+            options.onBlocked?.({ url: request.url(), request, error });
+        }
+        catch { /* Observers cannot change network enforcement. */ }
+    }
+    const proxy = new GuardedBrowserProxy({
+        ...options,
+        onBlocked: (url, error) => {
+            const origin = new URL(url).origin;
+            blockedOrigins.set(origin, error);
+            for (const request of requests)
+                if (new URL(request.url()).origin === origin)
+                    report(request, error);
+        },
+    });
+    let context;
+    try {
+        context = await browser.newContext({ ...contextOptions, proxy: await proxy.start(), serviceWorkers: 'block' });
+        context.once('close', () => { proxy.close(); requests.clear(); blockedOrigins.clear(); });
+        context.on('request', request => {
+            requests.add(request);
+            const error = blockedOrigins.get(new URL(request.url()).origin);
+            if (error)
+                report(request, error);
+        });
+        context.on('requestfinished', request => requests.delete(request));
+        context.on('requestfailed', request => requests.delete(request));
+        await context.route('**/*', async (route) => {
+            try {
+                if (blockedTypes.has(route.request().resourceType())) {
+                    await route.abort('blockedbyclient');
+                    return;
+                }
+                await assertPublicUrl(route.request().url(), options);
+                await route.continue();
+            }
+            catch (error) {
+                if (error instanceof BlockedUrlError)
+                    report(route.request(), error);
+                await route.abort(error instanceof BlockedUrlError ? 'blockedbyclient' : 'failed').catch(() => undefined);
+            }
+        });
+        await context.routeWebSocket('**/*', socket => socket.close());
+        return context;
+    }
+    catch (error) {
+        proxy.close();
+        await context?.close();
+        throw error;
+    }
+}

package/dist/network/privateNetworkAddresses.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Normalizes a URL hostname for address classification.
+ *
+ * WHATWG URL parsing already canonicalizes numeric IPv4 forms such as
+ * `2130706433` and `0x7f.1`. This helper additionally lowercases the value,
+ * removes IPv6 brackets and drops one trailing root-label dot so `localhost.`
+ * and `localhost` are treated the same.
+ *
+ * @param hostname - `URL.hostname` value to normalize.
+ * @returns Lowercase hostname or IP literal without brackets or a trailing dot.
+ *
+ * @example
+ * normalizedUrlHostname(new URL('http://[::1]/').hostname); // '::1'
+ */
+export declare function normalizedUrlHostname(hostname: string): string;
+/**
+ * Checks whether a hostname is reserved for the local machine.
+ *
+ * DNS names are otherwise judged by the addresses they resolve to, so this
+ * only covers names that never need a DNS query to reach loopback.
+ *
+ * @param hostname - Hostname normalized with {@link normalizedUrlHostname}.
+ * @returns True for `localhost` and any `*.localhost` name.
+ */
+export declare function isPrivateHostname(hostname: string): boolean;
+/**
+ * Checks whether an IP address must not be reached by server-side requests.
+ *
+ * The policy blocks loopback, private, link-local (including cloud metadata),
+ * carrier-grade NAT, benchmarking, documentation, multicast and reserved
+ * ranges. IPv6 forms that embed an IPv4 address (mapped, NAT64 and 6to4) are
+ * judged by the embedded address, and tunnelling or deprecated forms that can
+ * reach IPv4 hosts indirectly are blocked outright. Values that are not valid
+ * IP literals return false so callers can resolve them as hostnames instead.
+ *
+ * @param address - IPv4 or IPv6 literal, with or without IPv6 brackets.
+ * @returns True when the address is private, local or reserved.
+ *
+ * @example
+ * isPrivateIpAddress('169.254.169.254'); // true
+ * isPrivateIpAddress('::ffff:7f00:1'); // true (IPv4-mapped 127.0.0.1)
+ * isPrivateIpAddress('93.184.216.34'); // false
+ */
+export declare function isPrivateIpAddress(address: string): boolean;