@demigodmode/pi-web-agent 1.11.0 → 1.12.0

package/dist/fetch/destination-policy.d.ts

@@ -0,0 +1,32 @@
+import { BlockedAddressError, UnverifiedDestinationError, type NetworkGuard } from './network-guard.js';
+export type Destination = {
+    action: 'connect';
+    host: string;
+    address: string;
+} | {
+    action: 'delegate';
+    host: string;
+} | {
+    action: 'refuse';
+    error: BlockedAddressError | UnverifiedDestinationError;
+};
+export type DestinationMode = {
+    /** An upstream proxy is configured. */
+    upstream: boolean;
+    /**
+     * The user explicitly trusts the upstream proxy to enforce private-address
+     * restrictions. Only meaningful with an upstream.
+     */
+    trustProxyDns?: boolean;
+};
+/**
+ * Turns one resolution into where a connection may go (spec revision 2).
+ *
+ * - blocked answers are refused in every mode
+ * - by default the connection goes to an address we checked, never a hostname
+ *   something else will resolve
+ * - with a trusted upstream the hostname is delegated, which also covers names
+ *   only that proxy can resolve
+ * - anything else unresolvable is refused
+ */
+export declare function decideDestination(host: string, guard: NetworkGuard, mode: DestinationMode): Promise<Destination>;

package/dist/fetch/destination-policy.js

@@ -0,0 +1,24 @@
+import { BlockedAddressError, UnverifiedDestinationError } from './network-guard.js';
+/**
+ * Turns one resolution into where a connection may go (spec revision 2).
+ *
+ * - blocked answers are refused in every mode
+ * - by default the connection goes to an address we checked, never a hostname
+ *   something else will resolve
+ * - with a trusted upstream the hostname is delegated, which also covers names
+ *   only that proxy can resolve
+ * - anything else unresolvable is refused
+ */
+export async function decideDestination(host, guard, mode) {
+    const resolution = await guard.resolveHost(host);
+    if (resolution.status === 'blocked') {
+        return { action: 'refuse', error: new BlockedAddressError(resolution.host, resolution.address) };
+    }
+    if (mode.upstream && mode.trustProxyDns) {
+        return { action: 'delegate', host: resolution.host };
+    }
+    if (resolution.status === 'unresolved') {
+        return { action: 'refuse', error: new UnverifiedDestinationError(resolution.host) };
+    }
+    return { action: 'connect', host: resolution.host, address: resolution.addresses[0] };
+}

package/dist/fetch/firecrawl-fetch.js

@@ -1,3 +1,4 @@
+import { classifyHttpFailure, readResponseParts } from '../backends/provider-failure.js';
 function buildScrapeUrl(baseUrl) {
     return new URL('/v1/scrape', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`).toString();
 }
@@ -6,61 +7,79 @@ function errorMessage(error) {
 }
 export function createFirecrawlFetcher({ baseUrl, apiKey, options, fetchImpl = fetch }) {
     return async function firecrawlFetch(url) {
+        const failed = (message, failure) => ({
+            status: 'error',
+            url,
+            metadata: { method: 'firecrawl', cacheHit: false },
+            error: { code: 'FETCH_FAILED', message, failure }
+        });
+        const headers = { 'content-type': 'application/json' };
+        if (apiKey) {
+            headers.Authorization = `Bearer ${apiKey}`;
+        }
+        const body = {
+            url,
+            formats: options?.formats ?? ['markdown'],
+            ...(options?.onlyMainContent !== undefined ? { onlyMainContent: options.onlyMainContent } : {})
+        };
+        let response;
         try {
-            const headers = { 'content-type': 'application/json' };
-            if (apiKey) {
-                headers.Authorization = `Bearer ${apiKey}`;
-            }
-            const body = {
-                url,
-                formats: options?.formats ?? ['markdown'],
-                ...(options?.onlyMainContent !== undefined ? { onlyMainContent: options.onlyMainContent } : {})
-            };
-            const response = await fetchImpl(buildScrapeUrl(baseUrl), {
-                method: 'POST',
-                headers,
-                body: JSON.stringify(body)
-            });
-            if (!response.ok) {
-                throw new Error(`HTTP ${response.status}`);
-            }
-            const parsed = (await response.json());
-            if (parsed.success === false) {
-                throw new Error(typeof parsed.error === 'string' ? parsed.error : 'Firecrawl scrape failed.');
-            }
-            const text = typeof parsed.data?.markdown === 'string'
-                ? parsed.data.markdown
-                : typeof parsed.data?.html === 'string'
-                    ? parsed.data.html
-                    : '';
-            const resolvedUrl = typeof parsed.data?.metadata?.sourceURL === 'string'
-                ? parsed.data.metadata.sourceURL
-                : url;
-            const title = typeof parsed.data?.metadata?.title === 'string'
-                ? parsed.data.metadata.title
-                : undefined;
-            if (!text.trim()) {
+            response = await fetchImpl(buildScrapeUrl(baseUrl), { method: 'POST', headers, body: JSON.stringify(body) });
+        }
+        catch (error) {
+            return failed(`Firecrawl scrape failed: ${errorMessage(error)}`, { kind: 'transient' });
+        }
+        let parts;
+        try {
+            parts = await readResponseParts(response);
+        }
+        catch (error) {
+            return failed(`Firecrawl scrape response could not be read: ${errorMessage(error)}`, { kind: 'transient' });
+        }
+        if (!response.ok) {
+            const code = parts.json?.code;
+            if (response.status === 500 && code === 'SCRAPE_ALL_ENGINES_FAILED') {
+                // No extractable content (https://github.com/firecrawl/firecrawl/issues/2316): same as an empty page.
                 return {
                     status: 'needs_headless',
-                    url: resolvedUrl,
+                    url,
                     metadata: { method: 'firecrawl', cacheHit: false },
-                    error: { code: 'WEAK_EXTRACTION', message: 'Firecrawl did not return useful page text.' }
+                    error: { code: 'WEAK_EXTRACTION', message: 'Firecrawl could not extract content from this page.' }
                 };
             }
-            return {
-                status: 'ok',
-                url: resolvedUrl,
-                content: { title, text },
-                metadata: { method: 'firecrawl', cacheHit: false, truncated: text.length >= 4000 }
-            };
+            return failed(`Firecrawl scrape failed: HTTP ${response.status}`, classifyHttpFailure('firecrawl', parts));
         }
-        catch (error) {
+        const parsed = parts.json;
+        if (!parsed || typeof parsed !== 'object' || parsed.success === false) {
+            return failed('Firecrawl returned a response that did not match the expected format.', {
+                kind: 'bad_response',
+                httpStatus: response.status
+            });
+        }
+        const text = typeof parsed.data?.markdown === 'string'
+            ? parsed.data.markdown
+            : typeof parsed.data?.html === 'string'
+                ? parsed.data.html
+                : '';
+        const resolvedUrl = typeof parsed.data?.metadata?.sourceURL === 'string'
+            ? parsed.data.metadata.sourceURL
+            : url;
+        const title = typeof parsed.data?.metadata?.title === 'string'
+            ? parsed.data.metadata.title
+            : undefined;
+        if (!text.trim()) {
             return {
-                status: 'error',
-                url,
+                status: 'needs_headless',
+                url: resolvedUrl,
                 metadata: { method: 'firecrawl', cacheHit: false },
-                error: { code: 'FETCH_FAILED', message: `Firecrawl scrape failed: ${errorMessage(error)}` }
+                error: { code: 'WEAK_EXTRACTION', message: 'Firecrawl did not return useful page text.' }
             };
         }
+        return {
+            status: 'ok',
+            url: resolvedUrl,
+            content: { title, text },
+            metadata: { method: 'firecrawl', cacheHit: false, truncated: text.length >= 4000 }
+        };
     };
 }

package/dist/fetch/guard-proxy-fetch.d.ts

@@ -0,0 +1,17 @@
+import type { GuardProxy } from './guard-proxy.js';
+/**
+ * Model-chosen Node fetches connect through the guard proxy, which is where the
+ * address policy is enforced. A refusal comes back as a 403/502 on the tunnel
+ * or the forwarded request; the proxy's refusal log turns that into the typed
+ * guard error. Only refusals recorded during this call count, so an earlier
+ * block can't be blamed for a later unrelated failure.
+ */
+export type GuardProxyFetch = typeof fetch & {
+    /** Closes the ProxyAgent (awaiting it if still being created) and rejects later calls. Idempotent. */
+    close(): Promise<void>;
+};
+export declare function createGuardProxyFetch(getProxy: () => Promise<GuardProxy>, { tls }?: {
+    tls?: {
+        ca?: string | Buffer;
+    };
+}): GuardProxyFetch;

package/dist/fetch/guard-proxy-fetch.js

@@ -0,0 +1,82 @@
+import { ProxyAgent, fetch as undiciFetch } from 'undici';
+import { BLOCKED_HEADER } from './guard-proxy.js';
+function hostOf(input) {
+    const raw = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url;
+    try {
+        return new URL(raw).hostname.replace(/^\[|\]$/g, '').toLowerCase().replace(/\.+$/, '');
+    }
+    catch {
+        return undefined;
+    }
+}
+export function createGuardProxyFetch(getProxy, { tls } = {}) {
+    let ready;
+    let closed = false;
+    const ensure = () => {
+        if (ready)
+            return ready;
+        const started = getProxy().then((proxy) => {
+            const client = proxy.client('node');
+            const agent = new ProxyAgent({
+                uri: client.server,
+                token: `Basic ${Buffer.from(`${client.username}:${client.password}`).toString('base64')}`,
+                ...(tls ? { requestTls: tls } : {})
+            });
+            return { proxy, client, agent };
+        });
+        ready = started;
+        // Don't let one failed attempt poison every later fetch: clear it so the
+        // next call retries, unless a newer attempt has already replaced it.
+        started.catch(() => {
+            if (ready === started)
+                ready = undefined;
+        });
+        return started;
+    };
+    const guardedFetch = (async (input, init) => {
+        if (closed)
+            throw new Error('Guard proxy fetch is closed.');
+        const { proxy, client, agent } = await ensure();
+        const host = hostOf(input);
+        const since = proxy.sequence();
+        const refusalFor = () => proxy
+            .refusalsSince(client.username, since)
+            .reverse()
+            .find((entry) => host === undefined || entry.host === host);
+        let response;
+        try {
+            response = (await undiciFetch(input, { ...init, dispatcher: agent }));
+        }
+        catch (error) {
+            const refusal = refusalFor();
+            if (refusal)
+                throw refusal.error;
+            throw error;
+        }
+        const blockedHeader = response.headers.get(BLOCKED_HEADER);
+        if (blockedHeader) {
+            // Only the exact refusal the proxy named counts. A destination can send this
+            // header too, and a concurrent call may have been refused for the same host.
+            const seq = Number(/^\S+ (\d+)(?: |$)/.exec(blockedHeader)?.[1]);
+            const refusal = Number.isSafeInteger(seq)
+                ? proxy
+                    .refusalsSince(client.username, since)
+                    .find((entry) => entry.seq === seq && (host === undefined || entry.host === host))
+                : undefined;
+            if (refusal) {
+                await response.body?.cancel().catch(() => undefined);
+                throw refusal.error;
+            }
+        }
+        return response;
+    });
+    return Object.assign(guardedFetch, {
+        async close() {
+            closed = true;
+            if (!ready)
+                return; // never used: nothing to close, and don't start the proxy now
+            const created = await ready.catch(() => undefined);
+            await created?.agent.close().catch(() => undefined);
+        }
+    });
+}

package/dist/fetch/guard-proxy.d.ts

@@ -0,0 +1,58 @@
+import { type Socket } from 'node:net';
+import { type GuardError, type NetworkGuard } from './network-guard.js';
+/**
+ * Enforces the private-address policy where connections are opened (#53, spec
+ * revision 2). Every model-chosen connection, from Node or from Chromium, goes
+ * through here: the destination is resolved once, every answer is checked, and
+ * the socket is opened to an approved IP. Nothing downstream resolves the
+ * hostname again, so a redirect or a DNS change cannot move the connection.
+ *
+ * Listens on loopback only and requires per-start credentials, so it is not an
+ * open egress proxy for other local processes. It is an owned resource: call
+ * close() to stop it. unref() only keeps an idle listener from holding the
+ * process open.
+ */
+export declare const BLOCKED_HEADER = "x-pi-web-agent-blocked";
+export type UpstreamProxy = {
+    url: string;
+    username?: string;
+    password?: string;
+};
+export type OpenSocket = (host: string, port: number, useTls: boolean, servername?: string) => Socket;
+export type GuardProxyOptions = {
+    guard: NetworkGuard;
+    upstream?: UpstreamProxy;
+    trustProxyDns?: boolean;
+    /** TLS options for an https:// upstream proxy connection. */
+    upstreamTls?: {
+        ca?: string | Buffer;
+        rejectUnauthorized?: boolean;
+    };
+    /** Outbound sockets (direct or to the upstream) must connect within this. */
+    connectTimeoutMs?: number;
+    /** Client request/CONNECT headers and the upstream CONNECT response must arrive within this. */
+    handshakeTimeoutMs?: number;
+    /** Test seam for creating outbound sockets. */
+    openSocket?: OpenSocket;
+};
+export type GuardProxyClient = {
+    server: string;
+    username: string;
+    password: string;
+};
+export type Refusal = {
+    seq: number;
+    client: string;
+    host: string;
+    error: GuardError;
+};
+export type GuardProxy = {
+    url: string;
+    /** Credentials for one client. Refusals are recorded against its username. */
+    client(name?: string): GuardProxyClient;
+    sequence(): number;
+    refusalsSince(username: string, seq: number): Refusal[];
+    /** Idempotent. Stops the listener and destroys every socket, including ones still connecting. */
+    close(): Promise<void>;
+};
+export declare function startGuardProxy(options: GuardProxyOptions): Promise<GuardProxy>;