@demigodmode/pi-web-agent 1.11.0 → 1.13.0

package/dist/fetch/headless-fetch.js

@@ -1,13 +1,53 @@
 import { chromium } from 'playwright';
-import { extractReadableContentSafely } from '../extract/readability.js';
+import { extractReadableContentForQuery, extractReadableContentSafely } from '../extract/readability.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
 import { resolveBrowserExecutable } from './browser-resolution.js';
+import { BLOCKED_HEADER } from './guard-proxy.js';
+import { BLOCKED_PRIVATE_ADDRESS, BlockedAddressError, UPSTREAM_PROXY_REFUSED } from './network-guard.js';
 function cleanupRenderedText(text) {
     let cleaned = text.replace(/(Show more)(\s+\1){1,}/gi, '$1');
     cleaned = cleaned.replace(/(Privacy Terms)(\s+\1){1,}/gi, '$1');
     cleaned = cleaned.replace(/\s+/g, ' ').trim();
     return cleaned;
 }
-export async function headlessFetch(url, { configuredPath, proxy, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless, proxy }) => chromium.launch(executablePath ? { executablePath, headless, ...(proxy ? { proxy } : {}) } : { headless, ...(proxy ? { proxy } : {}) }), now = () => Date.now() } = {}) {
+function errorResult(url, code, message) {
+    const guard = code === BLOCKED_PRIVATE_ADDRESS || code === UPSTREAM_PROXY_REFUSED;
+    return {
+        status: 'error',
+        url,
+        metadata: { method: 'headless', cacheHit: false },
+        error: { code, message, ...(guard ? { failure: { kind: 'guard_refused' } } : {}) }
+    };
+}
+function hostnameOf(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Same normalization the guard applies, so refusal hosts and navigation hosts compare equal. */
+function normalizeHost(host) {
+    return host.replace(/^\[|\]$/g, '').toLowerCase().replace(/\.+$/, '');
+}
+export async function headlessFetch(url, { configuredPath, query, proxy, guard, guardProxy, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless, proxy }) => chromium.launch(executablePath ? { executablePath, headless, ...(proxy ? { proxy } : {}) } : { headless, ...(proxy ? { proxy } : {}) }), now = () => Date.now() } = {}) {
+    if (guard) {
+        const hostname = hostnameOf(url);
+        if (!guardProxy) {
+            // Enforcement lives in the guard proxy. Without it, loading the page would be unguarded.
+            return errorResult(url, BLOCKED_PRIVATE_ADDRESS, `Blocked ${hostname ?? url}: the browser could not enforce the private address guard.`);
+        }
+        if (hostname) {
+            // Early, clearer refusal for an obviously blocked main url. The guard
+            // proxy is what actually enforces the policy for every connection.
+            const verdict = await guard.checkHost(hostname);
+            if (!verdict.allowed) {
+                const error = new BlockedAddressError(verdict.host, verdict.address);
+                return errorResult(url, error.code, error.message);
+            }
+        }
+    }
     const resolved = await resolveBrowser({ configuredPath });
     if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
         return {
@@ -17,29 +57,159 @@ export async function headlessFetch(url, { configuredPath, proxy, resolveBrowser
             error: resolved.error
         };
     }
+    let enforcement;
+    if (guard && guardProxy) {
+        let activeProxy;
+        try {
+            activeProxy = await guardProxy();
+        }
+        catch {
+            // The guard proxy could not start (backend set closed, or the listener
+            // failed). Enforcement lives there, so the browser must not launch.
+            return errorResult(url, BLOCKED_PRIVATE_ADDRESS, `Blocked ${hostnameOf(url) ?? url}: the private address guard is not available, so the browser was not started.`);
+        }
+        const client = activeProxy.client('headless');
+        enforcement = {
+            proxy: activeProxy,
+            username: client.username,
+            since: activeProxy.sequence(),
+            // `<-loopback>` removes Chromium's implicit loopback bypass, so localhost
+            // and link-local connections go through the guard proxy too.
+            launchProxy: { server: client.server, username: client.username, password: client.password, bypass: '<-loopback>' }
+        };
+    }
+    const effectiveProxy = enforcement ? enforcement.launchProxy : proxy;
     const browserName = resolved.ok ? resolved.browser : 'chromium';
     const launchOptions = resolved.ok
-        ? { executablePath: resolved.executablePath, headless: true, ...(proxy ? { proxy } : {}) }
-        : { headless: true, ...(proxy ? { proxy } : {}) };
+        ? { executablePath: resolved.executablePath, headless: true, ...(effectiveProxy ? { proxy: effectiveProxy } : {}) }
+        : { headless: true, ...(effectiveProxy ? { proxy: effectiveProxy } : {}) };
+    // Browser requests that failed or were refused by the proxy, from passive
+    // page events only. Navigation hosts decide whether a refusal caused the
+    // navigation error; subresource failures are counted one per request.
+    const failedNavigationHosts = new Set();
+    const failedSubresourceHosts = [];
+    const seenRequests = new WeakSet();
+    const refusals = () => (enforcement ? enforcement.proxy.refusalsSince(enforcement.username, enforcement.since) : []);
+    const navigationRefusal = () => refusals().find((entry) => failedNavigationHosts.has(entry.host));
+    const subresourceRefusals = () => {
+        const refusedHosts = new Set(refusals().map((entry) => entry.host));
+        return failedSubresourceHosts.filter((host) => refusedHosts.has(host)).length;
+    };
     let browser;
     let context;
     let page;
     try {
         browser = await launchBrowser(launchOptions);
-        context = await browser.newContext();
-        page = await context.newPage();
+        // Service workers can fetch on a page's behalf; blocking them keeps the page's traffic simple to account for.
+        context = await browser.newContext(enforcement ? { serviceWorkers: 'block' } : undefined);
+        if (enforcement) {
+            // Watch every page in the context: popups opened by the page can hit blocked
+            // hosts too. Only the primary page's main-frame navigation is "the navigation".
+            let primaryPage;
+            const recordRequest = (request) => {
+                try {
+                    if (!request || seenRequests.has(request))
+                        return;
+                    seenRequests.add(request);
+                    const host = normalizeHost(new URL(request.url()).hostname);
+                    if (primaryPage && request.isNavigationRequest() && request.frame() === primaryPage.mainFrame()) {
+                        failedNavigationHosts.add(host);
+                    }
+                    else {
+                        failedSubresourceHosts.push(host);
+                    }
+                }
+                catch {
+                    // Unparseable URL or a detached frame: nothing to attribute.
+                }
+            };
+            const watchPage = (watched) => {
+                watched?.on?.('requestfailed', recordRequest);
+                watched?.on?.('response', (response) => {
+                    try {
+                        if (response.headers()[BLOCKED_HEADER])
+                            recordRequest(response.request());
+                    }
+                    catch {
+                        // ignore
+                    }
+                });
+                // Best effort: Playwright doesn't say why a WebSocket died, so an error or a
+                // close before any frame counts as failed. Only refused hosts get counted.
+                watched?.on?.('websocket', (ws) => {
+                    let host;
+                    try {
+                        host = normalizeHost(new URL(ws.url()).hostname);
+                    }
+                    catch {
+                        return;
+                    }
+                    let framed = false;
+                    let recorded = false;
+                    const fail = () => {
+                        if (recorded)
+                            return;
+                        recorded = true;
+                        failedSubresourceHosts.push(host);
+                    };
+                    ws.on?.('framereceived', () => (framed = true));
+                    ws.on?.('framesent', () => (framed = true));
+                    ws.on?.('socketerror', fail);
+                    ws.on?.('close', () => {
+                        if (!framed)
+                            fail();
+                    });
+                });
+            };
+            const watchedPages = new WeakSet();
+            const watchOnce = (candidate) => {
+                if (!candidate || watchedPages.has(candidate))
+                    return;
+                watchedPages.add(candidate);
+                watchPage(candidate);
+            };
+            // Registered before newPage(), so the primary page's own 'page' event is covered too.
+            context.on?.('page', watchOnce);
+            page = await context.newPage();
+            primaryPage = page;
+            watchOnce(page);
+        }
+        else {
+            page = await context.newPage();
+        }
         const startedAt = now();
-        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+        const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+        if (enforcement && response?.headers?.()[BLOCKED_HEADER]) {
+            // Normally already recorded by the 'response' event; this covers it if not.
+            try {
+                const request = response.request?.();
+                if (request && !seenRequests.has(request)) {
+                    seenRequests.add(request);
+                    failedNavigationHosts.add(normalizeHost(new URL(request.url()).hostname));
+                }
+            }
+            catch {
+                // ignore
+            }
+            const cause = navigationRefusal();
+            if (cause)
+                return errorResult(url, cause.error.code, cause.error.message);
+        }
         await page.waitForLoadState('load', { timeout: 10000 });
         await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
         const html = await page.content();
         const finishedAt = now();
-        const extraction = extractReadableContentSafely(html);
+        const blockedSubresources = subresourceRefusals();
+        const baselineExtraction = extractReadableContentSafely(html);
+        const queryExtraction = query ? extractReadableContentForQuery(html, query) : undefined;
+        const extraction = queryExtraction ?? baselineExtraction;
+        const cleanedBaselineText = cleanupRenderedText(baselineExtraction.content.text);
         const cleanedContent = {
             ...extraction.content,
-            text: cleanupRenderedText(extraction.content.text)
+            text: cleanupRenderedText(extraction.content.text),
+            ...(hasBotCheckContent(html, 'html') ? { botCheck: true } : {})
         };
-        if (!cleanedContent.text || cleanedContent.text.length < 40) {
+        if (!cleanedBaselineText || cleanedBaselineText.length < 40) {
             return {
                 status: 'blocked',
                 url,
@@ -47,7 +217,8 @@ export async function headlessFetch(url, { configuredPath, proxy, resolveBrowser
                     method: 'headless',
                     cacheHit: false,
                     browser: browserName,
-                    navigationMs: finishedAt - startedAt
+                    navigationMs: finishedAt - startedAt,
+                    ...(blockedSubresources > 0 ? { blockedSubresources } : {})
                 },
                 error: {
                     code: 'HEADLESS_EXTRACTION_WEAK',
@@ -64,18 +235,24 @@ export async function headlessFetch(url, { configuredPath, proxy, resolveBrowser
                 cacheHit: false,
                 browser: browserName,
                 navigationMs: finishedAt - startedAt,
-                truncated: cleanedContent.text.length >= 4000
+                truncated: queryExtraction?.omitted ?? cleanedContent.text.length >= 4000,
+                ...(blockedSubresources > 0 ? { blockedSubresources } : {})
             }
         };
     }
     catch (error) {
+        const cause = navigationRefusal();
+        if (cause)
+            return errorResult(url, cause.error.code, cause.error.message);
+        const blockedSubresources = subresourceRefusals();
         return {
             status: 'error',
             url,
             metadata: {
                 method: 'headless',
                 cacheHit: false,
-                browser: browserName
+                browser: browserName,
+                ...(blockedSubresources > 0 ? { blockedSubresources } : {})
             },
             error: {
                 code: 'HEADLESS_NAVIGATION_FAILED',

package/dist/fetch/http-fetch.d.ts

@@ -1,4 +1,4 @@
 import type { WebFetchResponse } from '../types.js';
 export declare function createHttpFetcher({ fetchImpl }?: {
     fetchImpl?: typeof fetch;
-}): (url: string) => Promise<WebFetchResponse>;
+}): (url: string, query?: string) => Promise<WebFetchResponse>;

package/dist/fetch/http-fetch.js

@@ -1,4 +1,6 @@
-import { extractReadableContentSafely } from '../extract/readability.js';
+import { extractReadableContentForQuery, extractReadableContentSafely } from '../extract/readability.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
+import { findGuardError } from './network-guard.js';
 function looksLikeScriptShell(html) {
     const lower = html.toLowerCase();
     return lower.includes('<script') && (lower.includes('id="app"') || lower.includes('id="root"'));
@@ -14,8 +16,22 @@ function isWeakHttpContent(options) {
     return veryShortBody && (lowDensity || hasGenericShellMarker);
 }
 export function createHttpFetcher({ fetchImpl = fetch } = {}) {
-    return async function httpFetch(url) {
-        const response = await fetchImpl(url);
+    return async function httpFetch(url, query) {
+        let response;
+        try {
+            response = await fetchImpl(url);
+        }
+        catch (error) {
+            const blocked = findGuardError(error);
+            if (!blocked)
+                throw error;
+            return {
+                status: 'error',
+                url,
+                metadata: { method: 'http', cacheHit: false },
+                error: { code: blocked.code, message: blocked.message, failure: { kind: 'guard_refused' } }
+            };
+        }
         const contentType = response.headers.get('content-type') ?? '';
         if (!contentType.includes('text/html')) {
             return {
@@ -25,11 +41,16 @@ export function createHttpFetcher({ fetchImpl = fetch } = {}) {
             };
         }
         const html = await response.text();
-        const extraction = extractReadableContentSafely(html);
-        const content = extraction.content;
+        const baselineExtraction = extractReadableContentSafely(html);
+        const queryExtraction = query ? extractReadableContentForQuery(html, query) : undefined;
+        const extraction = queryExtraction ?? baselineExtraction;
+        const content = {
+            ...extraction.content,
+            ...(hasBotCheckContent(html, 'html') ? { botCheck: true } : {})
+        };
         if (looksLikeScriptShell(html) ||
-            content.text.length < 40 ||
-            isWeakHttpContent({ html, title: content.title, text: content.text })) {
+            baselineExtraction.content.text.length < 40 ||
+            isWeakHttpContent({ html, title: baselineExtraction.content.title, text: baselineExtraction.content.text })) {
             return {
                 status: 'needs_headless',
                 url: response.url,
@@ -44,7 +65,7 @@ export function createHttpFetcher({ fetchImpl = fetch } = {}) {
             status: 'ok',
             url: response.url,
             content,
-            metadata: { method: 'http', cacheHit: false, contentType, truncated: content.text.length >= 4000 }
+            metadata: { method: 'http', cacheHit: false, contentType, truncated: queryExtraction?.omitted ?? content.text.length >= 4000 }
         };
     };
 }

package/dist/fetch/network-guard.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Decides whether a destination the *model* chose is safe to fetch (#53).
+ *
+ * web_explore fetches URLs the model picks, partly from pages it just read, so
+ * a hostile page can point it at cloud metadata (169.254.169.254), services on
+ * localhost, or the user's LAN. This module only answers "is this address
+ * allowed"; guarded-fetch.ts and headless-fetch.ts enforce it. User-configured
+ * endpoints (search APIs, SearXNG, Firecrawl, the proxy) never go through it.
+ */
+export type Cidr = {
+    family: 4 | 6;
+    network: bigint;
+    prefix: number;
+};
+export type LookupFn = (host: string) => Promise<Array<{
+    address: string;
+    family: number;
+}>>;
+export type NetworkGuardConfig = {
+    allowRanges?: string[];
+};
+export type GuardVerdict = {
+    allowed: true;
+    unresolved?: true;
+} | {
+    allowed: false;
+    host: string;
+    address: string;
+};
+export type HostResolution = {
+    status: 'allowed';
+    host: string;
+    addresses: string[];
+} | {
+    status: 'blocked';
+    host: string;
+    address: string;
+} | {
+    status: 'unresolved';
+    host: string;
+};
+export type NetworkGuard = {
+    isBlockedAddress(address: string): boolean;
+    /** Resolves once and checks every answer. The guard proxy connects to one of `addresses`, never re-resolving. */
+    resolveHost(host: string): Promise<HostResolution>;
+    checkHost(host: string): Promise<GuardVerdict>;
+    assertUrlAllowed(url: string): Promise<void>;
+};
+export declare const BLOCKED_PRIVATE_ADDRESS = "BLOCKED_PRIVATE_ADDRESS";
+export declare class BlockedAddressError extends Error {
+    readonly host: string;
+    readonly address: string;
+    readonly code = "BLOCKED_PRIVATE_ADDRESS";
+    constructor(host: string, address: string);
+}
+/** No address could be verified, so we refuse rather than let something else resolve it. */
+export declare class UnverifiedDestinationError extends Error {
+    readonly host: string;
+    readonly code = "BLOCKED_PRIVATE_ADDRESS";
+    constructor(host: string);
+}
+export declare const UPSTREAM_PROXY_REFUSED = "UPSTREAM_PROXY_REFUSED";
+/** The user's upstream proxy would not accept the approved IP. We never retry by hostname. */
+export declare class UpstreamProxyRefusedError extends Error {
+    readonly host: string;
+    readonly target: string;
+    readonly status: number | string;
+    readonly code = "UPSTREAM_PROXY_REFUSED";
+    constructor(host: string, target: string, status: number | string);
+}
+export type GuardError = BlockedAddressError | UnverifiedDestinationError | UpstreamProxyRefusedError;
+export declare function findGuardError(error: unknown): GuardError | undefined;
+/** undici wraps connect errors as `TypeError: fetch failed` with the real error in `cause`. */
+export declare function findBlockedAddressError(error: unknown): BlockedAddressError | undefined;
+export declare function parseCidr(text: string): Cidr | undefined;
+/** Parses an allow list, dropping invalid entries and anything that allows every address. */
+export declare function usableAllowRanges(allowRanges?: string[]): Cidr[];
+export declare function createNetworkGuard(config?: NetworkGuardConfig, { lookup, lookupTimeoutMs }?: {
+    lookup?: LookupFn;
+    /** A lookup that hasn't answered by then counts as unresolved, so no caller waits on DNS forever. */
+    lookupTimeoutMs?: number;
+}): NetworkGuard;