@demigodmode/pi-web-agent 1.11.0 → 1.12.0

package/CHANGELOG.md

@@ -18,6 +18,20 @@ The format is intentionally simple and release-oriented.
 ### Breaking
 - None.
 
+## [1.12.0] - 2026-09-17
+### Added
+- A network allow list, under Settings → Backends → Network allow list. web_explore now refuses private and local addresses when the link came from the model or a page it read, so if you genuinely want it to read something on your own network (an internal docs site, a service on localhost), add that range here as a CIDR like `10.0.0.0/24`. Entries are checked when you save, and ones that would allow everything (`0.0.0.0/0`, `::/0`) are rejected, since that would quietly turn the protection off. It is also the fix if every fetch suddenly fails: some proxy apps run in fake-IP mode and make every site look like it lives in `198.18.0.0/15`, and adding that range gets you going again. (#53)
+- A "Trust the upstream proxy to enforce private-address restrictions" setting (`backends.network.trustProxyDns`). With a proxy configured, pi-web-agent normally looks each address up itself and asks the proxy to connect to that exact IP, so the proxy can't send a request somewhere else. Some proxies only accept hostnames, and some networks only resolve names inside the proxy. Turning this on hands the hostname to the proxy instead. It is off by default because it moves the address decision to your proxy, and even when it is on, localhost, private IPs typed straight into a link, and anything your own machine resolves to a private address are still refused. `/web-agent doctor` shows the allow list and this setting. (#53)
+
+### Changed
+- Search and fetch fallback now act on why a backend failed. Rate limits pause that backend for the time the provider asks (capped at 15 minutes), exhausted quotas and bad API keys skip it until settings change, and timeouts or server errors get one retry. An empty result is treated as a real answer instead of a failure, a private-address refusal or an invalid shared proxy setting never falls back to another backend, and answers say when some search backends were unavailable. DuckDuckGo no longer retries on its own. (#55)
+
+### Fixed
+- web_explore no longer fetches private, loopback, or link-local addresses when the link came from the model or a page it read. The check happens where connections are opened, through a local guard proxy, so it also covers redirects, DNS changes, and everything a headless page loads. A web page could previously steer it at cloud metadata endpoints or services on your network. Addresses you configured yourself (search backends, SearXNG, Firecrawl, the proxy) are unaffected. If you do need to reach a private range, add it under Settings → Backends → Network allow list. With an upstream proxy that only accepts hostnames, turn on "Trust the upstream proxy to enforce private-address restrictions". (#53)
+
+### Breaking
+- None.
+
 ## [1.11.0] - 2026-09-16
 ### Added
 - Optional proxy support. Point web_explore at an HTTP/HTTPS proxy and everything outbound goes through it: search, fetch, Firecrawl, the GitHub/PDF/YouTube readers, doctor health checks, and the headless browser. Off unless you set it, and nothing changes if you don't. Set it in Settings → Backends, and keep credentials in `PI_WEB_AGENT_PROXY_USERNAME` / `PI_WEB_AGENT_PROXY_PASSWORD` rather than in the URL. Thanks to @lo-tp for building this. (#50)

package/dist/backends/config.d.ts

@@ -34,17 +34,25 @@ export type FetchBackendConfig = {
 export type HeadlessBackendConfig = {
     provider: 'local-browser';
 };
+export type NetworkConfig = {
+    /** CIDR ranges exempted from the private-address guard (#53). */
+    allowRanges?: string[];
+    /** Trust the upstream proxy to enforce private-address restrictions. */
+    trustProxyDns?: boolean;
+};
 export type BackendConfig = {
     search: SearchBackendConfig;
     fetch: FetchBackendConfig;
     headless: HeadlessBackendConfig;
     proxy?: ProxyConfig;
+    network?: NetworkConfig;
 };
 export type BackendConfigOverride = {
     search?: Partial<SearchBackendConfig>;
     fetch?: Partial<FetchBackendConfig>;
     headless?: Partial<HeadlessBackendConfig>;
     proxy?: ProxyConfig;
+    network?: NetworkConfig;
 };
 export type BackendConfigFile = {
     backends?: {
@@ -70,6 +78,10 @@ export type BackendConfigFile = {
             username?: unknown;
             password?: unknown;
         };
+        network?: {
+            allowRanges?: unknown;
+            trustProxyDns?: unknown;
+        };
     };
 };
 export declare const DEFAULT_BACKEND_CONFIG: BackendConfig;
@@ -88,6 +100,7 @@ export declare function stripProxyCredentials(url: string): string;
  */
 export declare function isValidProxyUrl(url: string): boolean;
 export declare function extractProxyConfig(value: unknown): ProxyConfig | undefined;
+export declare function extractNetworkConfig(value: unknown): NetworkConfig | undefined;
 export declare function usableSearchProviders(search: SearchBackendConfig, env?: NodeJS.ProcessEnv): SearchProviderName[];
 export declare function extractBackendConfigOverride(file: BackendConfigFile | null | undefined): BackendConfigOverride;
 export declare function validateBackendConfig(config: BackendConfig): string[];

package/dist/backends/config.js

@@ -1,3 +1,4 @@
+import { parseCidr } from '../fetch/network-guard.js';
 export const DEFAULT_BACKEND_CONFIG = {
     search: { provider: 'duckduckgo' },
     fetch: { provider: 'http' },
@@ -79,6 +80,19 @@ export function extractProxyConfig(value) {
         config.password = raw.password;
     return config;
 }
+export function extractNetworkConfig(value) {
+    if (!value || typeof value !== 'object')
+        return undefined;
+    const raw = value;
+    const config = {};
+    if (Array.isArray(raw.allowRanges)) {
+        // Coerce non-string entries so validation flags them instead of dropping the list.
+        config.allowRanges = raw.allowRanges.map((entry) => (typeof entry === 'string' ? entry : String(entry)));
+    }
+    if (typeof raw.trustProxyDns === 'boolean')
+        config.trustProxyDns = raw.trustProxyDns;
+    return Object.keys(config).length > 0 ? config : undefined;
+}
 function extractSearxngOptions(value) {
     if (!value || typeof value !== 'object')
         return undefined;
@@ -187,6 +201,10 @@ export function extractBackendConfigOverride(file) {
     if (proxy) {
         override.proxy = proxy;
     }
+    const network = extractNetworkConfig(backends?.network);
+    if (network) {
+        override.network = network;
+    }
     return override;
 }
 export function validateBackendConfig(config) {
@@ -214,6 +232,18 @@ export function validateBackendConfig(config) {
     if (config.fetch.provider === 'firecrawl' && !config.fetch.baseUrl) {
         issues.push('fetch provider firecrawl requires backends.fetch.baseUrl');
     }
+    for (const range of config.network?.allowRanges ?? []) {
+        const cidr = parseCidr(range);
+        if (!cidr) {
+            issues.push(`backends.network.allowRanges entry "${range}" is not a valid CIDR range`);
+        }
+        else if (cidr.prefix === 0) {
+            issues.push(`backends.network.allowRanges entry "${range}" allows every address, which turns the guard off; list specific ranges instead`);
+        }
+    }
+    if (config.network?.trustProxyDns && !config.proxy?.url?.trim()) {
+        issues.push('backends.network.trustProxyDns has no effect without backends.proxy');
+    }
     if (config.search.fallback === 'duckduckgo' && config.search.provider !== 'searxng' && config.search.provider !== 'brave' && config.search.provider !== 'youcom' && config.search.provider !== 'exa' && config.search.provider !== 'tavily') {
         issues.push('search fallback duckduckgo is only supported when search provider is searxng, brave, youcom, exa, or tavily');
     }
@@ -260,6 +290,17 @@ function mergeFetchConfig(current, override) {
     }
     return { ...current, ...override };
 }
+function mergeNetworkConfig(base, layer) {
+    if (!layer)
+        return base;
+    const next = { ...base };
+    // A layer's allow list replaces the lower one outright; trust is set independently.
+    if (layer.allowRanges)
+        next.allowRanges = [...layer.allowRanges];
+    if (layer.trustProxyDns !== undefined)
+        next.trustProxyDns = layer.trustProxyDns;
+    return next;
+}
 export function mergeBackendConfigLayers(...layers) {
     return layers.reduce((merged, layer) => ({
         search: mergeSearchConfig(merged.search, layer?.search),
@@ -269,6 +310,8 @@ export function mergeBackendConfigLayers(...layers) {
             ? layer.proxy.url === ''
                 ? undefined // explicit disable overrides any proxy from lower layers
                 : { ...merged.proxy, ...layer.proxy }
-            : merged.proxy
+            : merged.proxy,
+        // Replace, don't union: a project list is the whole list for that project.
+        network: mergeNetworkConfig(merged.network, layer?.network)
     }), DEFAULT_BACKEND_CONFIG);
 }

package/dist/backends/factory.d.ts

@@ -1,9 +1,13 @@
 import { createFirecrawlFetcher } from '../fetch/firecrawl-fetch.js';
+import { type GuardProxy, type GuardProxyOptions } from '../fetch/guard-proxy.js';
+import { type NetworkGuard } from '../fetch/network-guard.js';
 import { createBraveSearchTool } from '../search/brave.js';
 import { createYouComSearchTool } from '../search/youcom.js';
 import { createExaSearchTool } from '../search/exa.js';
 import { createTavilySearchTool } from '../search/tavily.js';
 import { createSearxngSearchTool } from '../search/searxng.js';
+import { type PolicyDeps } from './fallback-policy.js';
+import { type ProviderHealth } from './provider-health.js';
 import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
 import { createWebFetchTool } from '../tools/web-fetch.js';
 import { createWebSearchTool } from '../tools/web-search.js';
@@ -19,6 +23,8 @@ export type BackendSet = {
     headlessFetch: (input: {
         url: string;
     }) => Promise<WebFetchHeadlessResponse>;
+    /** Releases the guard proxy and its agents. Idempotent; never starts the proxy. */
+    close: () => Promise<void>;
 };
 export type BackendFactoryDeps = {
     createDuckDuckGoSearch?: typeof createWebSearchTool;
@@ -31,5 +37,14 @@ export type BackendFactoryDeps = {
     createFirecrawlFetch?: typeof createFirecrawlFetcher;
     createHeadlessFetch?: typeof createWebFetchHeadlessTool;
     createProxyFetch?: (proxy: ProxyConfig) => typeof fetch;
+    networkGuard?: NetworkGuard;
+    /** Test seam: the fetch used for model-chosen URLs, before redirect handling. */
+    createModelFetch?: (guard: NetworkGuard) => typeof fetch;
+    createGuardProxy?: (options: GuardProxyOptions) => Promise<GuardProxy>;
+    providerHealth?: ProviderHealth;
+    /** Test seam for the retry sleep, jitter, and clock. */
+    policy?: Omit<PolicyDeps, 'health'>;
+    /** Test seam for the per-call fanout provider timeout. */
+    fanoutTimeoutMs?: number;
 };
 export declare function createBackendSet(config?: BackendConfig, deps?: BackendFactoryDeps): BackendSet;

package/dist/backends/factory.js

@@ -2,13 +2,19 @@ import { createFirecrawlFetcher } from '../fetch/firecrawl-fetch.js';
 import { createHttpFetcher } from '../fetch/http-fetch.js';
 import { createProxyFetch, resolveProxyCredentials } from '../fetch/proxy-fetch.js';
 import { headlessFetch } from '../fetch/headless-fetch.js';
+import { createGuardedFetch } from '../fetch/guarded-fetch.js';
+import { startGuardProxy } from '../fetch/guard-proxy.js';
+import { createGuardProxyFetch } from '../fetch/guard-proxy-fetch.js';
+import { createNetworkGuard, findGuardError } from '../fetch/network-guard.js';
 import { createBraveSearchTool } from '../search/brave.js';
 import { createYouComSearchTool } from '../search/youcom.js';
 import { fetchDuckDuckGoHtml } from '../search/duckduckgo.js';
 import { createExaSearchTool } from '../search/exa.js';
 import { createTavilySearchTool } from '../search/tavily.js';
 import { createSearxngSearchTool } from '../search/searxng.js';
-import { createFanoutSearch } from '../search/fanout.js';
+import { createFanoutSearch, FANOUT_PROVIDER_TIMEOUT_MS, withCallTimeout } from '../search/fanout.js';
+import { chainSearch, withFetchPolicy, withSearchPolicy } from './fallback-policy.js';
+import { createProviderHealth } from './provider-health.js';
 import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
 import { buildSearchPresentation } from '../presentation/search-presentation.js';
 import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
@@ -27,7 +33,8 @@ function invalidSearxngSearch() {
             metadata: { backend: 'searxng', cacheHit: false },
             error: {
                 code: 'BACKEND_CONFIG_INVALID',
-                message: 'SearXNG search requires backends.search.baseUrl.'
+                message: 'SearXNG search requires backends.search.baseUrl.',
+                failure: { kind: 'not_configured' }
             }
         };
         return { ...result, presentation: buildSearchPresentation(result) };
@@ -41,47 +48,36 @@ function invalidFirecrawlFetch() {
             metadata: { method: 'firecrawl', cacheHit: false },
             error: {
                 code: 'BACKEND_CONFIG_INVALID',
-                message: 'Firecrawl fetch requires backends.fetch.baseUrl.'
+                message: 'Firecrawl fetch requires backends.fetch.baseUrl.',
+                failure: { kind: 'not_configured' }
             }
         };
         return { ...result, presentation: buildFetchPresentation(result) };
     };
 }
-function withSearchFallback(primary, fallback, fallbackFrom) {
+/**
+ * Checks the target URL before dispatching to the http fetcher, a content
+ * reader, or Firecrawl. Doing it here means a private URL is refused once, in
+ * one place, and never reaches Firecrawl or its http fallback (#53).
+ */
+function withTargetGuard(fetchPage, guard, method) {
     return async (input) => {
-        const first = await primary(input);
-        if (first.status !== 'error')
-            return first;
-        const second = await fallback(input);
-        const result = {
-            ...second,
-            metadata: {
-                ...second.metadata,
-                fallbackFrom,
-                fallbackReason: first.error?.message ?? `${fallbackFrom} search failed.`,
-                // Keep the primary's fanout provenance (which providers were tried/skipped) even though
-                // the answer came from the fallback backend.
-                ...(first.metadata.fanout ? { fanout: first.metadata.fanout } : {})
-            }
-        };
-        return { ...result, presentation: buildSearchPresentation(result) };
-    };
-}
-function withFetchFallback(primary, fallback) {
-    return async (input) => {
-        const first = await primary(input);
-        if (first.status !== 'error' && first.status !== 'needs_headless')
-            return first;
-        const second = await fallback(input);
-        const result = {
-            ...second,
-            metadata: {
-                ...second.metadata,
-                fallbackFrom: 'firecrawl',
-                fallbackReason: first.error?.message ?? 'Firecrawl fetch failed.'
-            }
-        };
-        return { ...result, presentation: buildFetchPresentation(result) };
+        try {
+            await guard.assertUrlAllowed(input.url);
+        }
+        catch (error) {
+            const blocked = findGuardError(error);
+            if (!blocked)
+                throw error;
+            const result = {
+                status: 'error',
+                url: input.url,
+                metadata: { method, cacheHit: false },
+                error: { code: blocked.code, message: blocked.message, failure: { kind: 'guard_refused' } }
+            };
+            return { ...result, presentation: buildFetchPresentation(result) };
+        }
+        return fetchPage(input);
     };
 }
 export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
@@ -109,7 +105,7 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
                     status: 'error',
                     results: [],
                     metadata: { backend: config.search.provider, cacheHit: false },
-                    error: { code: 'BACKEND_CONFIG_INVALID', message }
+                    error: { code: 'BACKEND_CONFIG_INVALID', message, failure: { kind: 'config_global' } }
                 };
                 return { ...result, presentation: buildSearchPresentation(result) };
             },
@@ -118,7 +114,7 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
                     status: 'error',
                     url,
                     metadata: { method: 'http', cacheHit: false },
-                    error: { code: 'BACKEND_CONFIG_INVALID', message }
+                    error: { code: 'BACKEND_CONFIG_INVALID', message, failure: { kind: 'config_global' } }
                 };
                 return { ...result, presentation: buildFetchPresentation(result) };
             },
@@ -127,24 +123,71 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
                     status: 'error',
                     url,
                     metadata: { method: 'headless', cacheHit: false },
-                    error: { code: 'BACKEND_CONFIG_INVALID', message }
+                    error: { code: 'BACKEND_CONFIG_INVALID', message, failure: { kind: 'config_global' } }
                 };
                 return { ...result, presentation: buildFetchPresentation(result) };
-            }
+            },
+            close: async () => undefined
         };
     }
-    // When a proxy is configured, every outbound HTTP request (search, fetch,
-    // readers, and doctor-style checks) goes through it; headless browser
-    // traffic gets the same proxy via Playwright launch options.
+    // When a proxy is configured, every outbound HTTP request goes through it.
+    // User-configured endpoints use fetchImpl directly; model-chosen fetches and
+    // the headless browser reach it through the guard proxy below.
     const fetchImpl = proxy ? makeProxyFetch(proxy) : fetch;
     const proxyCredentials = proxy ? resolveProxyCredentials(proxy) : undefined;
-    const proxyBrowserOptions = proxy
-        ? {
-            server: stripProxyCredentials(proxy.url),
-            ...(proxyCredentials?.username !== undefined ? { username: proxyCredentials.username } : {}),
-            ...(proxyCredentials?.password !== undefined ? { password: proxyCredentials.password } : {})
+    // Model-chosen URLs only. Search APIs and the configured SearXNG/Firecrawl
+    // endpoints keep using fetchImpl: the user typed those (#53).
+    const networkGuard = deps.networkGuard ?? createNetworkGuard({ allowRanges: config.network?.allowRanges ?? [] });
+    // One guard proxy per backend set, started on first use. It is the single
+    // place the address policy is enforced, for Node fetches and the browser,
+    // and it chains to the user's upstream proxy itself.
+    let guardProxy;
+    let isClosed = false;
+    const getGuardProxy = () => {
+        if (isClosed)
+            return Promise.reject(new Error('Backend set is closed.'));
+        if (guardProxy)
+            return guardProxy;
+        const started = (deps.createGuardProxy ?? startGuardProxy)({
+            guard: networkGuard,
+            ...(proxy
+                ? {
+                    upstream: {
+                        url: stripProxyCredentials(proxy.url),
+                        ...(proxyCredentials?.username !== undefined ? { username: proxyCredentials.username } : {}),
+                        ...(proxyCredentials?.password !== undefined ? { password: proxyCredentials.password } : {})
+                    }
+                }
+                : {}),
+            trustProxyDns: config.network?.trustProxyDns === true
+        });
+        guardProxy = started;
+        // A failed start should not stick around for the rest of the backend
+        // set's life: clear it so the next call tries again, unless a newer
+        // attempt has already replaced it.
+        started.catch(() => {
+            if (guardProxy === started)
+                guardProxy = undefined;
+        });
+        return started;
+    };
+    const modelFetch = deps.createModelFetch
+        ? deps.createModelFetch(networkGuard)
+        : createGuardProxyFetch(getGuardProxy);
+    const targetFetch = createGuardedFetch(modelFetch, networkGuard);
+    let closing;
+    const close = () => (closing ??= (async () => {
+        isClosed = true;
+        if ('close' in modelFetch)
+            await modelFetch.close();
+        if (guardProxy) {
+            const started = await guardProxy.catch(() => undefined);
+            await started?.close().catch(() => undefined);
         }
-        : undefined;
+    })());
+    // One health state per backend set: a rebuilt set (config change) starts fresh.
+    const policyDeps = { health: deps.providerHealth ?? createProviderHealth(deps.policy?.now ? { now: deps.policy.now } : {}), ...deps.policy };
+    const guarded = (name, search, healthKey) => withSearchPolicy(name, search, policyDeps, healthKey);
     const createDuckDuckGo = () => createDuckDuckGoSearch({ searchHtml: (query) => fetchDuckDuckGoHtml(query, { fetchImpl }) });
     function buildProviderSearch(name) {
         switch (name) {
@@ -165,34 +208,12 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
                 return createDuckDuckGo();
         }
     }
-    let search = config.search.provider === 'searxng'
-        ? config.search.baseUrl
-            ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options, fetchImpl })
-            : invalidSearxngSearch()
-        : config.search.provider === 'brave'
-            ? createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY, fetchImpl })
-            : config.search.provider === 'youcom'
-                ? createYouComSearch({ apiKey: process.env.YDC_API_KEY, fetchImpl })
-                : config.search.provider === 'exa'
-                    ? createExaSearch({ apiKey: process.env.EXA_API_KEY, fetchImpl })
-                    : config.search.provider === 'tavily'
-                        ? createTavilySearch({ apiKey: process.env.TAVILY_API_KEY, fetchImpl })
-                        : createDuckDuckGo();
-    if (config.search.provider === 'searxng' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGo(), 'searxng');
-    }
-    if (config.search.provider === 'brave' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGo(), 'brave');
-    }
-    if (config.search.provider === 'youcom' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGo(), 'youcom');
-    }
-    if (config.search.provider === 'exa' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGo(), 'exa');
-    }
-    if (config.search.provider === 'tavily' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGo(), 'tavily');
+    const primarySearch = guarded(config.search.provider, buildProviderSearch(config.search.provider));
+    const chain = [primarySearch];
+    if (config.search.provider !== 'duckduckgo' && config.search.fallback === 'duckduckgo') {
+        chain.push(guarded('duckduckgo', createDuckDuckGo()));
     }
+    let search = chainSearch(chain, policyDeps);
     const fanoutConfig = config.search.fanout;
     if (fanoutConfig && fanoutConfig.mode !== 'off') {
         const baseNames = fanoutConfig.providers && fanoutConfig.providers.length > 0
@@ -203,9 +224,12 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
             ? [...baseNames, 'duckduckgo']
             : baseNames;
         const ordered = [config.search.provider, ...providerNames.filter((n) => n !== config.search.provider)].filter((n, i, arr) => arr.indexOf(n) === i);
+        const timeoutMs = deps.fanoutTimeoutMs ?? FANOUT_PROVIDER_TIMEOUT_MS;
         search = createFanoutSearch({
-            providers: ordered.map((name) => ({ name, search: buildProviderSearch(name) })),
-            mode: fanoutConfig.mode
+            // The timeout sits inside the policy so a stalled provider gets its one transient retry.
+            providers: ordered.map((name) => ({ name, search: guarded(name, withCallTimeout(buildProviderSearch(name), timeoutMs, name)) })),
+            mode: fanoutConfig.mode,
+            timeoutMs
         });
     }
     // Keep the keyless Tavily safety net for the no-key DuckDuckGo default, even under fanout —
@@ -214,11 +238,12 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
     const keylessFallbackDisabled = process.env.PI_WEB_AGENT_DISABLE_KEYLESS_FALLBACK === '1';
     const usingDuckDuckGoDefault = config.search.provider === 'duckduckgo' || !config.search.provider;
     if (usingDuckDuckGoDefault && !keylessFallbackDisabled) {
-        search = withSearchFallback(search, createTavilySearch({ keyless: true, fetchImpl }), 'duckduckgo');
+        // chainSearch only falls back on non-terminal failures, so a terminal result or bad_request never reaches keyless Tavily.
+        search = chainSearch([search, guarded('tavily', createTavilySearch({ keyless: true, fetchImpl }), 'tavily-keyless')], policyDeps);
     }
-    const httpFetch = createHttpFetch({ fetchPage: createHttpFetcher({ fetchImpl }) });
-    let fetchPage = config.fetch.provider === 'firecrawl'
-        ? config.fetch.baseUrl
+    const httpFetch = createHttpFetch({ fetchPage: createHttpFetcher({ fetchImpl: targetFetch }) });
+    const fetchPage = config.fetch.provider === 'firecrawl'
+        ? withFetchPolicy(config.fetch.baseUrl
             ? createHttpFetch({
                 fetchPage: createFirecrawlFetch({
                     baseUrl: config.fetch.baseUrl,
@@ -227,19 +252,21 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
                     fetchImpl
                 })
             })
-            : createHttpFetch({ fetchPage: invalidFirecrawlFetch() })
+            : createHttpFetch({ fetchPage: invalidFirecrawlFetch() }), config.fetch.fallback === 'http' ? httpFetch : undefined, policyDeps)
         : httpFetch;
-    if (config.fetch.provider === 'firecrawl' && config.fetch.fallback === 'http') {
-        fetchPage = withFetchFallback(fetchPage, httpFetch);
-    }
     const fetchPageWithReaders = createSpecialContentResolver({
-        readers: [createGithubReader({ fetchImpl }), createPdfReader({ fetchImpl }), createYoutubeReader({ fetchImpl })],
+        readers: [
+            createGithubReader({ fetchImpl: targetFetch }),
+            createPdfReader({ fetchImpl: targetFetch }),
+            createYoutubeReader({ fetchImpl: targetFetch })
+        ],
         fallback: fetchPage
     });
-    const headlessPage = (url) => proxyBrowserOptions ? headlessFetch(url, { proxy: proxyBrowserOptions }) : headlessFetch(url);
+    const headlessPage = (url) => headlessFetch(url, { guard: networkGuard, guardProxy: getGuardProxy });
     return {
         search,
-        fetchPage: fetchPageWithReaders,
-        headlessFetch: createHeadlessFetch({ fetchPage: headlessPage })
+        fetchPage: withTargetGuard(fetchPageWithReaders, networkGuard, config.fetch.provider === 'firecrawl' ? 'firecrawl' : 'http'),
+        headlessFetch: createHeadlessFetch({ fetchPage: headlessPage }),
+        close
     };
 }

package/dist/backends/failure.d.ts

@@ -0,0 +1,11 @@
+import type { FailureInfo, FailureKind, ToolError } from '../types.js';
+/** Terminal failures never retry, never fall back, never escalate to headless (#55). */
+export declare function isTerminalFailure(failure: FailureInfo | undefined): boolean;
+export declare function shouldFallBack(kind: FailureKind): boolean;
+/** An error result's failure. Unclassified errors are treated as bad_response: fall back, no retry, no state. */
+export declare function failureOf(result: {
+    status: string;
+    error?: ToolError;
+}): FailureInfo | undefined;
+/** RFC 9110 Retry-After: delay-seconds (non-negative integer) or an HTTP-date. */
+export declare function parseRetryAfter(value: string | null | undefined, now: number): number | undefined;

package/dist/backends/failure.js

@@ -0,0 +1,34 @@
+const TERMINAL_KINDS = new Set(['config_global', 'guard_refused']);
+const NO_FALLBACK_KINDS = new Set(['bad_request', 'config_global', 'guard_refused']);
+/** Terminal failures never retry, never fall back, never escalate to headless (#55). */
+export function isTerminalFailure(failure) {
+    return failure !== undefined && TERMINAL_KINDS.has(failure.kind);
+}
+export function shouldFallBack(kind) {
+    return !NO_FALLBACK_KINDS.has(kind);
+}
+/** An error result's failure. Unclassified errors are treated as bad_response: fall back, no retry, no state. */
+export function failureOf(result) {
+    if (result.status !== 'error')
+        return undefined;
+    return result.error?.failure ?? { kind: 'bad_response' };
+}
+/** RFC 9110 Retry-After: delay-seconds (non-negative integer) or an HTTP-date. */
+export function parseRetryAfter(value, now) {
+    if (!value)
+        return undefined;
+    const trimmed = value.trim();
+    if (/^\d+$/.test(trimmed)) {
+        const ms = Number(trimmed) * 1000;
+        // A huge digit string overflows to Infinity; keep it finite (and JSON-safe) so the
+        // cooldown clamps to the maximum instead of falling back to the default.
+        return Number.isFinite(ms) ? Math.min(ms, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER;
+    }
+    // IMF-fixdate only (RFC 9110 preferred form). V8's Date.parse accepts things like "-5" or "abc 2099".
+    if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(trimmed))
+        return undefined;
+    const at = Date.parse(trimmed);
+    if (Number.isNaN(at) || at <= now)
+        return undefined;
+    return at - now;
+}

package/dist/backends/fallback-policy.d.ts

@@ -0,0 +1,33 @@
+import type { SearchProviderName, WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ProviderHealth } from './provider-health.js';
+export declare const RETRY_BASE_MS = 500;
+export declare const RETRY_JITTER_MS = 250;
+export type PolicyDeps = {
+    health: ProviderHealth;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+    random?: () => number;
+};
+type Search = (input: {
+    query: string;
+}) => Promise<WebSearchResponse>;
+type FetchPage = (input: {
+    url: string;
+}) => Promise<WebFetchResponse>;
+/**
+ * One provider under the #55 policy: skip when cooling down or disabled,
+ * retry exactly once on transient, record state. Never falls back itself.
+ */
+export declare function withSearchPolicy(name: SearchProviderName, search: Search, deps: PolicyDeps, healthKey?: string): Search;
+/**
+ * Tries providers in order under the precedence in the #55 spec: terminal and
+ * bad_request failures stop the chain; a result or a valid empty response is
+ * returned; everything else falls back.
+ */
+export declare function chainSearch(providers: Search[], deps: PolicyDeps): Search;
+/**
+ * Firecrawl under the policy, with the optional HTTP fallback. The HTTP fetcher
+ * talks to the model-chosen site, not a service, so it has no provider health.
+ */
+export declare function withFetchPolicy(primary: FetchPage, fallback: FetchPage | undefined, deps: PolicyDeps, healthKey?: string): FetchPage;
+export {};