@demigodmode/pi-web-agent 1.10.0 → 1.12.0

package/dist/backends/factory.js

@@ -1,16 +1,26 @@
 import { createFirecrawlFetcher } from '../fetch/firecrawl-fetch.js';
+import { createHttpFetcher } from '../fetch/http-fetch.js';
+import { createProxyFetch, resolveProxyCredentials } from '../fetch/proxy-fetch.js';
+import { headlessFetch } from '../fetch/headless-fetch.js';
+import { createGuardedFetch } from '../fetch/guarded-fetch.js';
+import { startGuardProxy } from '../fetch/guard-proxy.js';
+import { createGuardProxyFetch } from '../fetch/guard-proxy-fetch.js';
+import { createNetworkGuard, findGuardError } from '../fetch/network-guard.js';
 import { createBraveSearchTool } from '../search/brave.js';
 import { createYouComSearchTool } from '../search/youcom.js';
+import { fetchDuckDuckGoHtml } from '../search/duckduckgo.js';
 import { createExaSearchTool } from '../search/exa.js';
 import { createTavilySearchTool } from '../search/tavily.js';
 import { createSearxngSearchTool } from '../search/searxng.js';
-import { createFanoutSearch } from '../search/fanout.js';
+import { createFanoutSearch, FANOUT_PROVIDER_TIMEOUT_MS, withCallTimeout } from '../search/fanout.js';
+import { chainSearch, withFetchPolicy, withSearchPolicy } from './fallback-policy.js';
+import { createProviderHealth } from './provider-health.js';
 import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
 import { buildSearchPresentation } from '../presentation/search-presentation.js';
 import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
 import { createWebFetchTool } from '../tools/web-fetch.js';
 import { createWebSearchTool } from '../tools/web-search.js';
-import { DEFAULT_BACKEND_CONFIG, usableSearchProviders } from './config.js';
+import { DEFAULT_BACKEND_CONFIG, isValidProxyUrl, stripProxyCredentials, usableSearchProviders } from './config.js';
 import { createSpecialContentResolver } from '../readers/resolver.js';
 import { createGithubReader } from '../readers/github-reader.js';
 import { createPdfReader } from '../readers/pdf-reader.js';
@@ -23,7 +33,8 @@ function invalidSearxngSearch() {
             metadata: { backend: 'searxng', cacheHit: false },
             error: {
                 code: 'BACKEND_CONFIG_INVALID',
-                message: 'SearXNG search requires backends.search.baseUrl.'
+                message: 'SearXNG search requires backends.search.baseUrl.',
+                failure: { kind: 'not_configured' }
             }
         };
         return { ...result, presentation: buildSearchPresentation(result) };
@@ -37,47 +48,36 @@ function invalidFirecrawlFetch() {
             metadata: { method: 'firecrawl', cacheHit: false },
             error: {
                 code: 'BACKEND_CONFIG_INVALID',
-                message: 'Firecrawl fetch requires backends.fetch.baseUrl.'
+                message: 'Firecrawl fetch requires backends.fetch.baseUrl.',
+                failure: { kind: 'not_configured' }
             }
         };
         return { ...result, presentation: buildFetchPresentation(result) };
     };
 }
-function withSearchFallback(primary, fallback, fallbackFrom) {
+/**
+ * Checks the target URL before dispatching to the http fetcher, a content
+ * reader, or Firecrawl. Doing it here means a private URL is refused once, in
+ * one place, and never reaches Firecrawl or its http fallback (#53).
+ */
+function withTargetGuard(fetchPage, guard, method) {
     return async (input) => {
-        const first = await primary(input);
-        if (first.status !== 'error')
-            return first;
-        const second = await fallback(input);
-        const result = {
-            ...second,
-            metadata: {
-                ...second.metadata,
-                fallbackFrom,
-                fallbackReason: first.error?.message ?? `${fallbackFrom} search failed.`,
-                // Keep the primary's fanout provenance (which providers were tried/skipped) even though
-                // the answer came from the fallback backend.
-                ...(first.metadata.fanout ? { fanout: first.metadata.fanout } : {})
-            }
-        };
-        return { ...result, presentation: buildSearchPresentation(result) };
-    };
-}
-function withFetchFallback(primary, fallback) {
-    return async (input) => {
-        const first = await primary(input);
-        if (first.status !== 'error' && first.status !== 'needs_headless')
-            return first;
-        const second = await fallback(input);
-        const result = {
-            ...second,
-            metadata: {
-                ...second.metadata,
-                fallbackFrom: 'firecrawl',
-                fallbackReason: first.error?.message ?? 'Firecrawl fetch failed.'
-            }
-        };
-        return { ...result, presentation: buildFetchPresentation(result) };
+        try {
+            await guard.assertUrlAllowed(input.url);
+        }
+        catch (error) {
+            const blocked = findGuardError(error);
+            if (!blocked)
+                throw error;
+            const result = {
+                status: 'error',
+                url: input.url,
+                metadata: { method, cacheHit: false },
+                error: { code: blocked.code, message: blocked.message, failure: { kind: 'guard_refused' } }
+            };
+            return { ...result, presentation: buildFetchPresentation(result) };
+        }
+        return fetchPage(input);
     };
 }
 export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
@@ -90,53 +90,130 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
     const createHttpFetch = deps.createHttpFetch ?? createWebFetchTool;
     const createFirecrawlFetch = deps.createFirecrawlFetch ?? createFirecrawlFetcher;
     const createHeadlessFetch = deps.createHeadlessFetch ?? createWebFetchHeadlessTool;
+    const makeProxyFetch = deps.createProxyFetch ?? createProxyFetch;
+    // A blank url is the "disable proxy" marker: treat it as no proxy at all.
+    const proxy = config.proxy && config.proxy.url.trim() !== '' ? config.proxy : undefined;
+    // A configured proxy whose url fails validation must not be silently ignored
+    // — that would send traffic direct to the websites. Every request errors out
+    // instead. No connectivity check is needed: the url itself is the problem.
+    if (proxy && !isValidProxyUrl(proxy.url)) {
+        const message = `backends.proxy.url (${proxy.url}) is not a valid http or https URL. ` +
+            'Web requests are blocked until it is fixed; set backends.proxy.url to "" to disable the proxy.';
+        return {
+            search: async () => {
+                const result = {
+                    status: 'error',
+                    results: [],
+                    metadata: { backend: config.search.provider, cacheHit: false },
+                    error: { code: 'BACKEND_CONFIG_INVALID', message, failure: { kind: 'config_global' } }
+                };
+                return { ...result, presentation: buildSearchPresentation(result) };
+            },
+            fetchPage: async ({ url }) => {
+                const result = {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'http', cacheHit: false },
+                    error: { code: 'BACKEND_CONFIG_INVALID', message, failure: { kind: 'config_global' } }
+                };
+                return { ...result, presentation: buildFetchPresentation(result) };
+            },
+            headlessFetch: async ({ url }) => {
+                const result = {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'headless', cacheHit: false },
+                    error: { code: 'BACKEND_CONFIG_INVALID', message, failure: { kind: 'config_global' } }
+                };
+                return { ...result, presentation: buildFetchPresentation(result) };
+            },
+            close: async () => undefined
+        };
+    }
+    // When a proxy is configured, every outbound HTTP request goes through it.
+    // User-configured endpoints use fetchImpl directly; model-chosen fetches and
+    // the headless browser reach it through the guard proxy below.
+    const fetchImpl = proxy ? makeProxyFetch(proxy) : fetch;
+    const proxyCredentials = proxy ? resolveProxyCredentials(proxy) : undefined;
+    // Model-chosen URLs only. Search APIs and the configured SearXNG/Firecrawl
+    // endpoints keep using fetchImpl: the user typed those (#53).
+    const networkGuard = deps.networkGuard ?? createNetworkGuard({ allowRanges: config.network?.allowRanges ?? [] });
+    // One guard proxy per backend set, started on first use. It is the single
+    // place the address policy is enforced, for Node fetches and the browser,
+    // and it chains to the user's upstream proxy itself.
+    let guardProxy;
+    let isClosed = false;
+    const getGuardProxy = () => {
+        if (isClosed)
+            return Promise.reject(new Error('Backend set is closed.'));
+        if (guardProxy)
+            return guardProxy;
+        const started = (deps.createGuardProxy ?? startGuardProxy)({
+            guard: networkGuard,
+            ...(proxy
+                ? {
+                    upstream: {
+                        url: stripProxyCredentials(proxy.url),
+                        ...(proxyCredentials?.username !== undefined ? { username: proxyCredentials.username } : {}),
+                        ...(proxyCredentials?.password !== undefined ? { password: proxyCredentials.password } : {})
+                    }
+                }
+                : {}),
+            trustProxyDns: config.network?.trustProxyDns === true
+        });
+        guardProxy = started;
+        // A failed start should not stick around for the rest of the backend
+        // set's life: clear it so the next call tries again, unless a newer
+        // attempt has already replaced it.
+        started.catch(() => {
+            if (guardProxy === started)
+                guardProxy = undefined;
+        });
+        return started;
+    };
+    const modelFetch = deps.createModelFetch
+        ? deps.createModelFetch(networkGuard)
+        : createGuardProxyFetch(getGuardProxy);
+    const targetFetch = createGuardedFetch(modelFetch, networkGuard);
+    let closing;
+    const close = () => (closing ??= (async () => {
+        isClosed = true;
+        if ('close' in modelFetch)
+            await modelFetch.close();
+        if (guardProxy) {
+            const started = await guardProxy.catch(() => undefined);
+            await started?.close().catch(() => undefined);
+        }
+    })());
+    // One health state per backend set: a rebuilt set (config change) starts fresh.
+    const policyDeps = { health: deps.providerHealth ?? createProviderHealth(deps.policy?.now ? { now: deps.policy.now } : {}), ...deps.policy };
+    const guarded = (name, search, healthKey) => withSearchPolicy(name, search, policyDeps, healthKey);
+    const createDuckDuckGo = () => createDuckDuckGoSearch({ searchHtml: (query) => fetchDuckDuckGoHtml(query, { fetchImpl }) });
     function buildProviderSearch(name) {
         switch (name) {
             case 'searxng':
                 return config.search.baseUrl
-                    ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
+                    ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options, fetchImpl })
                     : invalidSearxngSearch();
             case 'brave':
-                return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY });
+                return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY, fetchImpl });
             case 'youcom':
-                return createYouComSearch({ apiKey: process.env.YDC_API_KEY });
+                return createYouComSearch({ apiKey: process.env.YDC_API_KEY, fetchImpl });
             case 'exa':
-                return createExaSearch({ apiKey: process.env.EXA_API_KEY });
+                return createExaSearch({ apiKey: process.env.EXA_API_KEY, fetchImpl });
             case 'tavily':
-                return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY });
+                return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY, fetchImpl });
             case 'duckduckgo':
             default:
-                return createDuckDuckGoSearch();
+                return createDuckDuckGo();
         }
     }
-    let search = config.search.provider === 'searxng'
-        ? config.search.baseUrl
-            ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
-            : invalidSearxngSearch()
-        : config.search.provider === 'brave'
-            ? createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY })
-            : config.search.provider === 'youcom'
-                ? createYouComSearch({ apiKey: process.env.YDC_API_KEY })
-                : config.search.provider === 'exa'
-                    ? createExaSearch({ apiKey: process.env.EXA_API_KEY })
-                    : config.search.provider === 'tavily'
-                        ? createTavilySearch({ apiKey: process.env.TAVILY_API_KEY })
-                        : createDuckDuckGoSearch();
-    if (config.search.provider === 'searxng' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGoSearch(), 'searxng');
-    }
-    if (config.search.provider === 'brave' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGoSearch(), 'brave');
-    }
-    if (config.search.provider === 'youcom' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGoSearch(), 'youcom');
-    }
-    if (config.search.provider === 'exa' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGoSearch(), 'exa');
-    }
-    if (config.search.provider === 'tavily' && config.search.fallback === 'duckduckgo') {
-        search = withSearchFallback(search, createDuckDuckGoSearch(), 'tavily');
+    const primarySearch = guarded(config.search.provider, buildProviderSearch(config.search.provider));
+    const chain = [primarySearch];
+    if (config.search.provider !== 'duckduckgo' && config.search.fallback === 'duckduckgo') {
+        chain.push(guarded('duckduckgo', createDuckDuckGo()));
     }
+    let search = chainSearch(chain, policyDeps);
     const fanoutConfig = config.search.fanout;
     if (fanoutConfig && fanoutConfig.mode !== 'off') {
         const baseNames = fanoutConfig.providers && fanoutConfig.providers.length > 0
@@ -147,9 +224,12 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
             ? [...baseNames, 'duckduckgo']
             : baseNames;
         const ordered = [config.search.provider, ...providerNames.filter((n) => n !== config.search.provider)].filter((n, i, arr) => arr.indexOf(n) === i);
+        const timeoutMs = deps.fanoutTimeoutMs ?? FANOUT_PROVIDER_TIMEOUT_MS;
         search = createFanoutSearch({
-            providers: ordered.map((name) => ({ name, search: buildProviderSearch(name) })),
-            mode: fanoutConfig.mode
+            // The timeout sits inside the policy so a stalled provider gets its one transient retry.
+            providers: ordered.map((name) => ({ name, search: guarded(name, withCallTimeout(buildProviderSearch(name), timeoutMs, name)) })),
+            mode: fanoutConfig.mode,
+            timeoutMs
         });
     }
     // Keep the keyless Tavily safety net for the no-key DuckDuckGo default, even under fanout —
@@ -158,30 +238,35 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
     const keylessFallbackDisabled = process.env.PI_WEB_AGENT_DISABLE_KEYLESS_FALLBACK === '1';
     const usingDuckDuckGoDefault = config.search.provider === 'duckduckgo' || !config.search.provider;
     if (usingDuckDuckGoDefault && !keylessFallbackDisabled) {
-        search = withSearchFallback(search, createTavilySearch({ keyless: true }), 'duckduckgo');
+        // chainSearch only falls back on non-terminal failures, so a terminal result or bad_request never reaches keyless Tavily.
+        search = chainSearch([search, guarded('tavily', createTavilySearch({ keyless: true, fetchImpl }), 'tavily-keyless')], policyDeps);
     }
-    const httpFetch = createHttpFetch();
-    let fetchPage = config.fetch.provider === 'firecrawl'
-        ? config.fetch.baseUrl
+    const httpFetch = createHttpFetch({ fetchPage: createHttpFetcher({ fetchImpl: targetFetch }) });
+    const fetchPage = config.fetch.provider === 'firecrawl'
+        ? withFetchPolicy(config.fetch.baseUrl
             ? createHttpFetch({
                 fetchPage: createFirecrawlFetch({
                     baseUrl: config.fetch.baseUrl,
                     apiKey: config.fetch.apiKey ?? process.env.PI_WEB_AGENT_FIRECRAWL_API_KEY,
-                    options: config.fetch.options
+                    options: config.fetch.options,
+                    fetchImpl
                 })
             })
-            : createHttpFetch({ fetchPage: invalidFirecrawlFetch() })
+            : createHttpFetch({ fetchPage: invalidFirecrawlFetch() }), config.fetch.fallback === 'http' ? httpFetch : undefined, policyDeps)
         : httpFetch;
-    if (config.fetch.provider === 'firecrawl' && config.fetch.fallback === 'http') {
-        fetchPage = withFetchFallback(fetchPage, httpFetch);
-    }
     const fetchPageWithReaders = createSpecialContentResolver({
-        readers: [createGithubReader(), createPdfReader(), createYoutubeReader()],
+        readers: [
+            createGithubReader({ fetchImpl: targetFetch }),
+            createPdfReader({ fetchImpl: targetFetch }),
+            createYoutubeReader({ fetchImpl: targetFetch })
+        ],
         fallback: fetchPage
     });
+    const headlessPage = (url) => headlessFetch(url, { guard: networkGuard, guardProxy: getGuardProxy });
     return {
         search,
-        fetchPage: fetchPageWithReaders,
-        headlessFetch: createHeadlessFetch()
+        fetchPage: withTargetGuard(fetchPageWithReaders, networkGuard, config.fetch.provider === 'firecrawl' ? 'firecrawl' : 'http'),
+        headlessFetch: createHeadlessFetch({ fetchPage: headlessPage }),
+        close
     };
 }

package/dist/backends/failure.d.ts

@@ -0,0 +1,11 @@
+import type { FailureInfo, FailureKind, ToolError } from '../types.js';
+/** Terminal failures never retry, never fall back, never escalate to headless (#55). */
+export declare function isTerminalFailure(failure: FailureInfo | undefined): boolean;
+export declare function shouldFallBack(kind: FailureKind): boolean;
+/** An error result's failure. Unclassified errors are treated as bad_response: fall back, no retry, no state. */
+export declare function failureOf(result: {
+    status: string;
+    error?: ToolError;
+}): FailureInfo | undefined;
+/** RFC 9110 Retry-After: delay-seconds (non-negative integer) or an HTTP-date. */
+export declare function parseRetryAfter(value: string | null | undefined, now: number): number | undefined;

package/dist/backends/failure.js

@@ -0,0 +1,34 @@
+const TERMINAL_KINDS = new Set(['config_global', 'guard_refused']);
+const NO_FALLBACK_KINDS = new Set(['bad_request', 'config_global', 'guard_refused']);
+/** Terminal failures never retry, never fall back, never escalate to headless (#55). */
+export function isTerminalFailure(failure) {
+    return failure !== undefined && TERMINAL_KINDS.has(failure.kind);
+}
+export function shouldFallBack(kind) {
+    return !NO_FALLBACK_KINDS.has(kind);
+}
+/** An error result's failure. Unclassified errors are treated as bad_response: fall back, no retry, no state. */
+export function failureOf(result) {
+    if (result.status !== 'error')
+        return undefined;
+    return result.error?.failure ?? { kind: 'bad_response' };
+}
+/** RFC 9110 Retry-After: delay-seconds (non-negative integer) or an HTTP-date. */
+export function parseRetryAfter(value, now) {
+    if (!value)
+        return undefined;
+    const trimmed = value.trim();
+    if (/^\d+$/.test(trimmed)) {
+        const ms = Number(trimmed) * 1000;
+        // A huge digit string overflows to Infinity; keep it finite (and JSON-safe) so the
+        // cooldown clamps to the maximum instead of falling back to the default.
+        return Number.isFinite(ms) ? Math.min(ms, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER;
+    }
+    // IMF-fixdate only (RFC 9110 preferred form). V8's Date.parse accepts things like "-5" or "abc 2099".
+    if (!/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(trimmed))
+        return undefined;
+    const at = Date.parse(trimmed);
+    if (Number.isNaN(at) || at <= now)
+        return undefined;
+    return at - now;
+}

package/dist/backends/fallback-policy.d.ts

@@ -0,0 +1,33 @@
+import type { SearchProviderName, WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ProviderHealth } from './provider-health.js';
+export declare const RETRY_BASE_MS = 500;
+export declare const RETRY_JITTER_MS = 250;
+export type PolicyDeps = {
+    health: ProviderHealth;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+    random?: () => number;
+};
+type Search = (input: {
+    query: string;
+}) => Promise<WebSearchResponse>;
+type FetchPage = (input: {
+    url: string;
+}) => Promise<WebFetchResponse>;
+/**
+ * One provider under the #55 policy: skip when cooling down or disabled,
+ * retry exactly once on transient, record state. Never falls back itself.
+ */
+export declare function withSearchPolicy(name: SearchProviderName, search: Search, deps: PolicyDeps, healthKey?: string): Search;
+/**
+ * Tries providers in order under the precedence in the #55 spec: terminal and
+ * bad_request failures stop the chain; a result or a valid empty response is
+ * returned; everything else falls back.
+ */
+export declare function chainSearch(providers: Search[], deps: PolicyDeps): Search;
+/**
+ * Firecrawl under the policy, with the optional HTTP fallback. The HTTP fetcher
+ * talks to the model-chosen site, not a service, so it has no provider health.
+ */
+export declare function withFetchPolicy(primary: FetchPage, fallback: FetchPage | undefined, deps: PolicyDeps, healthKey?: string): FetchPage;
+export {};

package/dist/backends/fallback-policy.js

@@ -0,0 +1,239 @@
+import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
+import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import { failureOf, shouldFallBack } from './failure.js';
+export const RETRY_BASE_MS = 500;
+export const RETRY_JITTER_MS = 250;
+const USER_FIXABLE_KINDS = new Set(['not_configured', 'auth_failed', 'quota_exhausted']);
+/** Only user-fixable failures keep the provider's message; everything else stays message-free. */
+function detailFor(failure, message) {
+    return message && USER_FIXABLE_KINDS.has(failure.kind) ? message : undefined;
+}
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function retryDelay(deps) {
+    return RETRY_BASE_MS + Math.floor((deps.random ?? Math.random)() * RETRY_JITTER_MS);
+}
+function skipAttempt(backend, state) {
+    return {
+        backend,
+        outcome: 'skipped',
+        failure: state.failure,
+        skipReason: state.state,
+        ...(state.state === 'cooling_down' ? { cooldownUntil: state.until } : {}),
+        ...(state.detail ? { detail: state.detail } : {})
+    };
+}
+function skipMessage(backend, state) {
+    const base = state.state === 'cooling_down'
+        ? `${backend} is cooling down after ${state.failure.kind} until ${new Date(state.until).toISOString()}.`
+        : `${backend} is disabled for this session after ${state.failure.kind}.`;
+    return state.detail ? `${base} ${state.detail}` : base;
+}
+function failedAttempt(backend, failure, state, detail) {
+    return {
+        backend,
+        outcome: 'failed',
+        failure,
+        ...(state.state === 'cooling_down' ? { cooldownUntil: state.until } : {}),
+        ...(detail ? { detail } : {})
+    };
+}
+/**
+ * One provider under the #55 policy: skip when cooling down or disabled,
+ * retry exactly once on transient, record state. Never falls back itself.
+ */
+export function withSearchPolicy(name, search, deps, healthKey = name) {
+    return async (input) => {
+        const state = deps.health.get(healthKey);
+        if (state.state !== 'available') {
+            return {
+                status: 'error',
+                results: [],
+                metadata: { backend: name, cacheHit: false, attempts: [skipAttempt(name, state)] },
+                error: { code: 'BACKEND_UNAVAILABLE', message: skipMessage(name, state), failure: state.failure }
+            };
+        }
+        const attempts = [];
+        let result = await search(input);
+        let failure = failureOf(result);
+        if (failure?.kind === 'transient') {
+            attempts.push({ backend: name, outcome: 'retried', failure });
+            await (deps.sleep ?? defaultSleep)(retryDelay(deps));
+            result = await search(input);
+            failure = failureOf(result);
+        }
+        if (failure) {
+            const detail = detailFor(failure, result.error?.message);
+            attempts.push(failedAttempt(name, failure, deps.health.record(healthKey, failure, detail), detail));
+        }
+        else {
+            attempts.push({ backend: name, outcome: result.results.length > 0 ? 'results' : 'empty' });
+        }
+        return { ...result, metadata: { ...result.metadata, attempts: [...(result.metadata.attempts ?? []), ...attempts] } };
+    };
+}
+function unavailableMessage(attempts, messages = new Map()) {
+    const byProvider = new Map();
+    for (const attempt of attempts) {
+        if (attempt.outcome === 'failed' || attempt.outcome === 'skipped')
+            byProvider.set(attempt.backend, attempt);
+    }
+    const entries = [...byProvider.values()].sort((a, b) => {
+        const aUntil = a.cooldownUntil ?? Number.POSITIVE_INFINITY;
+        const bUntil = b.cooldownUntil ?? Number.POSITIVE_INFINITY;
+        return aUntil - bUntil;
+    });
+    const parts = entries.map((attempt) => {
+        const kind = attempt.failure?.kind ?? 'bad_response';
+        if (attempt.cooldownUntil !== undefined) {
+            return `${attempt.backend} ${kind} (available again at ${new Date(attempt.cooldownUntil).toISOString()})`;
+        }
+        // Keep the provider's own hint for problems the user has to fix, e.g. a missing key or base URL.
+        const hint = USER_FIXABLE_KINDS.has(kind)
+            ? attempt.detail ?? (attempt.outcome === 'failed' ? messages.get(attempt.backend) : undefined)
+            : undefined;
+        return hint ? `${attempt.backend} ${kind} (${hint})` : `${attempt.backend} ${kind}`;
+    });
+    return `No search backend is available: ${parts.join(', ')}.`;
+}
+/**
+ * A failed link's unavailable providers, from its own attempts: the last failed or skipped
+ * attempt per backend with that attempt's kind. A fanout link reports each provider, not the
+ * aggregate. Falls back to the aggregate only when the link carries no failed or skipped attempts.
+ */
+function unavailableFor(result, failure) {
+    const message = result.error?.message ?? failure.kind;
+    const byProvider = new Map();
+    for (const entry of result.metadata.coverage?.unavailable ?? []) {
+        byProvider.set(entry.provider, { ...entry, message });
+    }
+    const attempts = result.metadata.attempts ?? [];
+    const relevant = attempts.filter((a) => a.outcome === 'failed' || a.outcome === 'skipped');
+    if (relevant.length === 0) {
+        byProvider.set(result.metadata.backend, { provider: result.metadata.backend, kind: failure.kind, message });
+    }
+    for (const attempt of relevant) {
+        byProvider.delete(attempt.backend); // keep insertion order at the latest attempt
+        byProvider.set(attempt.backend, {
+            provider: attempt.backend,
+            kind: attempt.failure?.kind ?? failure.kind,
+            message: attempt.detail ?? message
+        });
+    }
+    return [...byProvider.values()];
+}
+/**
+ * Tries providers in order under the precedence in the #55 spec: terminal and
+ * bad_request failures stop the chain; a result or a valid empty response is
+ * returned; everything else falls back.
+ */
+export function chainSearch(providers, deps) {
+    return async (input) => {
+        const attempts = [];
+        const unavailable = [];
+        let lastFailure;
+        let firstBackend;
+        // Keep an earlier fanout's provenance (which providers were tried) when a later link answers.
+        let fanout;
+        for (const provider of providers) {
+            const result = await provider(input);
+            firstBackend ??= result.metadata.backend;
+            attempts.push(...(result.metadata.attempts ?? []));
+            fanout ??= result.metadata.fanout;
+            const failure = failureOf(result);
+            if (!failure) {
+                const first = unavailable[0];
+                const merged = {
+                    ...result,
+                    metadata: {
+                        ...result.metadata,
+                        ...(fanout && !result.metadata.fanout ? { fanout } : {}),
+                        attempts,
+                        ...(first
+                            ? {
+                                fallbackFrom: first.provider,
+                                fallbackReason: first.message,
+                                coverage: { partial: true, unavailable: unavailable.map(({ provider, kind }) => ({ provider, kind })) }
+                            }
+                            : {})
+                    }
+                };
+                return { ...merged, presentation: buildSearchPresentation(merged) };
+            }
+            if (!shouldFallBack(failure.kind)) {
+                const stopped = { ...result, metadata: { ...result.metadata, attempts } };
+                return { ...stopped, presentation: buildSearchPresentation(stopped) };
+            }
+            for (const entry of unavailableFor(result, failure)) {
+                const existing = unavailable.findIndex((u) => u.provider === entry.provider);
+                if (existing >= 0)
+                    unavailable.splice(existing, 1);
+                unavailable.push(entry);
+            }
+            lastFailure = failure;
+        }
+        const exhausted = {
+            status: 'error',
+            results: [],
+            metadata: { backend: firstBackend ?? 'duckduckgo', cacheHit: false, attempts },
+            error: {
+                code: 'SEARCH_BACKENDS_UNAVAILABLE',
+                message: unavailableMessage(attempts, new Map(unavailable.map((entry) => [entry.provider, entry.message]))),
+                failure: lastFailure ?? { kind: 'bad_response' }
+            }
+        };
+        return { ...exhausted, presentation: buildSearchPresentation(exhausted) };
+    };
+}
+/**
+ * Firecrawl under the policy, with the optional HTTP fallback. The HTTP fetcher
+ * talks to the model-chosen site, not a service, so it has no provider health.
+ */
+export function withFetchPolicy(primary, fallback, deps, healthKey = 'firecrawl') {
+    const finish = (result) => ({ ...result, presentation: buildFetchPresentation(result) });
+    return async (input) => {
+        const attempts = [];
+        const state = deps.health.get(healthKey);
+        let first;
+        if (state.state !== 'available') {
+            attempts.push(skipAttempt('firecrawl', state));
+            first = {
+                status: 'error',
+                url: input.url,
+                metadata: { method: 'firecrawl', cacheHit: false },
+                error: { code: 'BACKEND_UNAVAILABLE', message: skipMessage('firecrawl', state), failure: state.failure }
+            };
+        }
+        else {
+            first = await primary(input);
+            let failure = failureOf(first);
+            if (failure?.kind === 'transient') {
+                attempts.push({ backend: 'firecrawl', outcome: 'retried', failure });
+                await (deps.sleep ?? defaultSleep)(retryDelay(deps));
+                first = await primary(input);
+                failure = failureOf(first);
+            }
+            if (failure) {
+                const detail = detailFor(failure, first.error?.message);
+                attempts.push(failedAttempt('firecrawl', failure, deps.health.record(healthKey, failure, detail), detail));
+            }
+            else
+                attempts.push({ backend: 'firecrawl', outcome: first.status === 'ok' ? 'results' : 'empty' });
+        }
+        const failure = failureOf(first);
+        const fallBack = first.status === 'needs_headless' || (failure !== undefined && shouldFallBack(failure.kind));
+        if (!fallback || !fallBack) {
+            return finish({ ...first, metadata: { ...first.metadata, attempts } });
+        }
+        const second = await fallback(input);
+        attempts.push({ backend: 'http', outcome: second.status === 'ok' ? 'results' : 'failed', ...(second.error?.failure ? { failure: second.error.failure } : {}) });
+        return finish({
+            ...second,
+            metadata: {
+                ...second.metadata,
+                attempts,
+                fallbackFrom: 'firecrawl',
+                fallbackReason: first.error?.message ?? 'Firecrawl fetch failed.'
+            }
+        });
+    };
+}