@demigodmode/pi-web-agent 1.10.0 → 1.12.0

package/dist/orchestration/research-worker.js

@@ -1,3 +1,4 @@
+import { failureOf, isTerminalFailure } from '../backends/failure.js';
 import { selectCandidates } from './candidate-selector.js';
 import { classifySourceProfile } from './source-profile.js';
 function classifySource(url) {
@@ -80,6 +81,22 @@ export function createResearchWorker({ search, fetchPage }) {
                 };
             }
             const searchResult = await search({ query });
+            const searchCoveragePartial = searchResult.metadata.coverage?.partial === true;
+            const searchAttempts = searchResult.metadata.attempts;
+            if (isTerminalFailure(failureOf(searchResult))) {
+                return {
+                    searchQueries,
+                    evidence,
+                    gaps: [],
+                    lowValueOutcomes,
+                    exhaustedBudget: false,
+                    searchAttempts,
+                    terminalFailure: {
+                        code: searchResult.error?.code ?? 'SEARCH_FAILED',
+                        message: `${searchResult.error?.message ?? 'Search failed.'} (${searchResult.error?.failure?.kind})`
+                    }
+                };
+            }
             const fanoutProviders = searchResult.metadata.fanout?.providers;
             const fanoutSkipped = searchResult.metadata.fanout?.skipped;
             if (searchResult.status !== 'ok') {
@@ -96,7 +113,9 @@ export function createResearchWorker({ search, fetchPage }) {
                     suggestedHeadlessUrl,
                     exhaustedBudget: false,
                     fanoutProviders,
-                    fanoutSkipped
+                    fanoutSkipped,
+                    searchCoveragePartial,
+                    searchAttempts
                 };
             }
             if (searchResult.results.length === 0) {
@@ -113,7 +132,9 @@ export function createResearchWorker({ search, fetchPage }) {
                     suggestedHeadlessUrl,
                     exhaustedBudget: false,
                     fanoutProviders,
-                    fanoutSkipped
+                    fanoutSkipped,
+                    searchCoveragePartial,
+                    searchAttempts
                 };
             }
             const candidates = selectCandidates({
@@ -122,8 +143,11 @@ export function createResearchWorker({ search, fetchPage }) {
                 seenUrls: new Set(evidence.map((item) => item.url)),
                 maxCandidates: maxFetches
             });
+            const fetchAttempts = [];
             for (const candidate of candidates) {
                 const fetched = await fetchPage({ url: candidate.url });
+                if (fetched.metadata.attempts)
+                    fetchAttempts.push(...fetched.metadata.attempts);
                 if (fetched.status === 'ok') {
                     const parsedEvidence = evidenceFromFetch(fetched, candidate.title);
                     if (parsedEvidence) {
@@ -136,6 +160,14 @@ export function createResearchWorker({ search, fetchPage }) {
                     }
                     continue;
                 }
+                // guard_refused and friends are final; never hand them to headless.
+                if (isTerminalFailure(failureOf(fetched))) {
+                    gaps.push({
+                        kind: 'fetch-failed',
+                        message: fetched.error?.message ?? `Fetch failed for ${candidate.url}`
+                    });
+                    continue;
+                }
                 if (fetched.status === 'needs_headless') {
                     if (!suggestedHeadlessUrl) {
                         suggestedHeadlessUrl = fetched.url;
@@ -156,7 +188,10 @@ export function createResearchWorker({ search, fetchPage }) {
                 suggestedHeadlessUrl,
                 exhaustedBudget: false,
                 fanoutProviders,
-                fanoutSkipped
+                fanoutSkipped,
+                searchCoveragePartial,
+                searchAttempts,
+                fetchAttempts
             };
         }
     };

package/dist/orchestration/stop-decider.js

@@ -1,6 +1,8 @@
 import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
 function activeCaveatReasons(evidence, quality) {
-    const reasons = quality?.caveatReasons ?? [];
+    // Partial search coverage is reported as a caveat but never changes the decision (#55):
+    // a strong answer stays an answer, and web-explore adds just the coverage sentence.
+    const reasons = (quality?.caveatReasons ?? []).filter((reason) => reason !== 'partial-search-coverage');
     if (!hasOfficialDocsAndApi(evidence))
         return reasons;
     return reasons.filter((reason) => reason !== 'low-diversity');

package/dist/presentation/config-store.js

@@ -61,6 +61,16 @@ function serializeBackendConfigOverride(config) {
     if (config.headless && Object.keys(config.headless).length > 0) {
         backends.headless = { ...config.headless };
     }
+    if (config.proxy && Object.keys(config.proxy).length > 0) {
+        const { password: _password, ...proxy } = config.proxy;
+        backends.proxy = { ...proxy };
+    }
+    if (config.network) {
+        backends.network = {
+            ...config.network,
+            ...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {})
+        };
+    }
     return { backends };
 }
 async function readConfigFileForWrite(filePath) {

package/dist/presentation/explore-presentation.js

@@ -1,3 +1,4 @@
+import { attemptLines } from './search-presentation.js';
 function internalReaderLabel(method) {
     if (method === 'headless')
         return 'web_fetch_headless';
@@ -49,7 +50,8 @@ export function buildExplorePresentation(result) {
         'Sources',
         ...result.sources.map((source) => `- [${internalReaderLabel(source.method)}] ${source.title}: ${source.url}`),
         internalSummary ? `\nInternal tools\n${internalSummary}` : undefined,
-        result.caveat ? `\nCaveat\n${result.caveat}` : undefined
+        result.caveat ? `\nCaveat\n${result.caveat}` : undefined,
+        attemptLines(result.metadata?.attempts)
     ]
         .filter((line) => line !== undefined)
         .join('\n');

package/dist/presentation/fetch-presentation.js

@@ -1,3 +1,4 @@
+import { attemptLines } from './search-presentation.js';
 function countWords(text) {
     return text?.trim() ? text.trim().split(/\s+/).length : undefined;
 }
@@ -24,15 +25,21 @@ export function buildFetchPresentation(result) {
             preview: result.content?.title
                 ? `${result.content.title}\n${firstExcerpt(result.content.text) ?? ''}`.trim()
                 : firstExcerpt(result.content?.text),
-            verbose: result.status === 'ok'
-                ? [
-                    `URL: ${result.url}`,
-                    result.content?.title ? `Title: ${result.content.title}` : undefined,
-                    firstExcerpt(result.content?.text, 500)
-                ]
-                    .filter(Boolean)
-                    .join('\n')
-                : undefined
+            verbose: [
+                ...(result.status === 'ok'
+                    ? [
+                        `URL: ${result.url}`,
+                        result.content?.title ? `Title: ${result.content.title}` : undefined,
+                        firstExcerpt(result.content?.text, 500),
+                        result.metadata.blockedSubresources
+                            ? `Blocked private-address requests: ${result.metadata.blockedSubresources}`
+                            : undefined
+                    ]
+                    : []),
+                attemptLines(result.metadata.attempts)
+            ]
+                .filter(Boolean)
+                .join('\n') || undefined
         },
         metrics: {
             wordCount,

package/dist/presentation/search-presentation.d.ts

@@ -1,3 +1,4 @@
-import type { WebSearchResponse } from '../types.js';
+import type { Attempt, WebSearchResponse } from '../types.js';
 import type { PresentationEnvelope } from './types.js';
+export declare function attemptLines(attempts: Attempt[] | undefined): string | undefined;
 export declare function buildSearchPresentation(result: WebSearchResponse): PresentationEnvelope;

package/dist/presentation/search-presentation.js

@@ -10,6 +10,18 @@ function fanoutNote(result) {
     }
     return '';
 }
+export function attemptLines(attempts) {
+    const interesting = (attempts ?? []).filter((a) => a.outcome !== 'results' && a.outcome !== 'empty');
+    if (interesting.length === 0)
+        return undefined;
+    return interesting
+        .map((a) => {
+        const kind = a.failure?.kind ? ` (${a.failure.kind})` : '';
+        const until = a.cooldownUntil !== undefined ? `, cooling down until ${new Date(a.cooldownUntil).toISOString()}` : '';
+        return `${a.backend}: ${a.outcome}${a.skipReason ? ` [${a.skipReason}]` : ''}${kind}${until}`;
+    })
+        .join('\n');
+}
 function formatCompact(result) {
     const fallbackPrefix = result.metadata.fallbackFrom
         ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
@@ -34,7 +46,7 @@ export function buildSearchPresentation(result) {
         views: {
             compact: formatCompact(result),
             preview: preview || undefined,
-            verbose: verbose || undefined
+            verbose: [verbose, attemptLines(result.metadata.attempts)].filter(Boolean).join('\n') || undefined
         },
         metrics: {
             resultCount: result.results.length,

package/dist/readers/youtube-reader.d.ts

@@ -5,6 +5,8 @@ type Subtitle = {
     text: string;
 };
 type YoutubeReaderDeps = {
+    /** Fetch implementation used for YouTube's requests (defaults to global fetch). */
+    fetchImpl?: typeof fetch;
     fetchSubtitles?: (input: {
         videoID: string;
         lang: string;
@@ -17,5 +19,5 @@ type YoutubeReaderDeps = {
         description?: string;
     }>;
 };
-export declare function createYoutubeReader({ fetchSubtitles, fetchDetails }?: YoutubeReaderDeps): SpecialContentReader;
+export declare function createYoutubeReader({ fetchImpl, fetchSubtitles, fetchDetails }?: YoutubeReaderDeps): SpecialContentReader;
 export {};

package/dist/readers/youtube-reader.js

@@ -21,7 +21,15 @@ function extractVideoId(url) {
     }
     return undefined;
 }
-export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetails = getVideoDetails } = {}) {
+export function createYoutubeReader({ fetchImpl = fetch, fetchSubtitles, fetchDetails } = {}) {
+    // Route YouTube's caption/metadata requests through the configured fetch
+    // client (e.g. the proxy) unless an explicit override is provided.
+    const doFetchSubtitles = fetchSubtitles ?? ((input) => getSubtitles({ ...input, fetch: fetchImpl }));
+    const doFetchDetails = fetchDetails ??
+        ((input) => getVideoDetails({ ...input, fetch: fetchImpl }).then((details) => ({
+            title: details.title,
+            description: details.description
+        })));
     return {
         name: 'youtube',
         canHandle(url) {
@@ -39,8 +47,8 @@ export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetail
             }
             try {
                 const [subtitles, details] = await Promise.all([
-                    fetchSubtitles({ videoID, lang: 'en' }),
-                    fetchDetails({ videoID, lang: 'en' }).catch(() => ({ title: undefined }))
+                    doFetchSubtitles({ videoID, lang: 'en' }),
+                    doFetchDetails({ videoID, lang: 'en' }).catch(() => ({ title: undefined }))
                 ]);
                 if (!subtitles || subtitles.length === 0) {
                     return {

package/dist/search/brave.d.ts

@@ -1,7 +1,6 @@
-import type { WebSearchResponse } from '../types.js';
 export declare function createBraveSearchTool({ apiKey, fetchImpl }: {
     apiKey?: string;
     fetchImpl?: typeof fetch;
 }): ({ query }: {
     query: string;
-}) => Promise<WebSearchResponse>;
+}) => Promise<import("../types.js").WebSearchResponse>;

package/dist/search/brave.js

@@ -1,83 +1,26 @@
-import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
 const BRAVE_WEB_SEARCH_URL = 'https://api.search.brave.com/res/v1/web/search';
-function resultWithPresentation(result) {
-    return { ...result, presentation: buildSearchPresentation(result) };
-}
-function normalizeResults(response) {
-    return (response.web?.results ?? []).flatMap((item) => {
-        if (typeof item.title !== 'string' || typeof item.url !== 'string') {
-            return [];
-        }
-        return [
-            {
-                title: item.title,
-                url: item.url,
-                snippet: typeof item.description === 'string' ? item.description : ''
-            }
-        ];
-    });
-}
-function buildBraveUrl(query) {
-    const url = new URL(BRAVE_WEB_SEARCH_URL);
-    url.searchParams.set('q', query);
-    return url.toString();
-}
 export function createBraveSearchTool({ apiKey, fetchImpl = fetch }) {
-    return async function braveSearch({ query }) {
-        const normalizedQuery = query.trim();
-        if (!normalizedQuery) {
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'brave', cacheHit: false },
-                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
-            });
-        }
-        if (!apiKey?.trim()) {
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'brave', cacheHit: false },
-                error: {
-                    code: 'BACKEND_CONFIG_INVALID',
-                    message: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.'
-                }
-            });
-        }
-        try {
-            const response = await fetchImpl(buildBraveUrl(normalizedQuery), {
-                headers: {
-                    Accept: 'application/json',
-                    'X-Subscription-Token': apiKey
-                }
-            });
-            if (!response.ok) {
-                throw new Error(`HTTP ${response.status}`);
-            }
-            const parsed = (await response.json());
-            const results = normalizeResults(parsed);
-            if (results.length === 0) {
-                return resultWithPresentation({
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'brave', cacheHit: false },
-                    error: { code: 'NO_RESULTS', message: 'Brave returned no usable results for this query.' }
-                });
-            }
-            return resultWithPresentation({
-                status: 'ok',
-                results,
-                metadata: { backend: 'brave', cacheHit: false }
-            });
-        }
-        catch (error) {
-            const rawMessage = error instanceof Error ? error.message : String(error);
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'brave', cacheHit: false },
-                error: { code: 'FETCH_FAILED', message: `Brave search request failed: ${rawMessage}` }
-            });
-        }
-    };
+    return createJsonSearchProvider({
+        name: 'brave',
+        label: 'Brave',
+        apiKey,
+        missingKeyMessage: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.',
+        fetchImpl,
+        request: (query) => {
+            const url = new URL(BRAVE_WEB_SEARCH_URL);
+            url.searchParams.set('q', query);
+            return { url: url.toString(), init: { headers: { Accept: 'application/json', 'X-Subscription-Token': apiKey ?? '' } } };
+        },
+        // A search response (`type: 'search'`) with no `web` block, or a `web` block with no `results`,
+        // is Brave's valid empty response. Anything else without `web` (an error-shaped or partial 200)
+        // is not trusted as empty.
+        normalize: (json) => normalizeResultsArray(json, (body) => {
+            if (body.web === undefined)
+                return body.type === 'search' ? [] : undefined;
+            if (!body.web || typeof body.web !== 'object')
+                return undefined;
+            return body.web.results === undefined ? [] : body.web.results;
+        }, 'description')
+    });
 }

package/dist/search/duckduckgo.d.ts

@@ -10,9 +10,13 @@ export declare const DUCKDUCKGO_HEADERS: {
     readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
     readonly 'Accept-Language': "en-US,en;q=0.9";
 };
-export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl, retries, sleep }?: {
+export declare class DuckDuckGoHttpError extends Error {
+    readonly status: number;
+    readonly headers: Headers;
+    constructor(status: number, headers: Headers);
+}
+/** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
+export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl }?: {
     fetchImpl?: typeof fetch;
-    retries?: number;
-    sleep?: (ms: number) => Promise<void>;
 }): Promise<string>;
 export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;

package/dist/search/duckduckgo.js

@@ -26,25 +26,24 @@ export const DUCKDUCKGO_HEADERS = {
     Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Language': 'en-US,en;q=0.9'
 };
-const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch, retries = 1, sleep = defaultSleep } = {}) {
-    let lastError;
-    for (let attempt = 0; attempt <= retries; attempt += 1) {
-        try {
-            const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
-            if (!response.ok) {
-                throw new Error(`DuckDuckGo request failed with ${response.status}`);
-            }
-            return response.text();
-        }
-        catch (error) {
-            lastError = error;
-            if (attempt < retries) {
-                await sleep(500);
-            }
-        }
+export class DuckDuckGoHttpError extends Error {
+    status;
+    headers;
+    constructor(status, headers) {
+        super(`DuckDuckGo request failed with ${status}`);
+        this.status = status;
+        this.headers = headers;
+        this.name = 'DuckDuckGoHttpError';
+    }
+}
+/** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
+export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch } = {}) {
+    const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
+    if (!response.ok) {
+        await response.body?.cancel().catch(() => undefined);
+        throw new DuckDuckGoHttpError(response.status, response.headers);
     }
-    throw lastError instanceof Error ? lastError : new Error('DuckDuckGo request failed');
+    return response.text();
 }
 export function parseDuckDuckGoResults(html) {
     const $ = cheerio.load(html);

package/dist/search/exa.d.ts

@@ -1,7 +1,6 @@
-import type { WebSearchResponse } from '../types.js';
 export declare function createExaSearchTool({ apiKey, fetchImpl }: {
     apiKey?: string;
     fetchImpl?: typeof fetch;
 }): ({ query }: {
     query: string;
-}) => Promise<WebSearchResponse>;
+}) => Promise<import("../types.js").WebSearchResponse>;

package/dist/search/exa.js

@@ -1,81 +1,20 @@
-import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
 const EXA_SEARCH_URL = 'https://api.exa.ai/search';
-function resultWithPresentation(result) {
-    return { ...result, presentation: buildSearchPresentation(result) };
-}
-function normalizeResults(response) {
-    return (response.results ?? []).flatMap((item) => {
-        if (typeof item.title !== 'string' || typeof item.url !== 'string') {
-            return [];
-        }
-        return [
-            {
-                title: item.title,
-                url: item.url,
-                snippet: typeof item.text === 'string' ? item.text : ''
-            }
-        ];
-    });
-}
 export function createExaSearchTool({ apiKey, fetchImpl = fetch }) {
-    return async function exaSearch({ query }) {
-        const normalizedQuery = query.trim();
-        if (!normalizedQuery) {
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'exa', cacheHit: false },
-                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
-            });
-        }
-        if (!apiKey?.trim()) {
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'exa', cacheHit: false },
-                error: {
-                    code: 'BACKEND_CONFIG_INVALID',
-                    message: 'Exa search requires EXA_API_KEY.'
-                }
-            });
-        }
-        try {
-            const response = await fetchImpl(EXA_SEARCH_URL, {
+    return createJsonSearchProvider({
+        name: 'exa',
+        label: 'Exa',
+        apiKey,
+        missingKeyMessage: 'Exa search requires EXA_API_KEY.',
+        fetchImpl,
+        request: (query) => ({
+            url: EXA_SEARCH_URL,
+            init: {
                 method: 'POST',
-                headers: {
-                    Accept: 'application/json',
-                    'Content-Type': 'application/json',
-                    'x-api-key': apiKey
-                },
-                body: JSON.stringify({ query: normalizedQuery, numResults: 10 })
-            });
-            if (!response.ok) {
-                throw new Error(`HTTP ${response.status}`);
-            }
-            const parsed = (await response.json());
-            const results = normalizeResults(parsed);
-            if (results.length === 0) {
-                return resultWithPresentation({
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'exa', cacheHit: false },
-                    error: { code: 'NO_RESULTS', message: 'Exa returned no usable results for this query.' }
-                });
+                headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'x-api-key': apiKey ?? '' },
+                body: JSON.stringify({ query, numResults: 10 })
             }
-            return resultWithPresentation({
-                status: 'ok',
-                results,
-                metadata: { backend: 'exa', cacheHit: false }
-            });
-        }
-        catch (error) {
-            const rawMessage = error instanceof Error ? error.message : String(error);
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'exa', cacheHit: false },
-                error: { code: 'FETCH_FAILED', message: `Exa search request failed: ${rawMessage}` }
-            });
-        }
-    };
+        }),
+        normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'text')
+    });
 }

package/dist/search/fanout.d.ts

@@ -5,10 +5,22 @@ export type FanoutProvider = {
         query: string;
     }) => Promise<WebSearchResponse>;
 };
+export declare const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
+/** Longest a policy-wrapped provider can legitimately take: two timed-out calls and the retry sleep. */
+export declare function fanoutBackstopMs(timeoutMs: number): number;
+type SearchFn = FanoutProvider['search'];
+/**
+ * Per-call timeout. Wrap it INSIDE the retry policy so a stalled call is a transient
+ * failure the policy can retry once (#55).
+ */
+export declare function withCallTimeout(search: SearchFn, timeoutMs: number, name: SearchProviderName): SearchFn;
 export declare function createFanoutSearch({ providers, mode, timeoutMs }: {
     providers: FanoutProvider[];
     mode: Exclude<FanoutMode, 'off'>;
+    /** Per-call timeout the providers apply themselves (see withCallTimeout). Fanout only keeps a
+     *  backstop that can't cut a retry short. */
     timeoutMs?: number;
 }): ({ query }: {
     query: string;
 }) => Promise<WebSearchResponse>;
+export {};