@demigodmode/pi-web-agent 1.11.0 → 1.12.0

package/dist/search/youcom.js

@@ -1,81 +1,20 @@
-import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
 const YOUCOM_SEARCH_URL = 'https://api.you.com/v1/agents/search';
-function resultWithPresentation(result) {
-    return { ...result, presentation: buildSearchPresentation(result) };
-}
-function normalizeResults(response) {
-    return (response.results ?? []).flatMap((item) => {
-        if (typeof item.title !== 'string' || typeof item.url !== 'string') {
-            return [];
-        }
-        return [
-            {
-                title: item.title,
-                url: item.url,
-                snippet: typeof item.snippet === 'string' ? item.snippet : ''
-            }
-        ];
-    });
-}
 export function createYouComSearchTool({ apiKey, fetchImpl = fetch }) {
-    return async function youComSearch({ query }) {
-        const normalizedQuery = query.trim();
-        if (!normalizedQuery) {
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'youcom', cacheHit: false },
-                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
-            });
-        }
-        if (!apiKey?.trim()) {
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'youcom', cacheHit: false },
-                error: {
-                    code: 'BACKEND_CONFIG_INVALID',
-                    message: 'You.com search requires YDC_API_KEY.'
-                }
-            });
-        }
-        try {
-            const response = await fetchImpl(YOUCOM_SEARCH_URL, {
+    return createJsonSearchProvider({
+        name: 'youcom',
+        label: 'You.com',
+        apiKey,
+        missingKeyMessage: 'You.com search requires YDC_API_KEY.',
+        fetchImpl,
+        request: (query) => ({
+            url: YOUCOM_SEARCH_URL,
+            init: {
                 method: 'POST',
-                headers: {
-                    Accept: 'application/json',
-                    'Content-Type': 'application/json',
-                    'X-API-Key': apiKey
-                },
-                body: JSON.stringify({ query: normalizedQuery, max_results: 10 })
-            });
-            if (!response.ok) {
-                throw new Error(`HTTP ${response.status}`);
-            }
-            const parsed = (await response.json());
-            const results = normalizeResults(parsed);
-            if (results.length === 0) {
-                return resultWithPresentation({
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'youcom', cacheHit: false },
-                    error: { code: 'NO_RESULTS', message: 'You.com returned no usable results for this query.' }
-                });
+                headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'X-API-Key': apiKey ?? '' },
+                body: JSON.stringify({ query, max_results: 10 })
             }
-            return resultWithPresentation({
-                status: 'ok',
-                results,
-                metadata: { backend: 'youcom', cacheHit: false }
-            });
-        }
-        catch (error) {
-            const rawMessage = error instanceof Error ? error.message : String(error);
-            return resultWithPresentation({
-                status: 'error',
-                results: [],
-                metadata: { backend: 'youcom', cacheHit: false },
-                error: { code: 'FETCH_FAILED', message: `You.com search request failed: ${rawMessage}` }
-            });
-        }
-    };
+        }),
+        normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'snippet')
+    });
 }

package/dist/tools/web-explore.d.ts

@@ -11,6 +11,10 @@ export declare function createWebExploreTool({ explore }?: {
             evidence: ResearchEvidence[];
             workerPass: unknown;
             metadata?: WebExploreResponse['metadata'];
+            terminalFailure?: {
+                code: string;
+                message: string;
+            };
         }>;
     } | ((input: {
         query: string;
@@ -21,6 +25,10 @@ export declare function createWebExploreTool({ explore }?: {
         evidence: ResearchEvidence[];
         workerPass: unknown;
         metadata?: WebExploreResponse['metadata'];
+        terminalFailure?: {
+            code: string;
+            message: string;
+        };
     }>);
 }): ({ query }: {
     query: string;
@@ -42,6 +50,7 @@ export declare function createWebExploreTool({ explore }?: {
         caveatReasons?: string[];
         fanoutProviders?: import("../types.js").SearchProviderName[];
         fanoutSkipped?: import("../types.js").SearchProviderName[];
+        attempts?: import("../types.js").Attempt[];
     };
     error?: import("../types.js").ToolError;
 }>;

package/dist/tools/web-explore.js

@@ -18,15 +18,29 @@ export function createWebExploreTool({ explore = createResearchWorkflow() } = {}
             };
         }
         const result = await runExplore({ query: normalizedQuery });
+        if (result.terminalFailure) {
+            const failed = {
+                status: 'error',
+                findings: [],
+                sources: [],
+                error: result.terminalFailure,
+                metadata: result.metadata
+            };
+            return { ...failed, presentation: buildExplorePresentation(failed) };
+        }
         const sources = result.evidence.slice(0, 4).map((item) => ({
             title: item.title,
             url: item.url,
             method: item.method
         }));
+        const reasons = (result.metadata?.caveatReasons ?? []);
+        const decisionPartial = result.decision.action !== 'answer';
+        const coveragePartial = reasons.includes('partial-search-coverage');
         const synthesized = synthesizeAnswer({
             evidence: result.evidence,
-            partial: result.decision.action !== 'answer',
-            caveatReasons: result.metadata?.caveatReasons
+            partial: decisionPartial || coveragePartial,
+            // A confident answer with partial coverage mentions only the coverage, not unrelated quality notes.
+            caveatReasons: decisionPartial ? reasons : ['partial-search-coverage']
         });
         const shaped = {
             status: 'ok',

package/dist/tools/web-search.js

@@ -1,25 +1,9 @@
 import { createCacheKey, createTtlCache } from '../cache/ttl-cache.js';
 import { buildSearchPresentation } from '../presentation/search-presentation.js';
-import { fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
-function classifySearchFailure(error) {
-    const rawMessage = error instanceof Error ? error.message : 'Unknown search failure.';
-    const normalized = rawMessage.toLowerCase();
-    if (normalized.includes('blocked') ||
-        normalized.includes('rate limit') ||
-        normalized.includes('rate-limit') ||
-        normalized.includes('403') ||
-        normalized.includes('429') ||
-        normalized.includes('captcha') ||
-        normalized.includes('challenge')) {
-        return {
-            code: 'BLOCKED',
-            message: 'DuckDuckGo search appears to be blocked or rate limited.'
-        };
-    }
-    return {
-        code: 'FETCH_FAILED',
-        message: `DuckDuckGo search request failed: ${rawMessage}`
-    };
+import { classifyHttpFailure } from '../backends/provider-failure.js';
+import { DuckDuckGoHttpError, fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
+function respond(result) {
+    return { ...result, presentation: buildSearchPresentation(result) };
 }
 function htmlLooksBlocked(html) {
     const normalized = html.toLowerCase();
@@ -37,107 +21,61 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
     return async function webSearch({ query }) {
         const normalizedQuery = query.trim();
         if (!normalizedQuery) {
-            const result = {
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
-                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.', failure: { kind: 'bad_request' } }
+            });
         }
         const cacheKey = createCacheKey(['web_search', normalizedQuery]);
         const cached = cache.get(cacheKey);
         if (cached) {
-            const result = {
-                ...cached,
-                metadata: { ...cached.metadata, cacheHit: true }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+            return respond({ ...cached, metadata: { ...cached.metadata, cacheHit: true } });
         }
+        let html;
         try {
-            let html = await searchHtml(normalizedQuery);
-            let parsed = parseDuckDuckGoResults(html);
-            // A 200-OK bot-wall reads as a successful fetch, so the fetch-layer retry never sees it.
-            // Give a page that looks blocked one more shot here before we classify it.
-            if (parsed.results.length === 0 && htmlLooksBlocked(html)) {
-                html = await searchHtml(normalizedQuery);
-                parsed = parseDuckDuckGoResults(html);
-            }
-            if (parsed.results.length > 0) {
-                const result = {
-                    status: 'ok',
-                    results: parsed.results,
-                    metadata: { backend: 'duckduckgo', cacheHit: false }
-                };
-                cache.set(cacheKey, result);
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            // Check for a bot-wall before "no results": a page can carry both markers, and BLOCKED is
-            // the honest call since it routes to the fallback instead of a dead end.
-            if (htmlLooksBlocked(html)) {
-                const result = {
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'duckduckgo', cacheHit: false },
-                    error: {
-                        code: 'BLOCKED',
-                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
-                    }
-                };
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            if (parsed.noResults) {
-                const result = {
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'duckduckgo', cacheHit: false },
-                    error: {
-                        code: 'NO_RESULTS',
-                        message: 'DuckDuckGo returned no usable results for this query.'
-                    }
-                };
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            const result = {
+            html = await searchHtml(normalizedQuery);
+        }
+        catch (error) {
+            const failure = error instanceof DuckDuckGoHttpError
+                ? classifyHttpFailure('duckduckgo', { status: error.status, headers: error.headers })
+                : { kind: 'transient' };
+            const blockedLike = failure.kind === 'blocked' || failure.kind === 'rate_limited';
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
                 error: {
-                    code: 'PARSE_FAILED',
-                    message: 'DuckDuckGo returned a page, but it did not match the expected results format.'
+                    code: blockedLike ? 'BLOCKED' : 'FETCH_FAILED',
+                    message: blockedLike
+                        ? 'DuckDuckGo search appears to be blocked or rate limited.'
+                        : `DuckDuckGo search request failed: ${error instanceof Error ? error.message : String(error)}`,
+                    failure
                 }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+            });
         }
-        catch (error) {
-            const result = {
+        const parsed = parseDuckDuckGoResults(html);
+        if (parsed.results.length > 0) {
+            const result = { status: 'ok', results: parsed.results, metadata: { backend: 'duckduckgo', cacheHit: false } };
+            cache.set(cacheKey, result);
+            return respond(result);
+        }
+        // Bot-wall check first: a page can carry both markers, and blocked routes to fallback.
+        const walled = htmlLooksBlocked(html) || (!parsed.hasResultContainers && !parsed.noResults);
+        if (walled) {
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
-                error: classifySearchFailure(error)
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+                error: {
+                    code: 'BLOCKED',
+                    message: 'DuckDuckGo search appears to be blocked or rate limited.',
+                    failure: { kind: 'blocked' }
+                }
+            });
         }
+        // DuckDuckGo said there are no results, or every result was filtered out: a valid empty search.
+        return respond({ status: 'ok', results: [], metadata: { backend: 'duckduckgo', cacheHit: false } });
     };
 }

package/dist/types.d.ts

@@ -12,10 +12,44 @@ export type FanoutMetadata = {
     mode: Exclude<FanoutMode, 'off'>;
     providers: SearchProviderName[];
     skipped?: SearchProviderName[];
+    outcomes?: FanoutOutcome[];
+};
+export type FailureKind = 'rate_limited' | 'quota_exhausted' | 'auth_failed' | 'not_configured' | 'transient' | 'blocked' | 'bad_response' | 'bad_request' | 'config_global' | 'guard_refused';
+export type FailureInfo = {
+    kind: FailureKind;
+    httpStatus?: number;
+    /** Documented provider body code or tag, e.g. Exa `NO_MORE_CREDITS`. */
+    providerCode?: string;
+    /** Exactly what the provider reported, uncapped. */
+    providerRetryAfterMs?: number;
+};
+export type Attempt = {
+    backend: string;
+    outcome: 'results' | 'empty' | 'failed' | 'skipped' | 'retried';
+    failure?: FailureInfo;
+    skipReason?: 'cooling_down' | 'disabled';
+    cooldownUntil?: number;
+    /** The provider's own message for a user-fixable failure, e.g. a missing key or base URL. */
+    detail?: string;
+};
+export type SearchCoverage = {
+    partial: true;
+    unavailable: Array<{
+        provider: string;
+        kind: FailureKind;
+    }>;
+};
+export type FanoutOutcome = {
+    provider: SearchProviderName;
+    outcome: 'results' | 'empty' | 'failed' | 'skipped';
+    count?: number;
+    failure?: FailureInfo;
+    skipReason?: 'cooling_down' | 'disabled';
 };
 export type ToolError = {
     code: string;
     message: string;
+    failure?: FailureInfo;
 };
 export type SearchMetadata = {
     backend: 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
@@ -23,6 +57,8 @@ export type SearchMetadata = {
     fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily' | 'duckduckgo';
     fallbackReason?: string;
     fanout?: FanoutMetadata;
+    attempts?: Attempt[];
+    coverage?: SearchCoverage;
 };
 export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type FetchMetadata = {
@@ -34,6 +70,9 @@ export type FetchMetadata = {
     truncated?: boolean;
     browser?: 'configured' | 'chrome' | 'edge' | 'brave' | 'chromium';
     navigationMs?: number;
+    /** Headless only: browser requests refused by the private-address guard (#53). */
+    blockedSubresources?: number;
+    attempts?: Attempt[];
 };
 export type ExtractedContent = {
     title?: string;
@@ -80,6 +119,7 @@ export type WebExploreResponse = {
         caveatReasons?: string[];
         fanoutProviders?: SearchProviderName[];
         fanoutSkipped?: SearchProviderName[];
+        attempts?: Attempt[];
     };
     presentation?: PresentationEnvelope;
     error?: ToolError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",
@@ -77,10 +77,10 @@
     "@earendil-works/pi-tui": "^0.80.10",
     "@types/jsdom": "^21.1.7",
     "@types/node": "^24.0.0",
-    "@vitest/coverage-v8": "^3.2.6",
+    "@vitest/coverage-v8": "^4.1.11",
     "typescript": "^5.8.0",
     "vitepress": "^1.6.4",
-    "vitest": "^3.2.6"
+    "vitest": "^4.1.11"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",