@demigodmode/pi-web-agent 1.11.0 → 1.13.0

package/dist/tools/web-explore.d.ts

@@ -11,6 +11,10 @@ export declare function createWebExploreTool({ explore }?: {
             evidence: ResearchEvidence[];
             workerPass: unknown;
             metadata?: WebExploreResponse['metadata'];
+            terminalFailure?: {
+                code: string;
+                message: string;
+            };
         }>;
     } | ((input: {
         query: string;
@@ -21,6 +25,10 @@ export declare function createWebExploreTool({ explore }?: {
         evidence: ResearchEvidence[];
         workerPass: unknown;
         metadata?: WebExploreResponse['metadata'];
+        terminalFailure?: {
+            code: string;
+            message: string;
+        };
     }>);
 }): ({ query }: {
     query: string;
@@ -42,6 +50,7 @@ export declare function createWebExploreTool({ explore }?: {
         caveatReasons?: string[];
         fanoutProviders?: import("../types.js").SearchProviderName[];
         fanoutSkipped?: import("../types.js").SearchProviderName[];
+        attempts?: import("../types.js").Attempt[];
     };
     error?: import("../types.js").ToolError;
 }>;

package/dist/tools/web-explore.js

@@ -18,15 +18,29 @@ export function createWebExploreTool({ explore = createResearchWorkflow() } = {}
             };
         }
         const result = await runExplore({ query: normalizedQuery });
+        if (result.terminalFailure) {
+            const failed = {
+                status: 'error',
+                findings: [],
+                sources: [],
+                error: result.terminalFailure,
+                metadata: result.metadata
+            };
+            return { ...failed, presentation: buildExplorePresentation(failed) };
+        }
         const sources = result.evidence.slice(0, 4).map((item) => ({
             title: item.title,
             url: item.url,
             method: item.method
         }));
+        const reasons = (result.metadata?.caveatReasons ?? []);
+        const decisionPartial = result.decision.action !== 'answer';
+        const coveragePartial = reasons.includes('partial-search-coverage');
         const synthesized = synthesizeAnswer({
             evidence: result.evidence,
-            partial: result.decision.action !== 'answer',
-            caveatReasons: result.metadata?.caveatReasons
+            partial: decisionPartial || coveragePartial,
+            // A confident answer with partial coverage mentions only the coverage, not unrelated quality notes.
+            caveatReasons: decisionPartial ? reasons : ['partial-search-coverage']
         });
         const shaped = {
             status: 'ok',

package/dist/tools/web-fetch-headless.d.ts

@@ -1,6 +1,4 @@
-import type { WebFetchHeadlessResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchHeadlessResponse } from '../types.js';
 export declare function createWebFetchHeadlessTool({ fetchPage }?: {
-    fetchPage?: (url: string) => Promise<WebFetchHeadlessResponse>;
-}): ({ url }: {
-    url: string;
-}) => Promise<WebFetchHeadlessResponse>;
+    fetchPage?: (input: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;
+}): ({ url, query }: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;

package/dist/tools/web-fetch-headless.js

@@ -1,7 +1,7 @@
 import { headlessFetch } from '../fetch/headless-fetch.js';
 import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
-export function createWebFetchHeadlessTool({ fetchPage = headlessFetch } = {}) {
-    return async function webFetchHeadless({ url }) {
+export function createWebFetchHeadlessTool({ fetchPage = ({ url, query }) => headlessFetch(url, { query }) } = {}) {
+    return async function webFetchHeadless({ url, query }) {
         if (!/^https?:\/\//.test(url)) {
             const result = {
                 status: 'unsupported',
@@ -14,7 +14,7 @@ export function createWebFetchHeadlessTool({ fetchPage = headlessFetch } = {}) {
                 presentation: buildFetchPresentation(result)
             };
         }
-        const result = await fetchPage(url);
+        const result = await fetchPage({ url, ...(query ? { query } : {}) });
         return {
             ...result,
             presentation: buildFetchPresentation(result)

package/dist/tools/web-fetch.d.ts

@@ -1,6 +1,4 @@
-import type { WebFetchResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchResponse } from '../types.js';
 export declare function createWebFetchTool({ fetchPage }?: {
-    fetchPage?: (url: string) => Promise<WebFetchResponse>;
-}): ({ url }: {
-    url: string;
-}) => Promise<WebFetchResponse>;
+    fetchPage?: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
+}): ({ url, query }: ResearchFetchInput) => Promise<WebFetchResponse>;

package/dist/tools/web-fetch.js

@@ -1,7 +1,7 @@
 import { createHttpFetcher } from '../fetch/http-fetch.js';
 import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
-export function createWebFetchTool({ fetchPage = createHttpFetcher() } = {}) {
-    return async function webFetch({ url }) {
+export function createWebFetchTool({ fetchPage = ({ url, query }) => createHttpFetcher()(url, query) } = {}) {
+    return async function webFetch({ url, query }) {
         if (!/^https?:\/\//.test(url)) {
             const result = {
                 status: 'unsupported',
@@ -14,7 +14,7 @@ export function createWebFetchTool({ fetchPage = createHttpFetcher() } = {}) {
                 presentation: buildFetchPresentation(result)
             };
         }
-        const result = await fetchPage(url);
+        const result = await fetchPage({ url, ...(query ? { query } : {}) });
         return {
             ...result,
             presentation: buildFetchPresentation(result)

package/dist/tools/web-search.js

@@ -1,25 +1,9 @@
 import { createCacheKey, createTtlCache } from '../cache/ttl-cache.js';
 import { buildSearchPresentation } from '../presentation/search-presentation.js';
-import { fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
-function classifySearchFailure(error) {
-    const rawMessage = error instanceof Error ? error.message : 'Unknown search failure.';
-    const normalized = rawMessage.toLowerCase();
-    if (normalized.includes('blocked') ||
-        normalized.includes('rate limit') ||
-        normalized.includes('rate-limit') ||
-        normalized.includes('403') ||
-        normalized.includes('429') ||
-        normalized.includes('captcha') ||
-        normalized.includes('challenge')) {
-        return {
-            code: 'BLOCKED',
-            message: 'DuckDuckGo search appears to be blocked or rate limited.'
-        };
-    }
-    return {
-        code: 'FETCH_FAILED',
-        message: `DuckDuckGo search request failed: ${rawMessage}`
-    };
+import { classifyHttpFailure } from '../backends/provider-failure.js';
+import { DuckDuckGoHttpError, fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
+function respond(result) {
+    return { ...result, presentation: buildSearchPresentation(result) };
 }
 function htmlLooksBlocked(html) {
     const normalized = html.toLowerCase();
@@ -37,107 +21,61 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
     return async function webSearch({ query }) {
         const normalizedQuery = query.trim();
         if (!normalizedQuery) {
-            const result = {
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
-                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.', failure: { kind: 'bad_request' } }
+            });
         }
         const cacheKey = createCacheKey(['web_search', normalizedQuery]);
         const cached = cache.get(cacheKey);
         if (cached) {
-            const result = {
-                ...cached,
-                metadata: { ...cached.metadata, cacheHit: true }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+            return respond({ ...cached, metadata: { ...cached.metadata, cacheHit: true } });
         }
+        let html;
         try {
-            let html = await searchHtml(normalizedQuery);
-            let parsed = parseDuckDuckGoResults(html);
-            // A 200-OK bot-wall reads as a successful fetch, so the fetch-layer retry never sees it.
-            // Give a page that looks blocked one more shot here before we classify it.
-            if (parsed.results.length === 0 && htmlLooksBlocked(html)) {
-                html = await searchHtml(normalizedQuery);
-                parsed = parseDuckDuckGoResults(html);
-            }
-            if (parsed.results.length > 0) {
-                const result = {
-                    status: 'ok',
-                    results: parsed.results,
-                    metadata: { backend: 'duckduckgo', cacheHit: false }
-                };
-                cache.set(cacheKey, result);
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            // Check for a bot-wall before "no results": a page can carry both markers, and BLOCKED is
-            // the honest call since it routes to the fallback instead of a dead end.
-            if (htmlLooksBlocked(html)) {
-                const result = {
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'duckduckgo', cacheHit: false },
-                    error: {
-                        code: 'BLOCKED',
-                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
-                    }
-                };
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            if (parsed.noResults) {
-                const result = {
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'duckduckgo', cacheHit: false },
-                    error: {
-                        code: 'NO_RESULTS',
-                        message: 'DuckDuckGo returned no usable results for this query.'
-                    }
-                };
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            const result = {
+            html = await searchHtml(normalizedQuery);
+        }
+        catch (error) {
+            const failure = error instanceof DuckDuckGoHttpError
+                ? classifyHttpFailure('duckduckgo', { status: error.status, headers: error.headers })
+                : { kind: 'transient' };
+            const blockedLike = failure.kind === 'blocked' || failure.kind === 'rate_limited';
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
                 error: {
-                    code: 'PARSE_FAILED',
-                    message: 'DuckDuckGo returned a page, but it did not match the expected results format.'
+                    code: blockedLike ? 'BLOCKED' : 'FETCH_FAILED',
+                    message: blockedLike
+                        ? 'DuckDuckGo search appears to be blocked or rate limited.'
+                        : `DuckDuckGo search request failed: ${error instanceof Error ? error.message : String(error)}`,
+                    failure
                 }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+            });
         }
-        catch (error) {
-            const result = {
+        const parsed = parseDuckDuckGoResults(html);
+        if (parsed.results.length > 0) {
+            const result = { status: 'ok', results: parsed.results, metadata: { backend: 'duckduckgo', cacheHit: false } };
+            cache.set(cacheKey, result);
+            return respond(result);
+        }
+        // Bot-wall check first: a page can carry both markers, and blocked routes to fallback.
+        const walled = htmlLooksBlocked(html) || (!parsed.hasResultContainers && !parsed.noResults);
+        if (walled) {
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
-                error: classifySearchFailure(error)
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+                error: {
+                    code: 'BLOCKED',
+                    message: 'DuckDuckGo search appears to be blocked or rate limited.',
+                    failure: { kind: 'blocked' }
+                }
+            });
         }
+        // DuckDuckGo said there are no results, or every result was filtered out: a valid empty search.
+        return respond({ status: 'ok', results: [], metadata: { backend: 'duckduckgo', cacheHit: false } });
     };
 }

package/dist/types.d.ts

@@ -12,10 +12,44 @@ export type FanoutMetadata = {
     mode: Exclude<FanoutMode, 'off'>;
     providers: SearchProviderName[];
     skipped?: SearchProviderName[];
+    outcomes?: FanoutOutcome[];
+};
+export type FailureKind = 'rate_limited' | 'quota_exhausted' | 'auth_failed' | 'not_configured' | 'transient' | 'blocked' | 'bad_response' | 'bad_request' | 'config_global' | 'guard_refused';
+export type FailureInfo = {
+    kind: FailureKind;
+    httpStatus?: number;
+    /** Documented provider body code or tag, e.g. Exa `NO_MORE_CREDITS`. */
+    providerCode?: string;
+    /** Exactly what the provider reported, uncapped. */
+    providerRetryAfterMs?: number;
+};
+export type Attempt = {
+    backend: string;
+    outcome: 'results' | 'empty' | 'failed' | 'skipped' | 'retried';
+    failure?: FailureInfo;
+    skipReason?: 'cooling_down' | 'disabled';
+    cooldownUntil?: number;
+    /** The provider's own message for a user-fixable failure, e.g. a missing key or base URL. */
+    detail?: string;
+};
+export type SearchCoverage = {
+    partial: true;
+    unavailable: Array<{
+        provider: string;
+        kind: FailureKind;
+    }>;
+};
+export type FanoutOutcome = {
+    provider: SearchProviderName;
+    outcome: 'results' | 'empty' | 'failed' | 'skipped';
+    count?: number;
+    failure?: FailureInfo;
+    skipReason?: 'cooling_down' | 'disabled';
 };
 export type ToolError = {
     code: string;
     message: string;
+    failure?: FailureInfo;
 };
 export type SearchMetadata = {
     backend: 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
@@ -23,8 +57,14 @@ export type SearchMetadata = {
     fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily' | 'duckduckgo';
     fallbackReason?: string;
     fanout?: FanoutMetadata;
+    attempts?: Attempt[];
+    coverage?: SearchCoverage;
 };
 export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
+export type ResearchFetchInput = {
+    url: string;
+    query?: string;
+};
 export type FetchMetadata = {
     method: FetchMethod;
     cacheHit: boolean;
@@ -34,11 +74,18 @@ export type FetchMetadata = {
     truncated?: boolean;
     browser?: 'configured' | 'chrome' | 'edge' | 'brave' | 'chromium';
     navigationMs?: number;
+    /** Headless only: browser requests refused by the private-address guard (#53). */
+    blockedSubresources?: number;
+    attempts?: Attempt[];
 };
 export type ExtractedContent = {
     title?: string;
     byline?: string;
     text: string;
+    /** A bot or security verification marker appeared in the source before query selection. */
+    botCheck?: boolean;
+    /** Anchor of the section chosen for research; direct fetches do not set this. */
+    sectionAnchor?: string;
 };
 export type WebSearchResponse = {
     status: 'ok' | 'error';
@@ -80,6 +127,7 @@ export type WebExploreResponse = {
         caveatReasons?: string[];
         fanoutProviders?: SearchProviderName[];
         fanoutSkipped?: SearchProviderName[];
+        attempts?: Attempt[];
     };
     presentation?: PresentationEnvelope;
     error?: ToolError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.11.0",
+  "version": "1.13.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",
@@ -77,10 +77,10 @@
     "@earendil-works/pi-tui": "^0.80.10",
     "@types/jsdom": "^21.1.7",
     "@types/node": "^24.0.0",
-    "@vitest/coverage-v8": "^3.2.6",
+    "@vitest/coverage-v8": "^4.1.11",
     "typescript": "^5.8.0",
     "vitepress": "^1.6.4",
-    "vitest": "^3.2.6"
+    "vitest": "^4.1.11"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",