@demigodmode/pi-web-agent 1.10.0 → 1.12.0

package/dist/tools/web-search.js

@@ -1,25 +1,9 @@
 import { createCacheKey, createTtlCache } from '../cache/ttl-cache.js';
 import { buildSearchPresentation } from '../presentation/search-presentation.js';
-import { fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
-function classifySearchFailure(error) {
-    const rawMessage = error instanceof Error ? error.message : 'Unknown search failure.';
-    const normalized = rawMessage.toLowerCase();
-    if (normalized.includes('blocked') ||
-        normalized.includes('rate limit') ||
-        normalized.includes('rate-limit') ||
-        normalized.includes('403') ||
-        normalized.includes('429') ||
-        normalized.includes('captcha') ||
-        normalized.includes('challenge')) {
-        return {
-            code: 'BLOCKED',
-            message: 'DuckDuckGo search appears to be blocked or rate limited.'
-        };
-    }
-    return {
-        code: 'FETCH_FAILED',
-        message: `DuckDuckGo search request failed: ${rawMessage}`
-    };
+import { classifyHttpFailure } from '../backends/provider-failure.js';
+import { DuckDuckGoHttpError, fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
+function respond(result) {
+    return { ...result, presentation: buildSearchPresentation(result) };
 }
 function htmlLooksBlocked(html) {
     const normalized = html.toLowerCase();
@@ -37,107 +21,61 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
     return async function webSearch({ query }) {
         const normalizedQuery = query.trim();
         if (!normalizedQuery) {
-            const result = {
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
-                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.', failure: { kind: 'bad_request' } }
+            });
         }
         const cacheKey = createCacheKey(['web_search', normalizedQuery]);
         const cached = cache.get(cacheKey);
         if (cached) {
-            const result = {
-                ...cached,
-                metadata: { ...cached.metadata, cacheHit: true }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+            return respond({ ...cached, metadata: { ...cached.metadata, cacheHit: true } });
         }
+        let html;
         try {
-            let html = await searchHtml(normalizedQuery);
-            let parsed = parseDuckDuckGoResults(html);
-            // A 200-OK bot-wall reads as a successful fetch, so the fetch-layer retry never sees it.
-            // Give a page that looks blocked one more shot here before we classify it.
-            if (parsed.results.length === 0 && htmlLooksBlocked(html)) {
-                html = await searchHtml(normalizedQuery);
-                parsed = parseDuckDuckGoResults(html);
-            }
-            if (parsed.results.length > 0) {
-                const result = {
-                    status: 'ok',
-                    results: parsed.results,
-                    metadata: { backend: 'duckduckgo', cacheHit: false }
-                };
-                cache.set(cacheKey, result);
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            // Check for a bot-wall before "no results": a page can carry both markers, and BLOCKED is
-            // the honest call since it routes to the fallback instead of a dead end.
-            if (htmlLooksBlocked(html)) {
-                const result = {
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'duckduckgo', cacheHit: false },
-                    error: {
-                        code: 'BLOCKED',
-                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
-                    }
-                };
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            if (parsed.noResults) {
-                const result = {
-                    status: 'error',
-                    results: [],
-                    metadata: { backend: 'duckduckgo', cacheHit: false },
-                    error: {
-                        code: 'NO_RESULTS',
-                        message: 'DuckDuckGo returned no usable results for this query.'
-                    }
-                };
-                return {
-                    ...result,
-                    presentation: buildSearchPresentation(result)
-                };
-            }
-            const result = {
+            html = await searchHtml(normalizedQuery);
+        }
+        catch (error) {
+            const failure = error instanceof DuckDuckGoHttpError
+                ? classifyHttpFailure('duckduckgo', { status: error.status, headers: error.headers })
+                : { kind: 'transient' };
+            const blockedLike = failure.kind === 'blocked' || failure.kind === 'rate_limited';
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
                 error: {
-                    code: 'PARSE_FAILED',
-                    message: 'DuckDuckGo returned a page, but it did not match the expected results format.'
+                    code: blockedLike ? 'BLOCKED' : 'FETCH_FAILED',
+                    message: blockedLike
+                        ? 'DuckDuckGo search appears to be blocked or rate limited.'
+                        : `DuckDuckGo search request failed: ${error instanceof Error ? error.message : String(error)}`,
+                    failure
                 }
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+            });
         }
-        catch (error) {
-            const result = {
+        const parsed = parseDuckDuckGoResults(html);
+        if (parsed.results.length > 0) {
+            const result = { status: 'ok', results: parsed.results, metadata: { backend: 'duckduckgo', cacheHit: false } };
+            cache.set(cacheKey, result);
+            return respond(result);
+        }
+        // Bot-wall check first: a page can carry both markers, and blocked routes to fallback.
+        const walled = htmlLooksBlocked(html) || (!parsed.hasResultContainers && !parsed.noResults);
+        if (walled) {
+            return respond({
                 status: 'error',
                 results: [],
                 metadata: { backend: 'duckduckgo', cacheHit: false },
-                error: classifySearchFailure(error)
-            };
-            return {
-                ...result,
-                presentation: buildSearchPresentation(result)
-            };
+                error: {
+                    code: 'BLOCKED',
+                    message: 'DuckDuckGo search appears to be blocked or rate limited.',
+                    failure: { kind: 'blocked' }
+                }
+            });
         }
+        // DuckDuckGo said there are no results, or every result was filtered out: a valid empty search.
+        return respond({ status: 'ok', results: [], metadata: { backend: 'duckduckgo', cacheHit: false } });
     };
 }

package/dist/types.d.ts

@@ -12,10 +12,44 @@ export type FanoutMetadata = {
     mode: Exclude<FanoutMode, 'off'>;
     providers: SearchProviderName[];
     skipped?: SearchProviderName[];
+    outcomes?: FanoutOutcome[];
+};
+export type FailureKind = 'rate_limited' | 'quota_exhausted' | 'auth_failed' | 'not_configured' | 'transient' | 'blocked' | 'bad_response' | 'bad_request' | 'config_global' | 'guard_refused';
+export type FailureInfo = {
+    kind: FailureKind;
+    httpStatus?: number;
+    /** Documented provider body code or tag, e.g. Exa `NO_MORE_CREDITS`. */
+    providerCode?: string;
+    /** Exactly what the provider reported, uncapped. */
+    providerRetryAfterMs?: number;
+};
+export type Attempt = {
+    backend: string;
+    outcome: 'results' | 'empty' | 'failed' | 'skipped' | 'retried';
+    failure?: FailureInfo;
+    skipReason?: 'cooling_down' | 'disabled';
+    cooldownUntil?: number;
+    /** The provider's own message for a user-fixable failure, e.g. a missing key or base URL. */
+    detail?: string;
+};
+export type SearchCoverage = {
+    partial: true;
+    unavailable: Array<{
+        provider: string;
+        kind: FailureKind;
+    }>;
+};
+export type FanoutOutcome = {
+    provider: SearchProviderName;
+    outcome: 'results' | 'empty' | 'failed' | 'skipped';
+    count?: number;
+    failure?: FailureInfo;
+    skipReason?: 'cooling_down' | 'disabled';
 };
 export type ToolError = {
     code: string;
     message: string;
+    failure?: FailureInfo;
 };
 export type SearchMetadata = {
     backend: 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
@@ -23,6 +57,8 @@ export type SearchMetadata = {
     fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily' | 'duckduckgo';
     fallbackReason?: string;
     fanout?: FanoutMetadata;
+    attempts?: Attempt[];
+    coverage?: SearchCoverage;
 };
 export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type FetchMetadata = {
@@ -34,6 +70,9 @@ export type FetchMetadata = {
     truncated?: boolean;
     browser?: 'configured' | 'chrome' | 'edge' | 'brave' | 'chromium';
     navigationMs?: number;
+    /** Headless only: browser requests refused by the private-address guard (#53). */
+    blockedSubresources?: number;
+    attempts?: Attempt[];
 };
 export type ExtractedContent = {
     title?: string;
@@ -80,6 +119,7 @@ export type WebExploreResponse = {
         caveatReasons?: string[];
         fanoutProviders?: SearchProviderName[];
         fanoutSkipped?: SearchProviderName[];
+        attempts?: Attempt[];
     };
     presentation?: PresentationEnvelope;
     error?: ToolError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.10.0",
+  "version": "1.12.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",
@@ -68,6 +68,7 @@
     "jsdom": "^26.0.0",
     "playwright": "^1.60.0",
     "typebox": "^1.1.37",
+    "undici": "^8.10.2",
     "unpdf": "^1.8.1",
     "youtube-caption-extractor": "^1.10.2"
   },
@@ -76,10 +77,10 @@
     "@earendil-works/pi-tui": "^0.80.10",
     "@types/jsdom": "^21.1.7",
     "@types/node": "^24.0.0",
-    "@vitest/coverage-v8": "^3.2.6",
+    "@vitest/coverage-v8": "^4.1.11",
     "typescript": "^5.8.0",
     "vitepress": "^1.6.4",
-    "vitest": "^3.2.6"
+    "vitest": "^4.1.11"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",

package/scripts/patch-jiti-compat.mjs

@@ -10,19 +10,63 @@
 // when this package is installed as a pi extension via `pi install npm:...`.
 // Safe to run multiple times and safe to no-op if the target files or
 // patterns are missing (e.g. a future dependency bump changes the shape).
-import { readFileSync, writeFileSync, existsSync } from "node:fs";
+//
+// src/jiti-compat.ts does the same two patches at extension load time, because
+// the shared ~/.pi/agent/npm tree means another extension's install can revert
+// them long after this hook ran (#34). The logic is deliberately duplicated
+// rather than shared: postinstall runs before `npm run build` in CI and in a
+// fresh clone, so this file cannot depend on dist/. Keep the two in sync.
+import { readFileSync, writeFileSync, existsSync, renameSync, unlinkSync } from "node:fs";
 import { createRequire } from "node:module";
 import { dirname, join, relative, sep } from "node:path";
 
-const resolveFromHere = createRequire(import.meta.url).resolve;
+const requireFromHere = createRequire(import.meta.url);
+const resolveFromHere = requireFromHere.resolve;
+
+// Resolve the copies jsdom actually loads. The shared ~/.pi/agent/npm tree
+// often holds a second, nested copy of a package when extensions disagree on
+// versions, and patching a hoisted copy jsdom never requires would fix
+// nothing. Mirrors resolveFromJsdom in src/jiti-compat.ts.
+function resolveFromJsdom(specifier, via) {
+  try {
+    const jsdomEntry = resolveFromHere("jsdom");
+    const importer = via ? createRequire(jsdomEntry).resolve(via) : jsdomEntry;
+    return createRequire(importer).resolve(specifier);
+  } catch {
+    return resolveFromHere(specifier);
+  }
+}
+
+// Temp file + rename, not an in-place write. `writeFileSync` truncates first,
+// so an interrupted write would leave a half-patched file that still contains
+// the marker comment and would be skipped as "already patched" forever.
+// Mirrors writeFileAtomic in src/jiti-compat.ts.
+function writeFileAtomic(path, contents) {
+  const temp = `${path}.pi-web-agent-${process.pid}.tmp`;
+  try {
+    writeFileSync(temp, contents);
+    renameSync(temp, path);
+  } catch (err) {
+    try {
+      if (existsSync(temp)) unlinkSync(temp);
+    } catch {
+      // Best effort.
+    }
+    throw err;
+  }
+}
 
 function findPackageRoot(entryFile, packageName) {
   let directory = dirname(entryFile);
   while (true) {
     const manifestFile = join(directory, "package.json");
     if (existsSync(manifestFile)) {
-      const manifest = JSON.parse(readFileSync(manifestFile, "utf8"));
-      if (manifest.name === packageName) return directory;
+      try {
+        const manifest = JSON.parse(readFileSync(manifestFile, "utf8"));
+        if (manifest.name === packageName) return directory;
+      } catch {
+        // Malformed package.json on the way up. Keep walking.
+      }
     }
 
     const parent = dirname(directory);
@@ -34,7 +78,7 @@ function findPackageRoot(entryFile, packageName) {
 }
 
 function patchTr46() {
-  const file = resolveFromHere("tr46");
+  const file = resolveFromJsdom("tr46", "whatwg-url");
   if (!existsSync(file)) {
     console.debug("patch-jiti-compat: tr46/index.js not found, skipping");
     return;
@@ -44,7 +88,7 @@ function patchTr46() {
     console.debug("patch-jiti-compat: tr46/index.js does not match expected pattern, skipping");
     return;
   }
-  writeFileSync(
+  writeFileAtomic(
     file,
     contents.replaceAll('require("punycode/")', 'require("punycode/punycode.js")'),
   );
@@ -61,7 +105,7 @@ module.exports[Symbol.iterator] = Set.prototype[Symbol.iterator].bind(module.exp
 `;
 
 function patchCssstyleSetExports() {
-  const packageRoot = findPackageRoot(resolveFromHere("cssstyle"), "cssstyle");
+  const packageRoot = findPackageRoot(resolveFromJsdom("cssstyle"), "cssstyle");
   const files = [
     join(packageRoot, "lib", "allExtraProperties.js"),
     join(packageRoot, "lib", "generated", "allProperties.js"),
@@ -79,7 +123,7 @@ function patchCssstyleSetExports() {
       console.debug(`patch-jiti-compat: ${label} does not match expected pattern, skipping`);
       continue;
     }
-    writeFileSync(file, contents + SET_SHIM);
+    writeFileAtomic(file, contents + SET_SHIM);
     console.log(`patch-jiti-compat: patched ${label}`);
   }
 }