@demigodmode/pi-web-agent 1.8.0 → 1.10.0

package/dist/search/fanout.d.ts

@@ -0,0 +1,14 @@
+import type { FanoutMode, SearchProviderName, WebSearchResponse } from '../types.js';
+export type FanoutProvider = {
+    name: SearchProviderName;
+    search: (input: {
+        query: string;
+    }) => Promise<WebSearchResponse>;
+};
+export declare function createFanoutSearch({ providers, mode, timeoutMs }: {
+    providers: FanoutProvider[];
+    mode: Exclude<FanoutMode, 'off'>;
+    timeoutMs?: number;
+}): ({ query }: {
+    query: string;
+}) => Promise<WebSearchResponse>;

package/dist/search/fanout.js

@@ -0,0 +1,118 @@
+import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import { canonicalizeUrl } from '../orchestration/url.js';
+const FANOUT_MIN_RESULTS = 3;
+function hostOf(url) {
+    try {
+        return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+    }
+    catch {
+        return undefined;
+    }
+}
+function primaryLooksWeak(results) {
+    if (results.length < FANOUT_MIN_RESULTS)
+        return true;
+    const hosts = new Set(results.map((r) => hostOf(r.url)).filter(Boolean));
+    return hosts.size <= 1;
+}
+function merge(lists) {
+    const byKey = new Map();
+    for (const { name, results } of lists) {
+        const seenThisProvider = new Set();
+        results.forEach((result, index) => {
+            const key = canonicalizeUrl(result.url) ?? result.url;
+            if (seenThisProvider.has(key))
+                return; // ignore duplicates within a single provider's own list
+            seenThisProvider.add(key);
+            const existing = byKey.get(key);
+            if (!existing) {
+                byKey.set(key, { result: { ...result }, providers: new Set([name]), bestRank: index });
+                return;
+            }
+            existing.providers.add(name);
+            existing.bestRank = Math.min(existing.bestRank, index);
+            if ((result.title?.length ?? 0) > (existing.result.title?.length ?? 0))
+                existing.result.title = result.title;
+            if ((result.snippet?.length ?? 0) > (existing.result.snippet?.length ?? 0))
+                existing.result.snippet = result.snippet;
+        });
+    }
+    return [...byKey.values()]
+        .sort((a, b) => b.providers.size - a.providers.size || a.bestRank - b.bestRank)
+        .map((entry) => entry.result);
+}
+function withPresentation(result) {
+    return { ...result, presentation: buildSearchPresentation(result) };
+}
+const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
+/** Resolve to undefined if the provider doesn't answer in time, so one slow/unreachable
+ *  provider (e.g. a down self-hosted SearXNG) can't stall the whole fanout across passes. */
+function withTimeout(promise, ms) {
+    return new Promise((resolve) => {
+        const timer = setTimeout(() => resolve(undefined), ms);
+        promise.then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, () => {
+            clearTimeout(timer);
+            resolve(undefined);
+        });
+    });
+}
+export function createFanoutSearch({ providers, mode, timeoutMs = FANOUT_PROVIDER_TIMEOUT_MS }) {
+    return async function fanoutSearch({ query }) {
+        const [primary, ...rest] = providers;
+        async function runSet(set) {
+            const settled = await Promise.all(set.map((p) => withTimeout(p.search({ query }), timeoutMs)));
+            const contributing = [];
+            const skipped = [];
+            set.forEach((provider, i) => {
+                const value = settled[i];
+                if (value && value.status === 'ok' && value.results.length > 0) {
+                    contributing.push({ name: provider.name, results: value.results });
+                }
+                else {
+                    skipped.push(provider.name);
+                }
+            });
+            return { contributing, skipped };
+        }
+        function finalize(lists, skipped, resolvedMode) {
+            if (lists.length === 0) {
+                const fanout = { mode: resolvedMode, providers: [] };
+                if (skipped.length > 0)
+                    fanout.skipped = skipped;
+                return withPresentation({
+                    status: 'error',
+                    results: [],
+                    metadata: { backend: primary.name, cacheHit: false, fanout },
+                    error: { code: 'FANOUT_NO_RESULTS', message: 'No fanout provider returned usable results.' }
+                });
+            }
+            const fanout = { mode: resolvedMode, providers: lists.map((l) => l.name) };
+            if (skipped.length > 0)
+                fanout.skipped = skipped;
+            return withPresentation({
+                status: 'ok',
+                results: merge(lists),
+                metadata: { backend: primary.name, cacheHit: false, fanout }
+            });
+        }
+        if (mode === 'auto') {
+            const primaryOutcome = await withTimeout(primary.search({ query }), timeoutMs);
+            const primaryResults = primaryOutcome?.status === 'ok' ? primaryOutcome.results : [];
+            if (primaryResults.length > 0 && !primaryLooksWeak(primaryResults)) {
+                return withPresentation(primaryOutcome); // strong primary: no fanout
+            }
+            const { contributing, skipped } = await runSet(rest);
+            const lists = [
+                ...(primaryResults.length ? [{ name: primary.name, results: primaryResults }] : []),
+                ...contributing
+            ];
+            const allSkipped = [...(primaryResults.length ? [] : [primary.name]), ...skipped];
+            return finalize(lists, allSkipped, 'auto');
+        }
+        const { contributing, skipped } = await runSet(providers);
+        return finalize(contributing, skipped, 'on');
+    };
+}

package/dist/search/tavily.d.ts

@@ -1,6 +1,7 @@
 import type { WebSearchResponse } from '../types.js';
-export declare function createTavilySearchTool({ apiKey, fetchImpl }: {
+export declare function createTavilySearchTool({ apiKey, keyless, fetchImpl }: {
     apiKey?: string;
+    keyless?: boolean;
     fetchImpl?: typeof fetch;
 }): ({ query }: {
     query: string;

package/dist/search/tavily.js

@@ -17,7 +17,7 @@ function normalizeResults(response) {
         ];
     });
 }
-export function createTavilySearchTool({ apiKey, fetchImpl = fetch }) {
+export function createTavilySearchTool({ apiKey, keyless = false, fetchImpl = fetch }) {
     return async function tavilySearch({ query }) {
         const normalizedQuery = query.trim();
         if (!normalizedQuery) {
@@ -28,7 +28,7 @@ export function createTavilySearchTool({ apiKey, fetchImpl = fetch }) {
                 error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
             });
         }
-        if (!apiKey?.trim()) {
+        if (!apiKey?.trim() && !keyless) {
             return resultWithPresentation({
                 status: 'error',
                 results: [],
@@ -45,7 +45,9 @@ export function createTavilySearchTool({ apiKey, fetchImpl = fetch }) {
                 headers: {
                     Accept: 'application/json',
                     'Content-Type': 'application/json',
-                    Authorization: `Bearer ${apiKey}`
+                    ...(apiKey?.trim()
+                        ? { Authorization: `Bearer ${apiKey}` }
+                        : { 'X-Tavily-Access-Mode': 'keyless' })
                 },
                 body: JSON.stringify({ query: normalizedQuery, max_results: 10 })
             });

package/dist/tools/web-explore.d.ts

@@ -40,6 +40,8 @@ export declare function createWebExploreTool({ explore }?: {
         headlessAttempts: number;
         exhaustedBudget: boolean;
         caveatReasons?: string[];
+        fanoutProviders?: import("../types.js").SearchProviderName[];
+        fanoutSkipped?: import("../types.js").SearchProviderName[];
     };
     error?: import("../types.js").ToolError;
 }>;

package/dist/tools/web-search.js

@@ -27,7 +27,11 @@ function htmlLooksBlocked(html) {
         normalized.includes('challenge') ||
         normalized.includes('verify you are human') ||
         normalized.includes('are you a robot') ||
-        normalized.includes('unusual traffic'));
+        normalized.includes('unusual traffic') ||
+        normalized.includes('automated requests') ||
+        normalized.includes('automated queries') ||
+        normalized.includes('detected unusual') ||
+        normalized.includes('too many requests'));
 }
 export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache = createTtlCache({ ttlMs: 30_000 }) } = {}) {
     return async function webSearch({ query }) {
@@ -57,8 +61,14 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
             };
         }
         try {
-            const html = await searchHtml(normalizedQuery);
-            const parsed = parseDuckDuckGoResults(html);
+            let html = await searchHtml(normalizedQuery);
+            let parsed = parseDuckDuckGoResults(html);
+            // A 200-OK bot-wall reads as a successful fetch, so the fetch-layer retry never sees it.
+            // Give a page that looks blocked one more shot here before we classify it.
+            if (parsed.results.length === 0 && htmlLooksBlocked(html)) {
+                html = await searchHtml(normalizedQuery);
+                parsed = parseDuckDuckGoResults(html);
+            }
             if (parsed.results.length > 0) {
                 const result = {
                     status: 'ok',
@@ -71,14 +81,16 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
                     presentation: buildSearchPresentation(result)
                 };
             }
-            if (parsed.noResults) {
+            // Check for a bot-wall before "no results": a page can carry both markers, and BLOCKED is
+            // the honest call since it routes to the fallback instead of a dead end.
+            if (htmlLooksBlocked(html)) {
                 const result = {
                     status: 'error',
                     results: [],
                     metadata: { backend: 'duckduckgo', cacheHit: false },
                     error: {
-                        code: 'NO_RESULTS',
-                        message: 'DuckDuckGo returned no usable results for this query.'
+                        code: 'BLOCKED',
+                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
                     }
                 };
                 return {
@@ -86,14 +98,14 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
                     presentation: buildSearchPresentation(result)
                 };
             }
-            if (htmlLooksBlocked(html)) {
+            if (parsed.noResults) {
                 const result = {
                     status: 'error',
                     results: [],
                     metadata: { backend: 'duckduckgo', cacheHit: false },
                     error: {
-                        code: 'BLOCKED',
-                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
+                        code: 'NO_RESULTS',
+                        message: 'DuckDuckGo returned no usable results for this query.'
                     }
                 };
                 return {

package/dist/types.d.ts

@@ -6,6 +6,13 @@ export type SearchResult = {
     url: string;
     snippet: string;
 };
+export type SearchProviderName = 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
+export type FanoutMode = 'off' | 'on' | 'auto';
+export type FanoutMetadata = {
+    mode: Exclude<FanoutMode, 'off'>;
+    providers: SearchProviderName[];
+    skipped?: SearchProviderName[];
+};
 export type ToolError = {
     code: string;
     message: string;
@@ -13,8 +20,9 @@ export type ToolError = {
 export type SearchMetadata = {
     backend: 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
     cacheHit: boolean;
-    fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
+    fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily' | 'duckduckgo';
     fallbackReason?: string;
+    fanout?: FanoutMetadata;
 };
 export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type FetchMetadata = {
@@ -70,6 +78,8 @@ export type WebExploreResponse = {
         headlessAttempts: number;
         exhaustedBudget: boolean;
         caveatReasons?: string[];
+        fanoutProviders?: SearchProviderName[];
+        fanoutSkipped?: SearchProviderName[];
     };
     presentation?: PresentationEnvelope;
     error?: ToolError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.8.0",
+  "version": "1.10.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",