@demigodmode/pi-web-agent 1.7.2 → 1.9.0

package/dist/readers/youtube-reader.js

@@ -0,0 +1,71 @@
+import { getSubtitles, getVideoDetails } from 'youtube-caption-extractor';
+import { READER_TEXT_CAP } from './limits.js';
+function extractVideoId(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return undefined;
+    }
+    const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
+    if (host === 'youtu.be') {
+        return parsed.pathname.split('/').filter(Boolean)[0];
+    }
+    if (host === 'youtube.com') {
+        if (parsed.pathname === '/watch')
+            return parsed.searchParams.get('v') ?? undefined;
+        const [prefix, id] = parsed.pathname.split('/').filter(Boolean);
+        if ((prefix === 'shorts' || prefix === 'live' || prefix === 'embed') && id)
+            return id;
+    }
+    return undefined;
+}
+export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetails = getVideoDetails } = {}) {
+    return {
+        name: 'youtube',
+        canHandle(url) {
+            return extractVideoId(url) !== undefined;
+        },
+        async read(url) {
+            const videoID = extractVideoId(url);
+            if (!videoID) {
+                return {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'youtube', cacheHit: false },
+                    error: { code: 'YOUTUBE_READ_FAILED', message: 'Could not extract a video id.' }
+                };
+            }
+            try {
+                const [subtitles, details] = await Promise.all([
+                    fetchSubtitles({ videoID, lang: 'en' }),
+                    fetchDetails({ videoID, lang: 'en' }).catch(() => ({ title: undefined }))
+                ]);
+                if (!subtitles || subtitles.length === 0) {
+                    return {
+                        status: 'unsupported',
+                        url,
+                        metadata: { method: 'youtube', cacheHit: false },
+                        error: { code: 'YOUTUBE_NO_CAPTIONS', message: 'No captions available for this video.' }
+                    };
+                }
+                const transcript = subtitles.map((line) => line.text).join(' ').replace(/\s+/g, ' ').trim();
+                return {
+                    status: 'ok',
+                    url,
+                    content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0, READER_TEXT_CAP) },
+                    metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >= READER_TEXT_CAP }
+                };
+            }
+            catch (err) {
+                return {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'youtube', cacheHit: false },
+                    error: { code: 'YOUTUBE_READ_FAILED', message: err instanceof Error ? err.message : 'YouTube read failed.' }
+                };
+            }
+        }
+    };
+}

package/dist/search/fanout.d.ts

@@ -0,0 +1,14 @@
+import type { FanoutMode, SearchProviderName, WebSearchResponse } from '../types.js';
+export type FanoutProvider = {
+    name: SearchProviderName;
+    search: (input: {
+        query: string;
+    }) => Promise<WebSearchResponse>;
+};
+export declare function createFanoutSearch({ providers, mode, timeoutMs }: {
+    providers: FanoutProvider[];
+    mode: Exclude<FanoutMode, 'off'>;
+    timeoutMs?: number;
+}): ({ query }: {
+    query: string;
+}) => Promise<WebSearchResponse>;

package/dist/search/fanout.js

@@ -0,0 +1,118 @@
+import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import { canonicalizeUrl } from '../orchestration/url.js';
+const FANOUT_MIN_RESULTS = 3;
+function hostOf(url) {
+    try {
+        return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+    }
+    catch {
+        return undefined;
+    }
+}
+function primaryLooksWeak(results) {
+    if (results.length < FANOUT_MIN_RESULTS)
+        return true;
+    const hosts = new Set(results.map((r) => hostOf(r.url)).filter(Boolean));
+    return hosts.size <= 1;
+}
+function merge(lists) {
+    const byKey = new Map();
+    for (const { name, results } of lists) {
+        const seenThisProvider = new Set();
+        results.forEach((result, index) => {
+            const key = canonicalizeUrl(result.url) ?? result.url;
+            if (seenThisProvider.has(key))
+                return; // ignore duplicates within a single provider's own list
+            seenThisProvider.add(key);
+            const existing = byKey.get(key);
+            if (!existing) {
+                byKey.set(key, { result: { ...result }, providers: new Set([name]), bestRank: index });
+                return;
+            }
+            existing.providers.add(name);
+            existing.bestRank = Math.min(existing.bestRank, index);
+            if ((result.title?.length ?? 0) > (existing.result.title?.length ?? 0))
+                existing.result.title = result.title;
+            if ((result.snippet?.length ?? 0) > (existing.result.snippet?.length ?? 0))
+                existing.result.snippet = result.snippet;
+        });
+    }
+    return [...byKey.values()]
+        .sort((a, b) => b.providers.size - a.providers.size || a.bestRank - b.bestRank)
+        .map((entry) => entry.result);
+}
+function withPresentation(result) {
+    return { ...result, presentation: buildSearchPresentation(result) };
+}
+const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
+/** Resolve to undefined if the provider doesn't answer in time, so one slow/unreachable
+ *  provider (e.g. a down self-hosted SearXNG) can't stall the whole fanout across passes. */
+function withTimeout(promise, ms) {
+    return new Promise((resolve) => {
+        const timer = setTimeout(() => resolve(undefined), ms);
+        promise.then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, () => {
+            clearTimeout(timer);
+            resolve(undefined);
+        });
+    });
+}
+export function createFanoutSearch({ providers, mode, timeoutMs = FANOUT_PROVIDER_TIMEOUT_MS }) {
+    return async function fanoutSearch({ query }) {
+        const [primary, ...rest] = providers;
+        async function runSet(set) {
+            const settled = await Promise.all(set.map((p) => withTimeout(p.search({ query }), timeoutMs)));
+            const contributing = [];
+            const skipped = [];
+            set.forEach((provider, i) => {
+                const value = settled[i];
+                if (value && value.status === 'ok' && value.results.length > 0) {
+                    contributing.push({ name: provider.name, results: value.results });
+                }
+                else {
+                    skipped.push(provider.name);
+                }
+            });
+            return { contributing, skipped };
+        }
+        function finalize(lists, skipped, resolvedMode) {
+            if (lists.length === 0) {
+                const fanout = { mode: resolvedMode, providers: [] };
+                if (skipped.length > 0)
+                    fanout.skipped = skipped;
+                return withPresentation({
+                    status: 'error',
+                    results: [],
+                    metadata: { backend: primary.name, cacheHit: false, fanout },
+                    error: { code: 'FANOUT_NO_RESULTS', message: 'No fanout provider returned usable results.' }
+                });
+            }
+            const fanout = { mode: resolvedMode, providers: lists.map((l) => l.name) };
+            if (skipped.length > 0)
+                fanout.skipped = skipped;
+            return withPresentation({
+                status: 'ok',
+                results: merge(lists),
+                metadata: { backend: primary.name, cacheHit: false, fanout }
+            });
+        }
+        if (mode === 'auto') {
+            const primaryOutcome = await withTimeout(primary.search({ query }), timeoutMs);
+            const primaryResults = primaryOutcome?.status === 'ok' ? primaryOutcome.results : [];
+            if (primaryResults.length > 0 && !primaryLooksWeak(primaryResults)) {
+                return withPresentation(primaryOutcome); // strong primary: no fanout
+            }
+            const { contributing, skipped } = await runSet(rest);
+            const lists = [
+                ...(primaryResults.length ? [{ name: primary.name, results: primaryResults }] : []),
+                ...contributing
+            ];
+            const allSkipped = [...(primaryResults.length ? [] : [primary.name]), ...skipped];
+            return finalize(lists, allSkipped, 'auto');
+        }
+        const { contributing, skipped } = await runSet(providers);
+        return finalize(contributing, skipped, 'on');
+    };
+}

package/dist/tools/web-explore.d.ts

@@ -31,7 +31,7 @@ export declare function createWebExploreTool({ explore }?: {
     sources: Array<{
         title: string;
         url: string;
-        method?: "http" | "headless" | "firecrawl";
+        method?: import("../types.js").FetchMethod;
     }>;
     caveat?: string;
     metadata?: {
@@ -40,6 +40,8 @@ export declare function createWebExploreTool({ explore }?: {
         headlessAttempts: number;
         exhaustedBudget: boolean;
         caveatReasons?: string[];
+        fanoutProviders?: import("../types.js").SearchProviderName[];
+        fanoutSkipped?: import("../types.js").SearchProviderName[];
     };
     error?: import("../types.js").ToolError;
 }>;

package/dist/types.d.ts

@@ -6,6 +6,13 @@ export type SearchResult = {
     url: string;
     snippet: string;
 };
+export type SearchProviderName = 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
+export type FanoutMode = 'off' | 'on' | 'auto';
+export type FanoutMetadata = {
+    mode: Exclude<FanoutMode, 'off'>;
+    providers: SearchProviderName[];
+    skipped?: SearchProviderName[];
+};
 export type ToolError = {
     code: string;
     message: string;
@@ -15,9 +22,11 @@ export type SearchMetadata = {
     cacheHit: boolean;
     fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
     fallbackReason?: string;
+    fanout?: FanoutMetadata;
 };
+export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type FetchMetadata = {
-    method: 'http' | 'headless' | 'firecrawl';
+    method: FetchMethod;
     cacheHit: boolean;
     fallbackFrom?: 'firecrawl';
     fallbackReason?: string;
@@ -60,7 +69,7 @@ export type WebExploreResponse = {
     sources: Array<{
         title: string;
         url: string;
-        method?: 'http' | 'headless' | 'firecrawl';
+        method?: FetchMethod;
     }>;
     caveat?: string;
     metadata?: {
@@ -69,6 +78,8 @@ export type WebExploreResponse = {
         headlessAttempts: number;
         exhaustedBudget: boolean;
         caveatReasons?: string[];
+        fanoutProviders?: SearchProviderName[];
+        fanoutSkipped?: SearchProviderName[];
     };
     presentation?: PresentationEnvelope;
     error?: ToolError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.7.2",
+  "version": "1.9.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",
@@ -67,7 +67,9 @@
     "cheerio": "^1.1.0",
     "jsdom": "^26.0.0",
     "playwright": "^1.60.0",
-    "typebox": "^1.1.37"
+    "typebox": "^1.1.37",
+    "unpdf": "^1.8.1",
+    "youtube-caption-extractor": "^1.10.2"
   },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.10",