@demigodmode/pi-web-agent 1.8.0 → 1.10.0

package/dist/extension.js

@@ -7,6 +7,30 @@ import { loadPresentationConfigLayers } from './presentation/config-store.js';
 import { selectPresentationView } from './presentation/select-view.js';
 import { createWebExploreTool } from './tools/web-explore.js';
 import { getUpdateChangelogNotice } from './changelog-notice.js';
+import { Text } from '@earendil-works/pi-tui';
+/**
+ * What the model receives as the tool result: the actual synthesized findings (which for a
+ * direct GitHub/PDF/YouTube read is the full extracted content), plus source citations and any
+ * caveat. This is separate from the terminal display (see renderResult), so the model always
+ * gets the substance regardless of the user's compact/preview/verbose presentation setting.
+ */
+function serializeForModel(result) {
+    if (result.status === 'error') {
+        return `Research failed: ${result.error?.message ?? 'Unknown research failure.'}`;
+    }
+    if (result.findings.length === 0 && result.sources.length === 0) {
+        return 'No usable evidence found.';
+    }
+    return [
+        result.findings.join('\n\n'),
+        result.sources.length
+            ? `Sources:\n${result.sources.map((source) => `- ${source.title}: ${source.url}`).join('\n')}`
+            : undefined,
+        result.caveat
+    ]
+        .filter(Boolean)
+        .join('\n\n');
+}
 async function loadWebAgentConfig(pi) {
     const store = pi.__presentationConfigStore;
     return store?.load?.() ?? loadPresentationConfigLayers();
@@ -29,11 +53,6 @@ async function getEffectiveBackendConfig(pi) {
         return DEFAULT_BACKEND_CONFIG;
     }
 }
-async function renderToolText(pi, toolName, details) {
-    const config = await getEffectivePresentationConfig(pi);
-    const mode = resolvePresentationMode(toolName, config);
-    return selectPresentationView(details.presentation, mode) ?? JSON.stringify(details, null, 2);
-}
 export default function extension(pi) {
     registerWebAgentConfigCommands(pi);
     const injectedWebExplore = pi.__webExploreTool;
@@ -79,16 +98,38 @@ export default function extension(pi) {
         async execute(_toolCallId, params) {
             const webExplore = await getConfiguredWebExplore();
             const result = await webExplore({ query: params.query });
+            // Terminal display honors the user's presentation mode; the model gets the full findings.
+            // The fallback must stay terse: never fall back to serializeForModel here, or a missing
+            // presentation would dump the full findings into the terminal.
+            const mode = resolvePresentationMode('web_explore', await getEffectivePresentationConfig(pi));
+            const terseFallback = 'web_explore result';
+            const terminalText = selectPresentationView(result.presentation, mode) ?? terseFallback;
+            const terminalTextExpanded = selectPresentationView(result.presentation, 'verbose') ?? terminalText;
             return {
-                content: [
-                    {
-                        type: 'text',
-                        text: await renderToolText(pi, 'web_explore', result)
-                    }
-                ],
-                details: result,
+                content: [{ type: 'text', text: serializeForModel(result) }],
+                details: { ...result, terminalText, terminalTextExpanded },
                 isError: result.status === 'error'
             };
+        },
+        renderResult(toolResult, options) {
+            const details = toolResult.details;
+            try {
+                // Legacy fallback: results persisted before this change carry `presentation` but no
+                // `terminalText`, so old sessions stay readable instead of rendering blank.
+                const legacy = options.expanded
+                    ? details?.presentation?.views?.verbose ?? details?.presentation?.views?.compact
+                    : details?.presentation?.views?.compact;
+                const text = (options.expanded ? details?.terminalTextExpanded : details?.terminalText) ??
+                    details?.terminalText ??
+                    legacy ??
+                    'web_explore result';
+                return new Text(text, 0, 0);
+            }
+            catch {
+                // Never throw: a thrown renderer makes Pi fall back to raw content, which would dump
+                // the full model-facing findings into the terminal.
+                return new Text('web_explore result', 0, 0);
+            }
         }
     });
 }

package/dist/orchestration/direct-url.js

@@ -1,13 +1,4 @@
-const TRACKING_PARAMS = new Set([
-    'utm_source',
-    'utm_medium',
-    'utm_campaign',
-    'utm_term',
-    'utm_content',
-    'utm_name',
-    'fbclid',
-    'gclid'
-]);
+import { canonicalizeUrl } from './url.js';
 function stripTrailingPunctuation(raw) {
     let next = raw.trim();
     while (/[),.;!?\]]$/.test(next)) {
@@ -19,21 +10,7 @@ function stripTrailingPunctuation(raw) {
     return next;
 }
 function normalizeDirectUrl(raw) {
-    try {
-        const url = new URL(stripTrailingPunctuation(raw));
-        if (url.protocol !== 'http:' && url.protocol !== 'https:')
-            return undefined;
-        for (const key of [...url.searchParams.keys()]) {
-            if (TRACKING_PARAMS.has(key.toLowerCase())) {
-                url.searchParams.delete(key);
-            }
-        }
-        url.hash = '';
-        return url.toString().replace(/\/$/, '');
-    }
-    catch {
-        return undefined;
-    }
+    return canonicalizeUrl(stripTrailingPunctuation(raw), { canonicalizeHost: false });
 }
 export function extractDirectUrls(query) {
     const matches = query.match(/https?:\/\/\S+/gi) ?? [];

package/dist/orchestration/evidence-quality.d.ts

@@ -7,6 +7,7 @@ export type EvidenceQualityReport = {
         community: number;
         thread: number;
         packagePage: number;
+        primaryContent: number;
         distinctHosts: number;
     };
     flags: {

package/dist/orchestration/evidence-quality.js

@@ -25,10 +25,11 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
     const community = evidence.filter((item) => item.sourceKind === 'community').length;
     const thread = evidence.filter((item) => item.sourceKind === 'issue-thread' || item.sourceKind === 'official-discussion').length;
     const packagePage = evidence.filter((item) => item.sourceKind === 'package-page').length;
+    const primaryContent = evidence.filter((item) => item.sourceKind === 'primary-content').length;
     const distinctHosts = new Set(evidence.map((item) => hostname(item.url))).size;
     const hasOfficialEvidence = official > 0;
-    const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0;
-    const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1;
+    const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0 && primaryContent === 0;
+    const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1 && primaryContent === 0;
     const hasUnreadableDirectSource = gaps.some((gap) => /Direct URL could not be read reliably/i.test(gap.message));
     const hasUnreadableThreadSource = gaps.some((gap) => /Thread source could not be read reliably/i.test(gap.message));
     const hasPossibleConflict = hasConflictMarkers(evidence);
@@ -47,6 +48,7 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
             community,
             thread,
             packagePage,
+            primaryContent,
             distinctHosts
         },
         flags: {

package/dist/orchestration/evidence-ranker.js

@@ -1,5 +1,7 @@
 function sourceRank(sourceKind) {
     switch (sourceKind) {
+        case 'primary-content':
+            return -1;
         case 'official-docs':
             return 0;
         case 'official-api':

package/dist/orchestration/index.d.ts

@@ -24,6 +24,8 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
             headlessAttempts: number;
             exhaustedBudget: boolean;
             caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
+            fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
+            fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
         };
     }>;
 };

package/dist/orchestration/research-orchestrator.d.ts

@@ -1,4 +1,4 @@
-import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
+import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
 import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
 import { type EvidenceCaveatReason } from './evidence-quality.js';
 export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
@@ -28,6 +28,8 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
             headlessAttempts: number;
             exhaustedBudget: boolean;
             caveatReasons: EvidenceCaveatReason[];
+            fanoutProviders: SearchProviderName[] | undefined;
+            fanoutSkipped: SearchProviderName[] | undefined;
         };
     }>;
 };

package/dist/orchestration/research-orchestrator.js

@@ -13,6 +13,9 @@ function classifyEvidenceUrl(url) {
 function summarizeText(text, maxLength = 180) {
     return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
 }
+function isReaderMethod(method) {
+    return method === 'github' || method === 'pdf' || method === 'youtube';
+}
 function isBotCheckContent({ title = '', text }) {
     return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
 }
@@ -21,6 +24,16 @@ function evidenceFromFetch(result) {
         return null;
     if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
         return null;
+    if (isReaderMethod(result.metadata.method)) {
+        return {
+            title: result.content.title ?? result.url,
+            url: result.url,
+            sourceKind: 'primary-content',
+            method: result.metadata.method,
+            summary: result.content.text,
+            supports: [result.content.text]
+        };
+    }
     return {
         title: result.content.title ?? result.url,
         url: result.url,
@@ -66,13 +79,15 @@ function shouldRetryDirectWithHeadless(result, evidence) {
         return false;
     return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
 }
-function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [] }) {
+function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped }) {
     return {
         searchPasses: previousQueries.length,
         fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
         headlessAttempts,
         exhaustedBudget,
-        caveatReasons
+        caveatReasons,
+        fanoutProviders,
+        fanoutSkipped
     };
 }
 function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
@@ -99,6 +114,13 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
             const suggestedHeadlessUrls = [];
             let headlessAttempts = 0;
             let lastPass;
+            const fanoutProvidersSeen = new Set();
+            const fanoutSkippedSeen = new Set();
+            function fanoutSnapshot() {
+                const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
+                const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
+                return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
+            }
             if (fetchDirect) {
                 for (const url of extractDirectUrls(query).slice(0, 3)) {
                     const directResult = await fetchDirect({ url });
@@ -134,6 +156,35 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     }
                 }
             }
+            if (allEvidence.some((item) => item.sourceKind === 'primary-content')) {
+                const ranked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
+                const quality = analyzeEvidenceQuality({
+                    evidence: ranked,
+                    gaps: allGaps,
+                    lowValueOutcomes: allLowValueOutcomes
+                });
+                return {
+                    decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
+                    evidence: ranked,
+                    workerPass: combinedWorkerPass({
+                        lastPass,
+                        previousQueries,
+                        allGaps,
+                        allLowValueOutcomes,
+                        exhaustedBudget: false
+                    }),
+                    metadata: buildMetadata({
+                        previousQueries,
+                        allEvidence,
+                        allGaps,
+                        allLowValueOutcomes,
+                        headlessAttempts,
+                        exhaustedBudget: false,
+                        caveatReasons: quality.caveatReasons,
+                        ...fanoutSnapshot()
+                    })
+                };
+            }
             for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
                 const queries = planSearchQueries({
                     originalQuery: query,
@@ -149,6 +200,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                         maxFetches: DEFAULT_MAX_FETCHES_PER_PASS
                     });
                     lastPass = pass;
+                    pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
+                    pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
                     allEvidence.push(...pass.evidence);
                     allGaps.push(...pass.gaps);
                     allLowValueOutcomes.push(...pass.lowValueOutcomes);
@@ -213,7 +266,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                                     allLowValueOutcomes,
                                     headlessAttempts,
                                     exhaustedBudget,
-                                    caveatReasons: updatedQuality.caveatReasons
+                                    caveatReasons: updatedQuality.caveatReasons,
+                                    ...fanoutSnapshot()
                                 })
                             };
                         }
@@ -239,7 +293,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                                 allLowValueOutcomes,
                                 headlessAttempts,
                                 exhaustedBudget: false,
-                                caveatReasons: quality.caveatReasons
+                                caveatReasons: quality.caveatReasons,
+                                ...fanoutSnapshot()
                             })
                         };
                     }
@@ -262,7 +317,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                                 allLowValueOutcomes,
                                 headlessAttempts,
                                 exhaustedBudget,
-                                caveatReasons: quality.caveatReasons
+                                caveatReasons: quality.caveatReasons,
+                                ...fanoutSnapshot()
                             })
                         };
                     }
@@ -291,7 +347,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     allLowValueOutcomes,
                     headlessAttempts,
                     exhaustedBudget: true,
-                    caveatReasons: quality.caveatReasons
+                    caveatReasons: quality.caveatReasons,
+                    ...fanoutSnapshot()
                 })
             };
         }

package/dist/orchestration/research-types.d.ts

@@ -1,4 +1,5 @@
-export type ResearchSourceKind = 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
+import type { SearchProviderName } from '../types.js';
+export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
 export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type ResearchEvidence = {
     title: string;
@@ -24,12 +25,16 @@ export type ResearchWorkerResult = {
     lowValueOutcomes: ResearchLowValueOutcome[];
     suggestedHeadlessUrl?: string;
     exhaustedBudget: boolean;
+    fanoutProviders?: SearchProviderName[];
+    fanoutSkipped?: SearchProviderName[];
 };
 export type ResearchRunMetadata = {
     searchPasses: number;
     fetchedPages: number;
     headlessAttempts: number;
     exhaustedBudget: boolean;
+    fanoutProviders?: SearchProviderName[];
+    fanoutSkipped?: SearchProviderName[];
 };
 export type ResearchOrchestratorDecision = {
     action: 'answer';

package/dist/orchestration/research-worker.js

@@ -6,6 +6,9 @@ function classifySource(url) {
 function summarizeText(text, maxLength = 180) {
     return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
 }
+function isReaderMethod(method) {
+    return method === 'github' || method === 'pdf' || method === 'youtube';
+}
 function isBotCheckContent({ title = '', text }) {
     return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
 }
@@ -15,6 +18,18 @@ function evidenceFromFetch(fetched, fallbackTitle) {
         return null;
     if (isBotCheckContent({ title: content.title, text: content.text }))
         return null;
+    // A successful reader read with usable text is primary content, exempt from the
+    // package-page filter below.
+    if (isReaderMethod(fetched.metadata.method) && content.text.trim()) {
+        return {
+            title: content.title ?? fallbackTitle,
+            url: fetched.url,
+            sourceKind: 'primary-content',
+            method: fetched.metadata.method,
+            summary: content.text,
+            supports: [content.text]
+        };
+    }
     const sourceKind = classifySource(fetched.url);
     if (sourceKind === 'package-page') {
         return null;
@@ -65,6 +80,8 @@ export function createResearchWorker({ search, fetchPage }) {
                 };
             }
             const searchResult = await search({ query });
+            const fanoutProviders = searchResult.metadata.fanout?.providers;
+            const fanoutSkipped = searchResult.metadata.fanout?.skipped;
             if (searchResult.status !== 'ok') {
                 return {
                     searchQueries,
@@ -77,7 +94,9 @@ export function createResearchWorker({ search, fetchPage }) {
                     ],
                     lowValueOutcomes,
                     suggestedHeadlessUrl,
-                    exhaustedBudget: false
+                    exhaustedBudget: false,
+                    fanoutProviders,
+                    fanoutSkipped
                 };
             }
             if (searchResult.results.length === 0) {
@@ -92,7 +111,9 @@ export function createResearchWorker({ search, fetchPage }) {
                         }
                     ],
                     suggestedHeadlessUrl,
-                    exhaustedBudget: false
+                    exhaustedBudget: false,
+                    fanoutProviders,
+                    fanoutSkipped
                 };
             }
             const candidates = selectCandidates({
@@ -133,7 +154,9 @@ export function createResearchWorker({ search, fetchPage }) {
                 gaps,
                 lowValueOutcomes,
                 suggestedHeadlessUrl,
-                exhaustedBudget: false
+                exhaustedBudget: false,
+                fanoutProviders,
+                fanoutSkipped
             };
         }
     };

package/dist/orchestration/url.d.ts

@@ -0,0 +1,6 @@
+declare const TRACKING_PARAMS: Set<string>;
+/** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
+export declare function canonicalizeUrl(raw: string, options?: {
+    canonicalizeHost?: boolean;
+}): string | undefined;
+export { TRACKING_PARAMS };

package/dist/orchestration/url.js

@@ -0,0 +1,34 @@
+const TRACKING_PARAMS = new Set([
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_term',
+    'utm_content',
+    'utm_name',
+    'fbclid',
+    'gclid'
+]);
+/** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
+export function canonicalizeUrl(raw, options = {}) {
+    const { canonicalizeHost = true } = options;
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        return undefined;
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        return undefined;
+    if (canonicalizeHost) {
+        url.hostname = url.hostname.replace(/^www\./, '');
+    }
+    for (const key of [...url.searchParams.keys()]) {
+        if (TRACKING_PARAMS.has(key.toLowerCase())) {
+            url.searchParams.delete(key);
+        }
+    }
+    url.hash = '';
+    return url.toString().replace(/\/$/, '');
+}
+export { TRACKING_PARAMS };

package/dist/presentation/explore-presentation.js

@@ -22,8 +22,15 @@ export function buildExplorePresentation(result) {
             }
         };
     }
+    const fanoutProviders = result.metadata?.fanoutProviders;
+    const fanoutSkipped = result.metadata?.fanoutSkipped;
+    const fanoutNote = fanoutProviders?.length
+        ? ` (fanout: ${fanoutProviders.join(', ')}${fanoutSkipped?.length ? `; skipped: ${fanoutSkipped.join(', ')}` : ''})`
+        : fanoutSkipped?.length
+            ? ` (fanout; skipped: ${fanoutSkipped.join(', ')})`
+            : '';
     const internalSummary = result.metadata
-        ? `Internal research: web_search ×${result.metadata.searchPasses}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
+        ? `Internal research: web_search ×${result.metadata.searchPasses}${fanoutNote}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
         : undefined;
     const hasEvidence = result.findings.length > 0 || result.sources.length > 0;
     const evidenceLines = hasEvidence
@@ -46,12 +53,13 @@ export function buildExplorePresentation(result) {
     ]
         .filter((line) => line !== undefined)
         .join('\n');
+    const compact = hasEvidence
+        ? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
+        : 'No usable evidence found';
     return {
         mode: 'compact',
         views: {
-            compact: hasEvidence
-                ? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
-                : 'No usable evidence found',
+            compact,
             preview,
             verbose
         },

package/dist/presentation/search-presentation.js

@@ -1,12 +1,24 @@
+function fanoutNote(result) {
+    const f = result.metadata.fanout;
+    if (!f)
+        return '';
+    if (f.providers.length) {
+        return ` (fanout: ${f.providers.join(', ')}${f.skipped?.length ? `; skipped: ${f.skipped.join(', ')}` : ''})`;
+    }
+    if (f.skipped?.length) {
+        return ` (fanout; skipped: ${f.skipped.join(', ')})`;
+    }
+    return '';
+}
 function formatCompact(result) {
     const fallbackPrefix = result.metadata.fallbackFrom
         ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
         : '';
     if (result.status === 'error') {
-        return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}`;
+        return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}${fanoutNote(result)}`;
     }
     const suffix = result.results.length === 1 ? 'result' : 'results';
-    return `${fallbackPrefix}Found ${result.results.length} ${suffix}`;
+    return `${fallbackPrefix}Found ${result.results.length} ${suffix}${fanoutNote(result)}`;
 }
 export function buildSearchPresentation(result) {
     const preview = result.results

package/dist/readers/github-reader.js

@@ -1,3 +1,4 @@
+import { READER_TEXT_CAP } from './limits.js';
 function parseGithub(url) {
     try {
         return new URL(url);
@@ -51,12 +52,12 @@ function fail(url, message) {
     };
 }
 function okResponse(url, title, text) {
-    const capped = text.slice(0, 4000);
+    const capped = text.slice(0, READER_TEXT_CAP);
     return {
         status: 'ok',
         url,
         content: { title, text: capped },
-        metadata: { method: 'github', cacheHit: false, truncated: text.length >= 4000 }
+        metadata: { method: 'github', cacheHit: false, truncated: text.length >= READER_TEXT_CAP }
     };
 }
 export function createGithubReader({ fetchImpl = fetch, token = process.env.GITHUB_TOKEN } = {}) {

package/dist/readers/limits.d.ts

@@ -0,0 +1,3 @@
+/** Max characters a special-content reader returns. Large enough for a typical transcript
+ *  or a medium PDF; very long documents still truncate (section-aware chunking is future work). */
+export declare const READER_TEXT_CAP = 24000;

package/dist/readers/limits.js

@@ -0,0 +1,3 @@
+/** Max characters a special-content reader returns. Large enough for a typical transcript
+ *  or a medium PDF; very long documents still truncate (section-aware chunking is future work). */
+export const READER_TEXT_CAP = 24000;

package/dist/readers/pdf-reader.js

@@ -1,4 +1,5 @@
 import { extractText, getDocumentProxy, getMeta } from 'unpdf';
+import { READER_TEXT_CAP } from './limits.js';
 async function defaultExtract(bytes) {
     const pdf = await getDocumentProxy(bytes);
     const [{ text }, meta] = await Promise.all([
@@ -58,8 +59,8 @@ export function createPdfReader({ fetchImpl = fetch, extractPdfText = defaultExt
                 return {
                     status: 'ok',
                     url,
-                    content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0, 4000) },
-                    metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >= 4000 }
+                    content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0, READER_TEXT_CAP) },
+                    metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >= READER_TEXT_CAP }
                 };
             }
             catch (err) {

package/dist/readers/youtube-reader.js

@@ -1,4 +1,5 @@
 import { getSubtitles, getVideoDetails } from 'youtube-caption-extractor';
+import { READER_TEXT_CAP } from './limits.js';
 function extractVideoId(url) {
     let parsed;
     try {
@@ -53,8 +54,8 @@ export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetail
                 return {
                     status: 'ok',
                     url,
-                    content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0, 4000) },
-                    metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >= 4000 }
+                    content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0, READER_TEXT_CAP) },
+                    metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >= READER_TEXT_CAP }
                 };
             }
             catch (err) {

package/dist/search/duckduckgo.d.ts

@@ -5,5 +5,14 @@ export type ParsedDuckDuckGoResults = {
     hasResultContainers: boolean;
 };
 export declare function buildSearchUrl(query: string): string;
-export declare function fetchDuckDuckGoHtml(query: string): Promise<string>;
+export declare const DUCKDUCKGO_HEADERS: {
+    readonly 'User-Agent': "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0";
+    readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+    readonly 'Accept-Language': "en-US,en;q=0.9";
+};
+export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl, retries, sleep }?: {
+    fetchImpl?: typeof fetch;
+    retries?: number;
+    sleep?: (ms: number) => Promise<void>;
+}): Promise<string>;
 export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;

package/dist/search/duckduckgo.js

@@ -21,12 +21,30 @@ export function buildSearchUrl(query) {
     const params = new URLSearchParams({ q: query });
     return `https://html.duckduckgo.com/html/?${params.toString()}`;
 }
-export async function fetchDuckDuckGoHtml(query) {
-    const response = await fetch(buildSearchUrl(query));
-    if (!response.ok) {
-        throw new Error(`DuckDuckGo request failed with ${response.status}`);
+export const DUCKDUCKGO_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9'
+};
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch, retries = 1, sleep = defaultSleep } = {}) {
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt += 1) {
+        try {
+            const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
+            if (!response.ok) {
+                throw new Error(`DuckDuckGo request failed with ${response.status}`);
+            }
+            return response.text();
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt < retries) {
+                await sleep(500);
+            }
+        }
     }
-    return response.text();
+    throw lastError instanceof Error ? lastError : new Error('DuckDuckGo request failed');
 }
 export function parseDuckDuckGoResults(html) {
     const $ = cheerio.load(html);