@demigodmode/pi-web-agent 1.7.2 → 1.9.0

package/dist/orchestration/research-orchestrator.js

@@ -13,6 +13,9 @@ function classifyEvidenceUrl(url) {
 function summarizeText(text, maxLength = 180) {
     return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
 }
+function isReaderMethod(method) {
+    return method === 'github' || method === 'pdf' || method === 'youtube';
+}
 function isBotCheckContent({ title = '', text }) {
     return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
 }
@@ -21,6 +24,16 @@ function evidenceFromFetch(result) {
         return null;
     if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
         return null;
+    if (isReaderMethod(result.metadata.method)) {
+        return {
+            title: result.content.title ?? result.url,
+            url: result.url,
+            sourceKind: 'primary-content',
+            method: result.metadata.method,
+            summary: result.content.text,
+            supports: [result.content.text]
+        };
+    }
     return {
         title: result.content.title ?? result.url,
         url: result.url,
@@ -66,13 +79,15 @@ function shouldRetryDirectWithHeadless(result, evidence) {
         return false;
     return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
 }
-function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [] }) {
+function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped }) {
     return {
         searchPasses: previousQueries.length,
         fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
         headlessAttempts,
         exhaustedBudget,
-        caveatReasons
+        caveatReasons,
+        fanoutProviders,
+        fanoutSkipped
     };
 }
 function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
@@ -99,6 +114,13 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
             const suggestedHeadlessUrls = [];
             let headlessAttempts = 0;
             let lastPass;
+            const fanoutProvidersSeen = new Set();
+            const fanoutSkippedSeen = new Set();
+            function fanoutSnapshot() {
+                const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
+                const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
+                return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
+            }
             if (fetchDirect) {
                 for (const url of extractDirectUrls(query).slice(0, 3)) {
                     const directResult = await fetchDirect({ url });
@@ -134,6 +156,35 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     }
                 }
             }
+            if (allEvidence.some((item) => item.sourceKind === 'primary-content')) {
+                const ranked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
+                const quality = analyzeEvidenceQuality({
+                    evidence: ranked,
+                    gaps: allGaps,
+                    lowValueOutcomes: allLowValueOutcomes
+                });
+                return {
+                    decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
+                    evidence: ranked,
+                    workerPass: combinedWorkerPass({
+                        lastPass,
+                        previousQueries,
+                        allGaps,
+                        allLowValueOutcomes,
+                        exhaustedBudget: false
+                    }),
+                    metadata: buildMetadata({
+                        previousQueries,
+                        allEvidence,
+                        allGaps,
+                        allLowValueOutcomes,
+                        headlessAttempts,
+                        exhaustedBudget: false,
+                        caveatReasons: quality.caveatReasons,
+                        ...fanoutSnapshot()
+                    })
+                };
+            }
             for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
                 const queries = planSearchQueries({
                     originalQuery: query,
@@ -149,6 +200,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                         maxFetches: DEFAULT_MAX_FETCHES_PER_PASS
                     });
                     lastPass = pass;
+                    pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
+                    pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
                     allEvidence.push(...pass.evidence);
                     allGaps.push(...pass.gaps);
                     allLowValueOutcomes.push(...pass.lowValueOutcomes);
@@ -213,7 +266,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                                     allLowValueOutcomes,
                                     headlessAttempts,
                                     exhaustedBudget,
-                                    caveatReasons: updatedQuality.caveatReasons
+                                    caveatReasons: updatedQuality.caveatReasons,
+                                    ...fanoutSnapshot()
                                 })
                             };
                         }
@@ -239,7 +293,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                                 allLowValueOutcomes,
                                 headlessAttempts,
                                 exhaustedBudget: false,
-                                caveatReasons: quality.caveatReasons
+                                caveatReasons: quality.caveatReasons,
+                                ...fanoutSnapshot()
                             })
                         };
                     }
@@ -262,7 +317,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                                 allLowValueOutcomes,
                                 headlessAttempts,
                                 exhaustedBudget,
-                                caveatReasons: quality.caveatReasons
+                                caveatReasons: quality.caveatReasons,
+                                ...fanoutSnapshot()
                             })
                         };
                     }
@@ -291,7 +347,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     allLowValueOutcomes,
                     headlessAttempts,
                     exhaustedBudget: true,
-                    caveatReasons: quality.caveatReasons
+                    caveatReasons: quality.caveatReasons,
+                    ...fanoutSnapshot()
                 })
             };
         }

package/dist/orchestration/research-types.d.ts

@@ -1,5 +1,6 @@
-export type ResearchSourceKind = 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
-export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl';
+import type { SearchProviderName } from '../types.js';
+export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
+export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type ResearchEvidence = {
     title: string;
     url: string;
@@ -24,12 +25,16 @@ export type ResearchWorkerResult = {
     lowValueOutcomes: ResearchLowValueOutcome[];
     suggestedHeadlessUrl?: string;
     exhaustedBudget: boolean;
+    fanoutProviders?: SearchProviderName[];
+    fanoutSkipped?: SearchProviderName[];
 };
 export type ResearchRunMetadata = {
     searchPasses: number;
     fetchedPages: number;
     headlessAttempts: number;
     exhaustedBudget: boolean;
+    fanoutProviders?: SearchProviderName[];
+    fanoutSkipped?: SearchProviderName[];
 };
 export type ResearchOrchestratorDecision = {
     action: 'answer';

package/dist/orchestration/research-worker.js

@@ -6,6 +6,9 @@ function classifySource(url) {
 function summarizeText(text, maxLength = 180) {
     return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
 }
+function isReaderMethod(method) {
+    return method === 'github' || method === 'pdf' || method === 'youtube';
+}
 function isBotCheckContent({ title = '', text }) {
     return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
 }
@@ -15,6 +18,18 @@ function evidenceFromFetch(fetched, fallbackTitle) {
         return null;
     if (isBotCheckContent({ title: content.title, text: content.text }))
         return null;
+    // A successful reader read with usable text is primary content, exempt from the
+    // package-page filter below.
+    if (isReaderMethod(fetched.metadata.method) && content.text.trim()) {
+        return {
+            title: content.title ?? fallbackTitle,
+            url: fetched.url,
+            sourceKind: 'primary-content',
+            method: fetched.metadata.method,
+            summary: content.text,
+            supports: [content.text]
+        };
+    }
     const sourceKind = classifySource(fetched.url);
     if (sourceKind === 'package-page') {
         return null;
@@ -65,6 +80,8 @@ export function createResearchWorker({ search, fetchPage }) {
                 };
             }
             const searchResult = await search({ query });
+            const fanoutProviders = searchResult.metadata.fanout?.providers;
+            const fanoutSkipped = searchResult.metadata.fanout?.skipped;
             if (searchResult.status !== 'ok') {
                 return {
                     searchQueries,
@@ -77,7 +94,9 @@ export function createResearchWorker({ search, fetchPage }) {
                     ],
                     lowValueOutcomes,
                     suggestedHeadlessUrl,
-                    exhaustedBudget: false
+                    exhaustedBudget: false,
+                    fanoutProviders,
+                    fanoutSkipped
                 };
             }
             if (searchResult.results.length === 0) {
@@ -92,7 +111,9 @@ export function createResearchWorker({ search, fetchPage }) {
                         }
                     ],
                     suggestedHeadlessUrl,
-                    exhaustedBudget: false
+                    exhaustedBudget: false,
+                    fanoutProviders,
+                    fanoutSkipped
                 };
             }
             const candidates = selectCandidates({
@@ -133,7 +154,9 @@ export function createResearchWorker({ search, fetchPage }) {
                 gaps,
                 lowValueOutcomes,
                 suggestedHeadlessUrl,
-                exhaustedBudget: false
+                exhaustedBudget: false,
+                fanoutProviders,
+                fanoutSkipped
             };
         }
     };

package/dist/orchestration/url.d.ts

@@ -0,0 +1,6 @@
+declare const TRACKING_PARAMS: Set<string>;
+/** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
+export declare function canonicalizeUrl(raw: string, options?: {
+    canonicalizeHost?: boolean;
+}): string | undefined;
+export { TRACKING_PARAMS };

package/dist/orchestration/url.js

@@ -0,0 +1,34 @@
+const TRACKING_PARAMS = new Set([
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_term',
+    'utm_content',
+    'utm_name',
+    'fbclid',
+    'gclid'
+]);
+/** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
+export function canonicalizeUrl(raw, options = {}) {
+    const { canonicalizeHost = true } = options;
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        return undefined;
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        return undefined;
+    if (canonicalizeHost) {
+        url.hostname = url.hostname.replace(/^www\./, '');
+    }
+    for (const key of [...url.searchParams.keys()]) {
+        if (TRACKING_PARAMS.has(key.toLowerCase())) {
+            url.searchParams.delete(key);
+        }
+    }
+    url.hash = '';
+    return url.toString().replace(/\/$/, '');
+}
+export { TRACKING_PARAMS };

package/dist/presentation/explore-presentation.js

@@ -5,6 +5,12 @@ function internalReaderLabel(method) {
         return 'firecrawl';
     if (method === 'http')
         return 'web_fetch';
+    if (method === 'github')
+        return 'github';
+    if (method === 'pdf')
+        return 'pdf';
+    if (method === 'youtube')
+        return 'youtube';
     return 'web_explore';
 }
 export function buildExplorePresentation(result) {
@@ -16,8 +22,15 @@ export function buildExplorePresentation(result) {
             }
         };
     }
+    const fanoutProviders = result.metadata?.fanoutProviders;
+    const fanoutSkipped = result.metadata?.fanoutSkipped;
+    const fanoutNote = fanoutProviders?.length
+        ? ` (fanout: ${fanoutProviders.join(', ')}${fanoutSkipped?.length ? `; skipped: ${fanoutSkipped.join(', ')}` : ''})`
+        : fanoutSkipped?.length
+            ? ` (fanout; skipped: ${fanoutSkipped.join(', ')})`
+            : '';
     const internalSummary = result.metadata
-        ? `Internal research: web_search ×${result.metadata.searchPasses}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
+        ? `Internal research: web_search ×${result.metadata.searchPasses}${fanoutNote}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
         : undefined;
     const hasEvidence = result.findings.length > 0 || result.sources.length > 0;
     const evidenceLines = hasEvidence
@@ -40,12 +53,13 @@ export function buildExplorePresentation(result) {
     ]
         .filter((line) => line !== undefined)
         .join('\n');
+    const compact = hasEvidence
+        ? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
+        : 'No usable evidence found';
     return {
         mode: 'compact',
         views: {
-            compact: hasEvidence
-                ? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
-                : 'No usable evidence found',
+            compact,
             preview,
             verbose
         },

package/dist/presentation/search-presentation.js

@@ -1,12 +1,24 @@
+function fanoutNote(result) {
+    const f = result.metadata.fanout;
+    if (!f)
+        return '';
+    if (f.providers.length) {
+        return ` (fanout: ${f.providers.join(', ')}${f.skipped?.length ? `; skipped: ${f.skipped.join(', ')}` : ''})`;
+    }
+    if (f.skipped?.length) {
+        return ` (fanout; skipped: ${f.skipped.join(', ')})`;
+    }
+    return '';
+}
 function formatCompact(result) {
     const fallbackPrefix = result.metadata.fallbackFrom
         ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
         : '';
     if (result.status === 'error') {
-        return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}`;
+        return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}${fanoutNote(result)}`;
     }
     const suffix = result.results.length === 1 ? 'result' : 'results';
-    return `${fallbackPrefix}Found ${result.results.length} ${suffix}`;
+    return `${fallbackPrefix}Found ${result.results.length} ${suffix}${fanoutNote(result)}`;
 }
 export function buildSearchPresentation(result) {
     const preview = result.results

package/dist/readers/github-reader.d.ts

@@ -0,0 +1,7 @@
+import type { SpecialContentReader } from './types.js';
+type GithubReaderDeps = {
+    fetchImpl?: typeof fetch;
+    token?: string;
+};
+export declare function createGithubReader({ fetchImpl, token }?: GithubReaderDeps): SpecialContentReader;
+export {};

package/dist/readers/github-reader.js

@@ -0,0 +1,130 @@
+import { READER_TEXT_CAP } from './limits.js';
+function parseGithub(url) {
+    try {
+        return new URL(url);
+    }
+    catch {
+        return undefined;
+    }
+}
+function classifyGithubShape(url) {
+    const parsed = parseGithub(url);
+    if (!parsed || parsed.hostname.toLowerCase() !== 'github.com') {
+        return undefined;
+    }
+    const segments = parsed.pathname.split('/').filter(Boolean);
+    const [owner, repo, type, ...rest] = segments;
+    // blob: [owner, repo, 'blob', ref, ...path] with rest.length >= 2
+    if (owner && repo && type === 'blob' && rest.length >= 2) {
+        const [ref, ...pathParts] = rest;
+        return { shape: 'blob', owner, repo, ref, path: pathParts.join('/') };
+    }
+    // issue: [owner, repo, 'issues', N]
+    if (owner && repo && type === 'issues' && rest[0]) {
+        return { shape: 'issue', owner, repo, num: rest[0] };
+    }
+    // pull: [owner, repo, 'pull', N]
+    if (owner && repo && type === 'pull' && rest[0]) {
+        return { shape: 'pull', owner, repo, num: rest[0] };
+    }
+    // repo-root: exactly [owner, repo]
+    if (owner && repo && !type) {
+        return { shape: 'repo-root', owner, repo };
+    }
+    return undefined;
+}
+function encodeSegment(segment) {
+    let decoded;
+    try {
+        decoded = decodeURIComponent(segment);
+    }
+    catch {
+        decoded = segment;
+    }
+    return encodeURIComponent(decoded);
+}
+function fail(url, message) {
+    return {
+        status: 'error',
+        url,
+        metadata: { method: 'github', cacheHit: false },
+        error: { code: 'GITHUB_FETCH_FAILED', message }
+    };
+}
+function okResponse(url, title, text) {
+    const capped = text.slice(0, READER_TEXT_CAP);
+    return {
+        status: 'ok',
+        url,
+        content: { title, text: capped },
+        metadata: { method: 'github', cacheHit: false, truncated: text.length >= READER_TEXT_CAP }
+    };
+}
+export function createGithubReader({ fetchImpl = fetch, token = process.env.GITHUB_TOKEN } = {}) {
+    function headers(json) {
+        const h = { 'User-Agent': 'pi-web-agent' };
+        if (json)
+            h.Accept = 'application/vnd.github+json';
+        if (token)
+            h.Authorization = `Bearer ${token}`;
+        return h;
+    }
+    async function getText(target) {
+        const res = await fetchImpl(target, { headers: headers(false) });
+        if (!res.ok)
+            throw new Error(`GitHub returned ${res.status} for ${target}`);
+        return res.text();
+    }
+    async function getJson(target) {
+        const res = await fetchImpl(target, { headers: headers(true) });
+        if (!res.ok)
+            throw new Error(`GitHub API returned ${res.status} for ${target}`);
+        return res.json();
+    }
+    async function readBlob(url, owner, repo, ref, path) {
+        const encodedPath = path.split('/').map(encodeSegment).join('/');
+        const raw = `https://raw.githubusercontent.com/${owner}/${repo}/${encodeSegment(ref)}/${encodedPath}`;
+        const text = await getText(raw);
+        return okResponse(url, `${owner}/${repo}/${path}`, text);
+    }
+    async function readThread(url, owner, repo, kind, num) {
+        const base = `https://api.github.com/repos/${owner}/${repo}/${kind}/${num}`;
+        const item = await getJson(base);
+        const comments = await getJson(`${base}/comments`);
+        const body = [item.body ?? '', ...comments.map((c) => c.body ?? '')].filter(Boolean).join('\n\n---\n\n');
+        return okResponse(url, item.title ?? `${owner}/${repo} ${kind} #${num}`, body);
+    }
+    async function readRepoRoot(url, owner, repo) {
+        const readmeMeta = await getJson(`https://api.github.com/repos/${owner}/${repo}/readme`);
+        const readme = readmeMeta.download_url ? await getText(readmeMeta.download_url) : '';
+        const tree = await getJson(`https://api.github.com/repos/${owner}/${repo}/contents`);
+        const listing = tree.map((entry) => `${entry.type === 'dir' ? '[dir] ' : ''}${entry.name}`).join('\n');
+        return okResponse(url, `${owner}/${repo}`, `${readme}\n\nTop-level contents:\n${listing}`);
+    }
+    return {
+        name: 'github',
+        canHandle(url) {
+            return classifyGithubShape(url) !== undefined;
+        },
+        async read(url) {
+            const shape = classifyGithubShape(url);
+            if (!shape)
+                return fail(url, 'Unsupported GitHub URL shape for the reader.');
+            try {
+                switch (shape.shape) {
+                    case 'blob':
+                        return await readBlob(url, shape.owner, shape.repo, shape.ref, shape.path);
+                    case 'issue':
+                        return await readThread(url, shape.owner, shape.repo, 'issues', shape.num);
+                    case 'pull':
+                        return await readThread(url, shape.owner, shape.repo, 'pulls', shape.num);
+                    case 'repo-root':
+                        return await readRepoRoot(url, shape.owner, shape.repo);
+                }
+            }
+            catch (err) {
+                return fail(url, err instanceof Error ? err.message : 'GitHub read failed.');
+            }
+        }
+    };
+}

package/dist/readers/limits.d.ts

@@ -0,0 +1,3 @@
+/** Max characters a special-content reader returns. Large enough for a typical transcript
+ *  or a medium PDF; very long documents still truncate (section-aware chunking is future work). */
+export declare const READER_TEXT_CAP = 24000;

package/dist/readers/limits.js

@@ -0,0 +1,3 @@
+/** Max characters a special-content reader returns. Large enough for a typical transcript
+ *  or a medium PDF; very long documents still truncate (section-aware chunking is future work). */
+export const READER_TEXT_CAP = 24000;

package/dist/readers/pdf-reader.d.ts

@@ -0,0 +1,11 @@
+import type { SpecialContentReader } from './types.js';
+type PdfReaderDeps = {
+    fetchImpl?: typeof fetch;
+    /** Injectable for tests; defaults to unpdf. */
+    extractPdfText?: (bytes: Uint8Array) => Promise<{
+        text: string;
+        title?: string;
+    }>;
+};
+export declare function createPdfReader({ fetchImpl, extractPdfText }?: PdfReaderDeps): SpecialContentReader;
+export {};

package/dist/readers/pdf-reader.js

@@ -0,0 +1,76 @@
+import { extractText, getDocumentProxy, getMeta } from 'unpdf';
+import { READER_TEXT_CAP } from './limits.js';
+async function defaultExtract(bytes) {
+    const pdf = await getDocumentProxy(bytes);
+    const [{ text }, meta] = await Promise.all([
+        extractText(pdf, { mergePages: true }),
+        getMeta(pdf).catch(() => ({ info: undefined }))
+    ]);
+    const info = meta.info;
+    const rawTitle = info?.Title?.trim();
+    return { text, title: rawTitle ? rawTitle : undefined };
+}
+function isPdfUrl(url) {
+    try {
+        return new URL(url).pathname.toLowerCase().endsWith('.pdf');
+    }
+    catch {
+        return false;
+    }
+}
+function filenameFromUrl(url) {
+    try {
+        const last = new URL(url).pathname.split('/').filter(Boolean).pop() ?? 'PDF';
+        return decodeURIComponent(last);
+    }
+    catch {
+        return 'PDF';
+    }
+}
+export function createPdfReader({ fetchImpl = fetch, extractPdfText = defaultExtract } = {}) {
+    return {
+        name: 'pdf',
+        canHandle: isPdfUrl,
+        canHandleContentType(contentType) {
+            return contentType.toLowerCase().includes('application/pdf');
+        },
+        async read(url) {
+            try {
+                const response = await fetchImpl(url);
+                if (!('ok' in response) || !response.ok) {
+                    return {
+                        status: 'error',
+                        url,
+                        metadata: { method: 'pdf', cacheHit: false },
+                        error: { code: 'PDF_READ_FAILED', message: `Fetching the PDF failed.` }
+                    };
+                }
+                const bytes = new Uint8Array(await response.arrayBuffer());
+                const extracted = await extractPdfText(bytes);
+                const text = extracted.text.trim();
+                if (text.length === 0) {
+                    return {
+                        status: 'unsupported',
+                        url,
+                        metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf' },
+                        error: { code: 'PDF_NO_TEXT', message: 'This looks like a scanned PDF with no extractable text.' }
+                    };
+                }
+                return {
+                    status: 'ok',
+                    url,
+                    content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0, READER_TEXT_CAP) },
+                    metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >= READER_TEXT_CAP }
+                };
+            }
+            catch (err) {
+                return {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'pdf', cacheHit: false },
+                    error: { code: 'PDF_READ_FAILED', message: err instanceof Error ? err.message : 'PDF read failed.' }
+                };
+            }
+        }
+    };
+}

package/dist/readers/resolver.d.ts

@@ -0,0 +1,12 @@
+import type { WebFetchResponse } from '../types.js';
+import type { SpecialContentReader } from './types.js';
+type ResolverDeps = {
+    readers: SpecialContentReader[];
+    fallback: (input: {
+        url: string;
+    }) => Promise<WebFetchResponse>;
+};
+export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input: {
+    url: string;
+}) => Promise<WebFetchResponse>;
+export {};

package/dist/readers/resolver.js

@@ -0,0 +1,16 @@
+export function createSpecialContentResolver({ readers, fallback }) {
+    return async function resolve(input) {
+        const matched = readers.find((reader) => reader.canHandle(input.url));
+        if (matched) {
+            return matched.read(input.url);
+        }
+        const response = await fallback(input);
+        if (response.status === 'unsupported' && response.metadata.contentType) {
+            const byContentType = readers.find((reader) => reader.canHandleContentType?.(response.metadata.contentType));
+            if (byContentType) {
+                return byContentType.read(input.url);
+            }
+        }
+        return response;
+    };
+}

package/dist/readers/types.d.ts

@@ -0,0 +1,10 @@
+import type { WebFetchResponse } from '../types.js';
+export type SpecialContentReader = {
+    name: string;
+    /** Cheap URL-shape check. No network. */
+    canHandle(url: string): boolean;
+    /** Optional: claim a response by content-type after a normal fetch (e.g. application/pdf). */
+    canHandleContentType?(contentType: string): boolean;
+    /** Produce a normal WebFetchResponse. Must never throw. */
+    read(url: string): Promise<WebFetchResponse>;
+};

package/dist/readers/types.js

@@ -0,0 +1 @@
+export {};

package/dist/readers/youtube-reader.d.ts

@@ -0,0 +1,21 @@
+import type { SpecialContentReader } from './types.js';
+type Subtitle = {
+    start: string;
+    dur: string;
+    text: string;
+};
+type YoutubeReaderDeps = {
+    fetchSubtitles?: (input: {
+        videoID: string;
+        lang: string;
+    }) => Promise<Subtitle[]>;
+    fetchDetails?: (input: {
+        videoID: string;
+        lang: string;
+    }) => Promise<{
+        title?: string;
+        description?: string;
+    }>;
+};
+export declare function createYoutubeReader({ fetchSubtitles, fetchDetails }?: YoutubeReaderDeps): SpecialContentReader;
+export {};