@demigodmode/pi-web-agent 1.7.1 → 1.8.0

package/CHANGELOG.md

@@ -18,6 +18,35 @@ The format is intentionally simple and release-oriented.
 ### Breaking
 - None.
 
+## [1.8.0] - 2026-08-14
+### Added
+- Read the content behind GitHub, PDF, and YouTube links directly instead of the page shell. GitHub files, issues and PRs come from the API/raw endpoints (optional `GITHUB_TOKEN` raises the rate limit), PDFs are parsed with unpdf, and YouTube links return the transcript. All keyless. Scanned PDFs and caption-less videos are caveated rather than failing. (#39, #40, #41)
+
+### Changed
+- None.
+
+### Fixed
+- None.
+
+### Breaking
+- None.
+
+## [1.7.2] - 2026-08-12
+### Added
+- None.
+
+### Changed
+- None.
+
+### Fixed
+- Candidate scoring now matches GitHub sources by parsed hostname instead of a `url.includes('github.com/')` substring check that could also match unrelated hosts like `evil.com/github.com/`.
+- `decodeHtmlEntities` now decodes `&amp;` last so an already-encoded entity such as `&amp;lt;` no longer collapses into `<` (double-unescape).
+- The CI workflow now sets an explicit read-only default `GITHUB_TOKEN` permission.
+- Bumped transitive `postcss`, `undici`, and `brace-expansion` in the lockfile to clear reported advisories.
+
+### Breaking
+- None.
+
 ## [1.7.1] - 2026-07-31
 ### Added
 - None.

package/dist/backends/factory.js

@@ -10,6 +10,10 @@ import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
 import { createWebFetchTool } from '../tools/web-fetch.js';
 import { createWebSearchTool } from '../tools/web-search.js';
 import { DEFAULT_BACKEND_CONFIG } from './config.js';
+import { createSpecialContentResolver } from '../readers/resolver.js';
+import { createGithubReader } from '../readers/github-reader.js';
+import { createPdfReader } from '../readers/pdf-reader.js';
+import { createYoutubeReader } from '../readers/youtube-reader.js';
 function invalidSearxngSearch() {
     return async function search() {
         const result = {
@@ -125,9 +129,13 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
     if (config.fetch.provider === 'firecrawl' && config.fetch.fallback === 'http') {
         fetchPage = withFetchFallback(fetchPage, httpFetch);
     }
+    const fetchPageWithReaders = createSpecialContentResolver({
+        readers: [createGithubReader(), createPdfReader(), createYoutubeReader()],
+        fallback: fetchPage
+    });
     return {
         search,
-        fetchPage,
+        fetchPage: fetchPageWithReaders,
         headlessFetch: createHeadlessFetch()
     };
 }

package/dist/extract/readability.js

@@ -25,18 +25,24 @@ export function extractReadableContent(html, maxLength = 4000) {
         text
     };
 }
+const NAMED_ENTITIES = {
+    ' ': ' ',
+    '&': '&',
+    '&lt;': '<',
+    '&gt;': '>',
+    '&quot;': '"'
+};
 function decodeHtmlEntities(text) {
-    return text
-        .replace(/&nbsp;/gi, ' ')
-        .replace(/&amp;/gi, '&')
-        .replace(/&lt;/gi, '<')
-        .replace(/&gt;/gi, '>')
-        .replace(/&quot;/gi, '"')
-        .replace(/&#39;/gi, "'")
-        .replace(/&#x27;/gi, "'")
-        .replace(/&#x2F;/gi, '/')
-        .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
-        .replace(/&#x([\da-f]+);/gi, (_, code) => String.fromCharCode(parseInt(code, 16)));
+    // Single pass so each entity is decoded exactly once. A multi-pass decode
+    // rescans its own output, which double-unescapes either &amp;lt; -> < or
+    // &#38;amp; -> & depending on the pass order. One pass avoids both.
+    return text.replace(/&(?:#(\d+)|#x([\da-f]+)|nbsp|amp|lt|gt|quot);/gi, (match, dec, hex) => {
+        if (dec !== undefined)
+            return String.fromCharCode(Number(dec));
+        if (hex !== undefined)
+            return String.fromCharCode(parseInt(hex, 16));
+        return NAMED_ENTITIES[match.toLowerCase()] ?? match;
+    });
 }
 function extractTitle(html) {
     const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);

package/dist/orchestration/candidate-selector.js

@@ -1,9 +1,10 @@
-import { classifySourceProfile } from './source-profile.js';
+import { classifySourceProfile, hostOf } from './source-profile.js';
 function wantsDiscussionSources(query = '') {
     return /reddit|forum|forums|discussion|thread|comments|community|user experience|people recommend/i.test(query);
 }
 function candidateScore(result, query) {
-    const url = result.url.toLowerCase();
+    const host = hostOf(result.url);
+    const isGithub = host === 'github.com' || (host?.endsWith('.github.com') ?? false);
     const profile = classifySourceProfile(result.url);
     const wantsThreads = wantsDiscussionSources(query);
     if (profile.kind === 'official-docs')
@@ -15,7 +16,7 @@ function candidateScore(result, query) {
             return 2;
         if (profile.kind === 'issue-thread')
             return 3;
-        if (url.includes('github.com/'))
+        if (isGithub)
             return 4;
         if (profile.kind === 'package-page')
             return 6;
@@ -23,7 +24,7 @@ function candidateScore(result, query) {
     }
     if (profile.kind === 'issue-thread')
         return 2;
-    if (url.includes('github.com/'))
+    if (isGithub)
         return 3;
     if (profile.kind === 'package-page')
         return 5;

package/dist/orchestration/research-types.d.ts

@@ -1,5 +1,5 @@
 export type ResearchSourceKind = 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
-export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl';
+export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type ResearchEvidence = {
     title: string;
     url: string;

package/dist/orchestration/source-profile.d.ts

@@ -5,4 +5,5 @@ export type SourceProfile = {
     sourceKind: ResearchSourceKind;
     shouldPreferHeadlessWhenWeak: boolean;
 };
+export declare function hostOf(rawUrl: string): string | undefined;
 export declare function classifySourceProfile(rawUrl: string): SourceProfile;

package/dist/orchestration/source-profile.js

@@ -10,6 +10,9 @@ function parseUrl(rawUrl) {
         return undefined;
     }
 }
+export function hostOf(rawUrl) {
+    return parseUrl(rawUrl)?.hostname.toLowerCase().replace(/^www\./, '');
+}
 function isOfficialApi(host, path) {
     return ((host === 'playwright.dev' && path.startsWith('/docs/api/')) ||
         (host === 'vitest.dev' && path.startsWith('/config/')));

package/dist/presentation/explore-presentation.js

@@ -5,6 +5,12 @@ function internalReaderLabel(method) {
         return 'firecrawl';
     if (method === 'http')
         return 'web_fetch';
+    if (method === 'github')
+        return 'github';
+    if (method === 'pdf')
+        return 'pdf';
+    if (method === 'youtube')
+        return 'youtube';
     return 'web_explore';
 }
 export function buildExplorePresentation(result) {

package/dist/readers/github-reader.d.ts

@@ -0,0 +1,7 @@
+import type { SpecialContentReader } from './types.js';
+type GithubReaderDeps = {
+    fetchImpl?: typeof fetch;
+    token?: string;
+};
+export declare function createGithubReader({ fetchImpl, token }?: GithubReaderDeps): SpecialContentReader;
+export {};

package/dist/readers/github-reader.js

@@ -0,0 +1,129 @@
+function parseGithub(url) {
+    try {
+        return new URL(url);
+    }
+    catch {
+        return undefined;
+    }
+}
+function classifyGithubShape(url) {
+    const parsed = parseGithub(url);
+    if (!parsed || parsed.hostname.toLowerCase() !== 'github.com') {
+        return undefined;
+    }
+    const segments = parsed.pathname.split('/').filter(Boolean);
+    const [owner, repo, type, ...rest] = segments;
+    // blob: [owner, repo, 'blob', ref, ...path] with rest.length >= 2
+    if (owner && repo && type === 'blob' && rest.length >= 2) {
+        const [ref, ...pathParts] = rest;
+        return { shape: 'blob', owner, repo, ref, path: pathParts.join('/') };
+    }
+    // issue: [owner, repo, 'issues', N]
+    if (owner && repo && type === 'issues' && rest[0]) {
+        return { shape: 'issue', owner, repo, num: rest[0] };
+    }
+    // pull: [owner, repo, 'pull', N]
+    if (owner && repo && type === 'pull' && rest[0]) {
+        return { shape: 'pull', owner, repo, num: rest[0] };
+    }
+    // repo-root: exactly [owner, repo]
+    if (owner && repo && !type) {
+        return { shape: 'repo-root', owner, repo };
+    }
+    return undefined;
+}
+function encodeSegment(segment) {
+    let decoded;
+    try {
+        decoded = decodeURIComponent(segment);
+    }
+    catch {
+        decoded = segment;
+    }
+    return encodeURIComponent(decoded);
+}
+function fail(url, message) {
+    return {
+        status: 'error',
+        url,
+        metadata: { method: 'github', cacheHit: false },
+        error: { code: 'GITHUB_FETCH_FAILED', message }
+    };
+}
+function okResponse(url, title, text) {
+    const capped = text.slice(0, 4000);
+    return {
+        status: 'ok',
+        url,
+        content: { title, text: capped },
+        metadata: { method: 'github', cacheHit: false, truncated: text.length >= 4000 }
+    };
+}
+export function createGithubReader({ fetchImpl = fetch, token = process.env.GITHUB_TOKEN } = {}) {
+    function headers(json) {
+        const h = { 'User-Agent': 'pi-web-agent' };
+        if (json)
+            h.Accept = 'application/vnd.github+json';
+        if (token)
+            h.Authorization = `Bearer ${token}`;
+        return h;
+    }
+    async function getText(target) {
+        const res = await fetchImpl(target, { headers: headers(false) });
+        if (!res.ok)
+            throw new Error(`GitHub returned ${res.status} for ${target}`);
+        return res.text();
+    }
+    async function getJson(target) {
+        const res = await fetchImpl(target, { headers: headers(true) });
+        if (!res.ok)
+            throw new Error(`GitHub API returned ${res.status} for ${target}`);
+        return res.json();
+    }
+    async function readBlob(url, owner, repo, ref, path) {
+        const encodedPath = path.split('/').map(encodeSegment).join('/');
+        const raw = `https://raw.githubusercontent.com/${owner}/${repo}/${encodeSegment(ref)}/${encodedPath}`;
+        const text = await getText(raw);
+        return okResponse(url, `${owner}/${repo}/${path}`, text);
+    }
+    async function readThread(url, owner, repo, kind, num) {
+        const base = `https://api.github.com/repos/${owner}/${repo}/${kind}/${num}`;
+        const item = await getJson(base);
+        const comments = await getJson(`${base}/comments`);
+        const body = [item.body ?? '', ...comments.map((c) => c.body ?? '')].filter(Boolean).join('\n\n---\n\n');
+        return okResponse(url, item.title ?? `${owner}/${repo} ${kind} #${num}`, body);
+    }
+    async function readRepoRoot(url, owner, repo) {
+        const readmeMeta = await getJson(`https://api.github.com/repos/${owner}/${repo}/readme`);
+        const readme = readmeMeta.download_url ? await getText(readmeMeta.download_url) : '';
+        const tree = await getJson(`https://api.github.com/repos/${owner}/${repo}/contents`);
+        const listing = tree.map((entry) => `${entry.type === 'dir' ? '[dir] ' : ''}${entry.name}`).join('\n');
+        return okResponse(url, `${owner}/${repo}`, `${readme}\n\nTop-level contents:\n${listing}`);
+    }
+    return {
+        name: 'github',
+        canHandle(url) {
+            return classifyGithubShape(url) !== undefined;
+        },
+        async read(url) {
+            const shape = classifyGithubShape(url);
+            if (!shape)
+                return fail(url, 'Unsupported GitHub URL shape for the reader.');
+            try {
+                switch (shape.shape) {
+                    case 'blob':
+                        return await readBlob(url, shape.owner, shape.repo, shape.ref, shape.path);
+                    case 'issue':
+                        return await readThread(url, shape.owner, shape.repo, 'issues', shape.num);
+                    case 'pull':
+                        return await readThread(url, shape.owner, shape.repo, 'pulls', shape.num);
+                    case 'repo-root':
+                        return await readRepoRoot(url, shape.owner, shape.repo);
+                }
+            }
+            catch (err) {
+                return fail(url, err instanceof Error ? err.message : 'GitHub read failed.');
+            }
+        }
+    };
+}

package/dist/readers/pdf-reader.d.ts

@@ -0,0 +1,11 @@
+import type { SpecialContentReader } from './types.js';
+type PdfReaderDeps = {
+    fetchImpl?: typeof fetch;
+    /** Injectable for tests; defaults to unpdf. */
+    extractPdfText?: (bytes: Uint8Array) => Promise<{
+        text: string;
+        title?: string;
+    }>;
+};
+export declare function createPdfReader({ fetchImpl, extractPdfText }?: PdfReaderDeps): SpecialContentReader;
+export {};

package/dist/readers/pdf-reader.js

@@ -0,0 +1,75 @@
+import { extractText, getDocumentProxy, getMeta } from 'unpdf';
+async function defaultExtract(bytes) {
+    const pdf = await getDocumentProxy(bytes);
+    const [{ text }, meta] = await Promise.all([
+        extractText(pdf, { mergePages: true }),
+        getMeta(pdf).catch(() => ({ info: undefined }))
+    ]);
+    const info = meta.info;
+    const rawTitle = info?.Title?.trim();
+    return { text, title: rawTitle ? rawTitle : undefined };
+}
+function isPdfUrl(url) {
+    try {
+        return new URL(url).pathname.toLowerCase().endsWith('.pdf');
+    }
+    catch {
+        return false;
+    }
+}
+function filenameFromUrl(url) {
+    try {
+        const last = new URL(url).pathname.split('/').filter(Boolean).pop() ?? 'PDF';
+        return decodeURIComponent(last);
+    }
+    catch {
+        return 'PDF';
+    }
+}
+export function createPdfReader({ fetchImpl = fetch, extractPdfText = defaultExtract } = {}) {
+    return {
+        name: 'pdf',
+        canHandle: isPdfUrl,
+        canHandleContentType(contentType) {
+            return contentType.toLowerCase().includes('application/pdf');
+        },
+        async read(url) {
+            try {
+                const response = await fetchImpl(url);
+                if (!('ok' in response) || !response.ok) {
+                    return {
+                        status: 'error',
+                        url,
+                        metadata: { method: 'pdf', cacheHit: false },
+                        error: { code: 'PDF_READ_FAILED', message: `Fetching the PDF failed.` }
+                    };
+                }
+                const bytes = new Uint8Array(await response.arrayBuffer());
+                const extracted = await extractPdfText(bytes);
+                const text = extracted.text.trim();
+                if (text.length === 0) {
+                    return {
+                        status: 'unsupported',
+                        url,
+                        metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf' },
+                        error: { code: 'PDF_NO_TEXT', message: 'This looks like a scanned PDF with no extractable text.' }
+                    };
+                }
+                return {
+                    status: 'ok',
+                    url,
+                    content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0, 4000) },
+                    metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >= 4000 }
+                };
+            }
+            catch (err) {
+                return {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'pdf', cacheHit: false },
+                    error: { code: 'PDF_READ_FAILED', message: err instanceof Error ? err.message : 'PDF read failed.' }
+                };
+            }
+        }
+    };
+}

package/dist/readers/resolver.d.ts

@@ -0,0 +1,12 @@
+import type { WebFetchResponse } from '../types.js';
+import type { SpecialContentReader } from './types.js';
+type ResolverDeps = {
+    readers: SpecialContentReader[];
+    fallback: (input: {
+        url: string;
+    }) => Promise<WebFetchResponse>;
+};
+export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input: {
+    url: string;
+}) => Promise<WebFetchResponse>;
+export {};

package/dist/readers/resolver.js

@@ -0,0 +1,16 @@
+export function createSpecialContentResolver({ readers, fallback }) {
+    return async function resolve(input) {
+        const matched = readers.find((reader) => reader.canHandle(input.url));
+        if (matched) {
+            return matched.read(input.url);
+        }
+        const response = await fallback(input);
+        if (response.status === 'unsupported' && response.metadata.contentType) {
+            const byContentType = readers.find((reader) => reader.canHandleContentType?.(response.metadata.contentType));
+            if (byContentType) {
+                return byContentType.read(input.url);
+            }
+        }
+        return response;
+    };
+}

package/dist/readers/types.d.ts

@@ -0,0 +1,10 @@
+import type { WebFetchResponse } from '../types.js';
+export type SpecialContentReader = {
+    name: string;
+    /** Cheap URL-shape check. No network. */
+    canHandle(url: string): boolean;
+    /** Optional: claim a response by content-type after a normal fetch (e.g. application/pdf). */
+    canHandleContentType?(contentType: string): boolean;
+    /** Produce a normal WebFetchResponse. Must never throw. */
+    read(url: string): Promise<WebFetchResponse>;
+};

package/dist/readers/types.js

@@ -0,0 +1 @@
+export {};

package/dist/readers/youtube-reader.d.ts

@@ -0,0 +1,21 @@
+import type { SpecialContentReader } from './types.js';
+type Subtitle = {
+    start: string;
+    dur: string;
+    text: string;
+};
+type YoutubeReaderDeps = {
+    fetchSubtitles?: (input: {
+        videoID: string;
+        lang: string;
+    }) => Promise<Subtitle[]>;
+    fetchDetails?: (input: {
+        videoID: string;
+        lang: string;
+    }) => Promise<{
+        title?: string;
+        description?: string;
+    }>;
+};
+export declare function createYoutubeReader({ fetchSubtitles, fetchDetails }?: YoutubeReaderDeps): SpecialContentReader;
+export {};

package/dist/readers/youtube-reader.js

@@ -0,0 +1,70 @@
+import { getSubtitles, getVideoDetails } from 'youtube-caption-extractor';
+function extractVideoId(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return undefined;
+    }
+    const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
+    if (host === 'youtu.be') {
+        return parsed.pathname.split('/').filter(Boolean)[0];
+    }
+    if (host === 'youtube.com') {
+        if (parsed.pathname === '/watch')
+            return parsed.searchParams.get('v') ?? undefined;
+        const [prefix, id] = parsed.pathname.split('/').filter(Boolean);
+        if ((prefix === 'shorts' || prefix === 'live' || prefix === 'embed') && id)
+            return id;
+    }
+    return undefined;
+}
+export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetails = getVideoDetails } = {}) {
+    return {
+        name: 'youtube',
+        canHandle(url) {
+            return extractVideoId(url) !== undefined;
+        },
+        async read(url) {
+            const videoID = extractVideoId(url);
+            if (!videoID) {
+                return {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'youtube', cacheHit: false },
+                    error: { code: 'YOUTUBE_READ_FAILED', message: 'Could not extract a video id.' }
+                };
+            }
+            try {
+                const [subtitles, details] = await Promise.all([
+                    fetchSubtitles({ videoID, lang: 'en' }),
+                    fetchDetails({ videoID, lang: 'en' }).catch(() => ({ title: undefined }))
+                ]);
+                if (!subtitles || subtitles.length === 0) {
+                    return {
+                        status: 'unsupported',
+                        url,
+                        metadata: { method: 'youtube', cacheHit: false },
+                        error: { code: 'YOUTUBE_NO_CAPTIONS', message: 'No captions available for this video.' }
+                    };
+                }
+                const transcript = subtitles.map((line) => line.text).join(' ').replace(/\s+/g, ' ').trim();
+                return {
+                    status: 'ok',
+                    url,
+                    content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0, 4000) },
+                    metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >= 4000 }
+                };
+            }
+            catch (err) {
+                return {
+                    status: 'error',
+                    url,
+                    metadata: { method: 'youtube', cacheHit: false },
+                    error: { code: 'YOUTUBE_READ_FAILED', message: err instanceof Error ? err.message : 'YouTube read failed.' }
+                };
+            }
+        }
+    };
+}

package/dist/tools/web-explore.d.ts

@@ -31,7 +31,7 @@ export declare function createWebExploreTool({ explore }?: {
     sources: Array<{
         title: string;
         url: string;
-        method?: "http" | "headless" | "firecrawl";
+        method?: import("../types.js").FetchMethod;
     }>;
     caveat?: string;
     metadata?: {

package/dist/types.d.ts

@@ -16,8 +16,9 @@ export type SearchMetadata = {
     fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
     fallbackReason?: string;
 };
+export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 export type FetchMetadata = {
-    method: 'http' | 'headless' | 'firecrawl';
+    method: FetchMethod;
     cacheHit: boolean;
     fallbackFrom?: 'firecrawl';
     fallbackReason?: string;
@@ -60,7 +61,7 @@ export type WebExploreResponse = {
     sources: Array<{
         title: string;
         url: string;
-        method?: 'http' | 'headless' | 'firecrawl';
+        method?: FetchMethod;
     }>;
     caveat?: string;
     metadata?: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",
@@ -67,7 +67,9 @@
     "cheerio": "^1.1.0",
     "jsdom": "^26.0.0",
     "playwright": "^1.60.0",
-    "typebox": "^1.1.37"
+    "typebox": "^1.1.37",
+    "unpdf": "^1.8.1",
+    "youtube-caption-extractor": "^1.10.2"
   },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.10",