@demigodmode/pi-web-agent 0.2.1 → 0.2.2

package/dist/src/cache/ttl-cache.js

@@ -0,0 +1,21 @@
+export function createCacheKey(parts) {
+    return JSON.stringify(parts);
+}
+export function createTtlCache({ ttlMs, now = () => Date.now() }) {
+    const entries = new Map();
+    return {
+        get(key) {
+            const entry = entries.get(key);
+            if (!entry)
+                return undefined;
+            if (entry.expiresAt <= now()) {
+                entries.delete(key);
+                return undefined;
+            }
+            return entry.value;
+        },
+        set(key, value) {
+            entries.set(key, { value, expiresAt: now() + ttlMs });
+        }
+    };
+}

package/dist/src/extension.d.ts

@@ -0,0 +1,2 @@
+import type { ExtensionAPI } from '@mariozechner/pi-coding-agent';
+export default function extension(pi: ExtensionAPI): void;

package/dist/src/extension.js

@@ -0,0 +1,155 @@
+import { Type } from '@sinclair/typebox';
+import { createWebExploreTool } from './tools/web-explore.js';
+import { createWebFetchTool } from './tools/web-fetch.js';
+import { createWebFetchHeadlessTool } from './tools/web-fetch-headless.js';
+import { createWebSearchTool } from './tools/web-search.js';
+export default function extension(pi) {
+    const webSearch = createWebSearchTool();
+    const webFetch = createWebFetchTool();
+    const webFetchHeadless = createWebFetchHeadlessTool();
+    const webExplore = createWebExploreTool();
+    let webExploreUsedInCurrentFlow = false;
+    const postWebExploreGuardError = {
+        code: 'POST_WEB_EXPLORE_GUARD',
+        message: 'web_explore already ran for this research task. Only use low-level web tools if there is a specific unresolved gap.'
+    };
+    function guardSearchResponse() {
+        const result = {
+            status: 'error',
+            results: [],
+            metadata: {
+                backend: 'duckduckgo',
+                cacheHit: false
+            },
+            error: postWebExploreGuardError
+        };
+        return {
+            content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+            details: result,
+            isError: true
+        };
+    }
+    function guardFetchResponse(url) {
+        const result = {
+            status: 'error',
+            url,
+            metadata: {
+                method: 'http',
+                cacheHit: false
+            },
+            error: postWebExploreGuardError
+        };
+        return {
+            content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+            details: result,
+            isError: true
+        };
+    }
+    function guardHeadlessResponse(url) {
+        const result = {
+            status: 'error',
+            url,
+            metadata: {
+                method: 'headless',
+                cacheHit: false
+            },
+            error: postWebExploreGuardError
+        };
+        return {
+            content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+            details: result,
+            isError: true
+        };
+    }
+    pi.on('before_agent_start', async (event) => {
+        webExploreUsedInCurrentFlow = false;
+        return {
+            systemPrompt: `${event.systemPrompt}\n\n` +
+                'For web research questions that require finding and comparing multiple sources, prefer web_explore. ' +
+                'Use web_search, web_fetch, and web_fetch_headless for direct/manual operations like explicit search calls, specific URL reads, or debugging. ' +
+                'After using web_explore, only call low-level web tools if there is a specific unresolved gap. ' +
+                'Do not keep searching or fetching just for extra confirmation.'
+        };
+    });
+    pi.registerTool({
+        name: 'web_search',
+        label: 'Web Search',
+        description: 'Direct search tool for manual discovery of links and snippets. Use for explicit search requests or when the user wants raw search results. Prefer web_explore for broader research questions.',
+        parameters: Type.Object({
+            query: Type.String({ description: 'Search query.' })
+        }),
+        async execute(_toolCallId, params) {
+            if (webExploreUsedInCurrentFlow) {
+                return guardSearchResponse();
+            }
+            const result = await webSearch({ query: params.query });
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                details: result,
+                isError: result.status === 'error'
+            };
+        }
+    });
+    pi.registerTool({
+        name: 'web_fetch',
+        label: 'Web Fetch',
+        description: 'Direct HTTP page fetch for a specific URL. Use when the user wants one page read directly. Prefer web_explore for broader research across multiple sources.',
+        parameters: Type.Object({
+            url: Type.String({ description: 'HTTP or HTTPS URL to fetch.' })
+        }),
+        async execute(_toolCallId, params) {
+            if (webExploreUsedInCurrentFlow) {
+                return guardFetchResponse(params.url);
+            }
+            const result = await webFetch({ url: params.url });
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                details: result,
+                isError: result.status === 'error'
+            };
+        }
+    });
+    pi.registerTool({
+        name: 'web_fetch_headless',
+        label: 'Web Fetch Headless',
+        description: 'Direct headless page fetch for a specific URL when browser rendering is explicitly needed. Prefer web_explore for research tasks; it decides headless escalation internally.',
+        parameters: Type.Object({
+            url: Type.String({ description: 'HTTP or HTTPS URL to fetch in headless mode.' })
+        }),
+        async execute(_toolCallId, params) {
+            if (webExploreUsedInCurrentFlow) {
+                return guardHeadlessResponse(params.url);
+            }
+            const result = await webFetchHeadless({ url: params.url });
+            return {
+                content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                details: result,
+                isError: result.status === 'error'
+            };
+        }
+    });
+    pi.registerTool({
+        name: 'web_explore',
+        label: 'Web Explore',
+        description: 'Research a web question using bounded search/fetch passes, source ranking, and targeted headless escalation. Prefer this for multi-source web research, current docs/discussion lookups, and recommendation summaries. Use this instead of chaining low-level web tools for the same research task.',
+        parameters: Type.Object({
+            query: Type.String({ description: 'Web research question to explore.' })
+        }),
+        async execute(_toolCallId, params) {
+            const result = await webExplore({ query: params.query });
+            if (result.status === 'ok') {
+                webExploreUsedInCurrentFlow = true;
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: result.status === 'ok' ? result.text : JSON.stringify(result, null, 2)
+                    }
+                ],
+                details: result,
+                isError: result.status === 'error'
+            };
+        }
+    });
+}

package/dist/src/extract/readability.d.ts

@@ -0,0 +1,8 @@
+import type { ExtractedContent } from '../types.js';
+export type ReadableExtractionMode = 'readability' | 'fallback';
+export type SafeReadableExtraction = {
+    mode: ReadableExtractionMode;
+    content: ExtractedContent;
+};
+export declare function extractReadableContent(html: string, maxLength?: number): ExtractedContent;
+export declare function extractReadableContentSafely(html: string, maxLength?: number): SafeReadableExtraction;

package/dist/src/extract/readability.js

@@ -0,0 +1,93 @@
+import { Readability } from '@mozilla/readability';
+import { JSDOM, VirtualConsole } from 'jsdom';
+export function extractReadableContent(html, maxLength = 4000) {
+    let stylesheetError;
+    const virtualConsole = new VirtualConsole();
+    virtualConsole.on('jsdomError', (error) => {
+        if (!stylesheetError && error.message.includes('Could not parse CSS stylesheet')) {
+            stylesheetError = error;
+        }
+    });
+    const dom = new JSDOM(html, {
+        url: 'https://example.com',
+        virtualConsole
+    });
+    if (stylesheetError) {
+        throw stylesheetError;
+    }
+    const article = new Readability(dom.window.document).parse();
+    const rawText = (article?.textContent ?? dom.window.document.body.textContent ?? '').trim();
+    const text = rawText.slice(0, maxLength);
+    const fallbackTitle = dom.window.document.title || undefined;
+    return {
+        title: article?.title ?? fallbackTitle,
+        byline: article?.byline || undefined,
+        text
+    };
+}
+function decodeHtmlEntities(text) {
+    return text
+        .replace(/ /gi, ' ')
+        .replace(/&/gi, '&')
+        .replace(/&lt;/gi, '<')
+        .replace(/&gt;/gi, '>')
+        .replace(/&quot;/gi, '"')
+        .replace(/&#39;/gi, "'")
+        .replace(/&#x27;/gi, "'")
+        .replace(/&#x2F;/gi, '/')
+        .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
+        .replace(/&#x([\da-f]+);/gi, (_, code) => String.fromCharCode(parseInt(code, 16)));
+}
+function extractTitle(html) {
+    const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    if (!match)
+        return undefined;
+    return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()) || undefined;
+}
+function stripTagContent(html, tagName) {
+    return html.replace(new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}>`, 'gi'), ' ');
+}
+function extractPreferredSection(html) {
+    const mainMatch = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
+    if (mainMatch)
+        return mainMatch[1];
+    const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
+    if (articleMatch)
+        return articleMatch[1];
+    const bodyMatch = html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
+    if (bodyMatch)
+        return bodyMatch[1];
+    return html;
+}
+function extractFallbackText(html, maxLength) {
+    const title = extractTitle(html);
+    let section = extractPreferredSection(html);
+    section = stripTagContent(section, 'script');
+    section = stripTagContent(section, 'style');
+    section = stripTagContent(section, 'noscript');
+    section = stripTagContent(section, 'svg');
+    section = stripTagContent(section, 'template');
+    const text = decodeHtmlEntities(section)
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, maxLength);
+    return {
+        title,
+        text
+    };
+}
+export function extractReadableContentSafely(html, maxLength = 4000) {
+    try {
+        return {
+            mode: 'readability',
+            content: extractReadableContent(html, maxLength)
+        };
+    }
+    catch {
+        return {
+            mode: 'fallback',
+            content: extractFallbackText(html, maxLength)
+        };
+    }
+}

package/dist/src/fetch/browser-resolution.d.ts

@@ -0,0 +1,15 @@
+export type BrowserResolutionResult = {
+    ok: true;
+    executablePath: string;
+    browser: 'configured' | 'chrome' | 'edge';
+} | {
+    ok: false;
+    error: {
+        code: 'BROWSER_NOT_FOUND' | 'CONFIGURED_BROWSER_NOT_FOUND';
+        message: string;
+    };
+};
+export declare function resolveBrowserExecutable({ configuredPath, fileExists }: {
+    configuredPath?: string;
+    fileExists?: (path: string) => Promise<boolean>;
+}): Promise<BrowserResolutionResult>;

package/dist/src/fetch/browser-resolution.js

@@ -0,0 +1,55 @@
+const WINDOWS_CANDIDATES = {
+    chrome: [
+        'C:/Program Files/Google/Chrome/Application/chrome.exe',
+        'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe'
+    ],
+    edge: [
+        'C:/Program Files/Microsoft/Edge/Application/msedge.exe',
+        'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe'
+    ]
+};
+export async function resolveBrowserExecutable({ configuredPath, fileExists = defaultFileExists }) {
+    if (configuredPath) {
+        if (await fileExists(configuredPath)) {
+            return {
+                ok: true,
+                executablePath: configuredPath,
+                browser: 'configured'
+            };
+        }
+        return {
+            ok: false,
+            error: {
+                code: 'CONFIGURED_BROWSER_NOT_FOUND',
+                message: `Configured browser path was not found: ${configuredPath}`
+            }
+        };
+    }
+    for (const path of WINDOWS_CANDIDATES.chrome) {
+        if (await fileExists(path)) {
+            return { ok: true, executablePath: path, browser: 'chrome' };
+        }
+    }
+    for (const path of WINDOWS_CANDIDATES.edge) {
+        if (await fileExists(path)) {
+            return { ok: true, executablePath: path, browser: 'edge' };
+        }
+    }
+    return {
+        ok: false,
+        error: {
+            code: 'BROWSER_NOT_FOUND',
+            message: 'No compatible local browser was found for headless fetch.'
+        }
+    };
+}
+async function defaultFileExists(path) {
+    try {
+        const { access } = await import('node:fs/promises');
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/src/fetch/headless-fetch.d.ts

@@ -0,0 +1,18 @@
+import { type BrowserResolutionResult } from './browser-resolution.js';
+import type { WebFetchHeadlessResponse } from '../types.js';
+export declare function headlessFetch(url: string, { configuredPath, resolveBrowser, launchBrowser, now }?: {
+    configuredPath?: string;
+    resolveBrowser?: (options?: {
+        configuredPath?: string;
+    }) => Promise<BrowserResolutionResult>;
+    launchBrowser?: (options: {
+        executablePath: string;
+    }) => Promise<{
+        newContext: () => Promise<{
+            newPage: () => Promise<any>;
+            close: () => Promise<void>;
+        }>;
+        close: () => Promise<void>;
+    }>;
+    now?: () => number;
+}): Promise<WebFetchHeadlessResponse>;

package/dist/src/fetch/headless-fetch.js

@@ -0,0 +1,87 @@
+import { chromium } from 'playwright-core';
+import { extractReadableContentSafely } from '../extract/readability.js';
+import { resolveBrowserExecutable } from './browser-resolution.js';
+function cleanupRenderedText(text) {
+    let cleaned = text.replace(/(Show more)(\s+\1){1,}/gi, '$1');
+    cleaned = cleaned.replace(/(Privacy Terms)(\s+\1){1,}/gi, '$1');
+    cleaned = cleaned.replace(/\s+/g, ' ').trim();
+    return cleaned;
+}
+export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath }) => chromium.launch({ executablePath, headless: true }), now = () => Date.now() } = {}) {
+    const resolved = await resolveBrowser({ configuredPath });
+    if (!resolved.ok) {
+        return {
+            status: 'error',
+            url,
+            metadata: { method: 'headless', cacheHit: false },
+            error: resolved.error
+        };
+    }
+    let browser;
+    let context;
+    let page;
+    try {
+        browser = await launchBrowser({ executablePath: resolved.executablePath });
+        context = await browser.newContext();
+        page = await context.newPage();
+        const startedAt = now();
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+        await page.waitForLoadState('load', { timeout: 10000 });
+        await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
+        const html = await page.content();
+        const finishedAt = now();
+        const extraction = extractReadableContentSafely(html);
+        const cleanedContent = {
+            ...extraction.content,
+            text: cleanupRenderedText(extraction.content.text)
+        };
+        if (!cleanedContent.text || cleanedContent.text.length < 40) {
+            return {
+                status: 'blocked',
+                url,
+                metadata: {
+                    method: 'headless',
+                    cacheHit: false,
+                    browser: resolved.browser,
+                    navigationMs: finishedAt - startedAt
+                },
+                error: {
+                    code: 'HEADLESS_EXTRACTION_WEAK',
+                    message: 'Rendered page did not produce enough readable content.'
+                }
+            };
+        }
+        return {
+            status: 'ok',
+            url,
+            content: cleanedContent,
+            metadata: {
+                method: 'headless',
+                cacheHit: false,
+                browser: resolved.browser,
+                navigationMs: finishedAt - startedAt,
+                truncated: cleanedContent.text.length >= 4000
+            }
+        };
+    }
+    catch (error) {
+        return {
+            status: 'error',
+            url,
+            metadata: {
+                method: 'headless',
+                cacheHit: false,
+                browser: resolved.browser
+            },
+            error: {
+                code: 'HEADLESS_NAVIGATION_FAILED',
+                message: error instanceof Error ? error.message : 'Unknown headless navigation failure.'
+            }
+        };
+    }
+    finally {
+        await page?.close?.().catch(() => undefined);
+        await context?.close?.().catch(() => undefined);
+        await browser?.close?.().catch(() => undefined);
+    }
+}

package/dist/src/fetch/http-fetch.d.ts

@@ -0,0 +1,4 @@
+import type { WebFetchResponse } from '../types.js';
+export declare function createHttpFetcher({ fetchImpl }?: {
+    fetchImpl?: typeof fetch;
+}): (url: string) => Promise<WebFetchResponse>;

package/dist/src/fetch/http-fetch.js

@@ -0,0 +1,50 @@
+import { extractReadableContentSafely } from '../extract/readability.js';
+function looksLikeScriptShell(html) {
+    const lower = html.toLowerCase();
+    return lower.includes('<script') && (lower.includes('id="app"') || lower.includes('id="root"'));
+}
+function isWeakHttpContent(options) {
+    const normalizedText = options.text.replace(/\s+/g, ' ').trim();
+    const normalizedHtml = options.html.replace(/\s+/g, ' ').trim();
+    const textLength = normalizedText.length;
+    const htmlLength = normalizedHtml.length;
+    const hasGenericShellMarker = /enable javascript|javascript required|please turn on javascript/i.test(options.html);
+    const veryShortBody = textLength > 0 && textLength < 120;
+    const lowDensity = htmlLength > 0 && textLength / htmlLength < 0.02;
+    return veryShortBody && (lowDensity || hasGenericShellMarker);
+}
+export function createHttpFetcher({ fetchImpl = fetch } = {}) {
+    return async function httpFetch(url) {
+        const response = await fetchImpl(url);
+        const contentType = response.headers.get('content-type') ?? '';
+        if (!contentType.includes('text/html')) {
+            return {
+                status: 'unsupported',
+                url: response.url,
+                metadata: { method: 'http', cacheHit: false, contentType }
+            };
+        }
+        const html = await response.text();
+        const extraction = extractReadableContentSafely(html);
+        const content = extraction.content;
+        if (looksLikeScriptShell(html) ||
+            content.text.length < 40 ||
+            isWeakHttpContent({ html, title: content.title, text: content.text })) {
+            return {
+                status: 'needs_headless',
+                url: response.url,
+                metadata: { method: 'http', cacheHit: false, contentType },
+                error: {
+                    code: 'WEAK_EXTRACTION',
+                    message: 'HTTP extraction was not reliable enough.'
+                }
+            };
+        }
+        return {
+            status: 'ok',
+            url: response.url,
+            content,
+            metadata: { method: 'http', cacheHit: false, contentType, truncated: content.text.length >= 4000 }
+        };
+    };
+}

package/dist/src/orchestration/index.d.ts

@@ -0,0 +1,41 @@
+import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
+export declare function createResearchWorkflow({ search, fetchPage, headlessFetch }?: {
+    search?: (input: {
+        query: string;
+    }) => Promise<WebSearchResponse>;
+    fetchPage?: (input: {
+        url: string;
+    }) => Promise<WebFetchResponse>;
+    headlessFetch?: (input: {
+        url: string;
+    }) => Promise<WebFetchHeadlessResponse>;
+}): {
+    run({ query }: {
+        query: string;
+    }): Promise<{
+        decision: {
+            action: "answer";
+            rationale: string;
+            approvedEvidence: import("./research-types.js").ResearchEvidence[];
+        };
+        evidence: import("./research-types.js").ResearchEvidence[];
+        workerPass: import("./research-types.js").ResearchWorkerResult;
+    } | {
+        decision: {
+            action: "escalate-headless";
+            rationale: string;
+            url: string;
+            approvedEvidence: import("./research-types.js").ResearchEvidence[];
+        };
+        evidence: import("./research-types.js").ResearchEvidence[];
+        workerPass: import("./research-types.js").ResearchWorkerResult;
+    } | {
+        decision: {
+            action: "research-again";
+            rationale: string;
+            followupQuery: string;
+        };
+        evidence: import("./research-types.js").ResearchEvidence[];
+        workerPass: import("./research-types.js").ResearchWorkerResult;
+    }>;
+};

package/dist/src/orchestration/index.js

@@ -0,0 +1,9 @@
+import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
+import { createWebFetchTool } from '../tools/web-fetch.js';
+import { createWebSearchTool } from '../tools/web-search.js';
+import { createResearchOrchestrator } from './research-orchestrator.js';
+import { createResearchWorker } from './research-worker.js';
+export function createResearchWorkflow({ search = createWebSearchTool(), fetchPage = createWebFetchTool(), headlessFetch = createWebFetchHeadlessTool() } = {}) {
+    const worker = createResearchWorker({ search, fetchPage });
+    return createResearchOrchestrator({ worker, headlessFetch });
+}

package/dist/src/orchestration/research-orchestrator.d.ts

@@ -0,0 +1,43 @@
+import type { WebFetchHeadlessResponse } from '../types.js';
+import type { ResearchEvidence, ResearchWorkerResult } from './research-types.js';
+export declare function createResearchOrchestrator({ worker, headlessFetch }: {
+    worker: {
+        run: (input: {
+            query: string;
+            maxSearchRounds: number;
+            maxFetches: number;
+        }) => Promise<ResearchWorkerResult>;
+    };
+    headlessFetch: (input: {
+        url: string;
+    }) => Promise<WebFetchHeadlessResponse>;
+}): {
+    run({ query }: {
+        query: string;
+    }): Promise<{
+        decision: {
+            action: "answer";
+            rationale: string;
+            approvedEvidence: ResearchEvidence[];
+        };
+        evidence: ResearchEvidence[];
+        workerPass: ResearchWorkerResult;
+    } | {
+        decision: {
+            action: "escalate-headless";
+            rationale: string;
+            url: string;
+            approvedEvidence: ResearchEvidence[];
+        };
+        evidence: ResearchEvidence[];
+        workerPass: ResearchWorkerResult;
+    } | {
+        decision: {
+            action: "research-again";
+            rationale: string;
+            followupQuery: string;
+        };
+        evidence: ResearchEvidence[];
+        workerPass: ResearchWorkerResult;
+    }>;
+};