@demigodmode/pi-web-agent 0.2.2 → 0.4.0

package/dist/src/extract/readability.js

@@ -1,93 +0,0 @@
-import { Readability } from '@mozilla/readability';
-import { JSDOM, VirtualConsole } from 'jsdom';
-export function extractReadableContent(html, maxLength = 4000) {
-    let stylesheetError;
-    const virtualConsole = new VirtualConsole();
-    virtualConsole.on('jsdomError', (error) => {
-        if (!stylesheetError && error.message.includes('Could not parse CSS stylesheet')) {
-            stylesheetError = error;
-        }
-    });
-    const dom = new JSDOM(html, {
-        url: 'https://example.com',
-        virtualConsole
-    });
-    if (stylesheetError) {
-        throw stylesheetError;
-    }
-    const article = new Readability(dom.window.document).parse();
-    const rawText = (article?.textContent ?? dom.window.document.body.textContent ?? '').trim();
-    const text = rawText.slice(0, maxLength);
-    const fallbackTitle = dom.window.document.title || undefined;
-    return {
-        title: article?.title ?? fallbackTitle,
-        byline: article?.byline || undefined,
-        text
-    };
-}
-function decodeHtmlEntities(text) {
-    return text
-        .replace(/ /gi, ' ')
-        .replace(/&/gi, '&')
-        .replace(/&lt;/gi, '<')
-        .replace(/&gt;/gi, '>')
-        .replace(/&quot;/gi, '"')
-        .replace(/&#39;/gi, "'")
-        .replace(/&#x27;/gi, "'")
-        .replace(/&#x2F;/gi, '/')
-        .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
-        .replace(/&#x([\da-f]+);/gi, (_, code) => String.fromCharCode(parseInt(code, 16)));
-}
-function extractTitle(html) {
-    const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-    if (!match)
-        return undefined;
-    return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()) || undefined;
-}
-function stripTagContent(html, tagName) {
-    return html.replace(new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}>`, 'gi'), ' ');
-}
-function extractPreferredSection(html) {
-    const mainMatch = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
-    if (mainMatch)
-        return mainMatch[1];
-    const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
-    if (articleMatch)
-        return articleMatch[1];
-    const bodyMatch = html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
-    if (bodyMatch)
-        return bodyMatch[1];
-    return html;
-}
-function extractFallbackText(html, maxLength) {
-    const title = extractTitle(html);
-    let section = extractPreferredSection(html);
-    section = stripTagContent(section, 'script');
-    section = stripTagContent(section, 'style');
-    section = stripTagContent(section, 'noscript');
-    section = stripTagContent(section, 'svg');
-    section = stripTagContent(section, 'template');
-    const text = decodeHtmlEntities(section)
-        .replace(/<[^>]+>/g, ' ')
-        .replace(/\s+/g, ' ')
-        .trim()
-        .slice(0, maxLength);
-    return {
-        title,
-        text
-    };
-}
-export function extractReadableContentSafely(html, maxLength = 4000) {
-    try {
-        return {
-            mode: 'readability',
-            content: extractReadableContent(html, maxLength)
-        };
-    }
-    catch {
-        return {
-            mode: 'fallback',
-            content: extractFallbackText(html, maxLength)
-        };
-    }
-}

package/dist/src/fetch/browser-resolution.d.ts

@@ -1,15 +0,0 @@
-export type BrowserResolutionResult = {
-    ok: true;
-    executablePath: string;
-    browser: 'configured' | 'chrome' | 'edge';
-} | {
-    ok: false;
-    error: {
-        code: 'BROWSER_NOT_FOUND' | 'CONFIGURED_BROWSER_NOT_FOUND';
-        message: string;
-    };
-};
-export declare function resolveBrowserExecutable({ configuredPath, fileExists }: {
-    configuredPath?: string;
-    fileExists?: (path: string) => Promise<boolean>;
-}): Promise<BrowserResolutionResult>;

package/dist/src/fetch/browser-resolution.js

@@ -1,55 +0,0 @@
-const WINDOWS_CANDIDATES = {
-    chrome: [
-        'C:/Program Files/Google/Chrome/Application/chrome.exe',
-        'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe'
-    ],
-    edge: [
-        'C:/Program Files/Microsoft/Edge/Application/msedge.exe',
-        'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe'
-    ]
-};
-export async function resolveBrowserExecutable({ configuredPath, fileExists = defaultFileExists }) {
-    if (configuredPath) {
-        if (await fileExists(configuredPath)) {
-            return {
-                ok: true,
-                executablePath: configuredPath,
-                browser: 'configured'
-            };
-        }
-        return {
-            ok: false,
-            error: {
-                code: 'CONFIGURED_BROWSER_NOT_FOUND',
-                message: `Configured browser path was not found: ${configuredPath}`
-            }
-        };
-    }
-    for (const path of WINDOWS_CANDIDATES.chrome) {
-        if (await fileExists(path)) {
-            return { ok: true, executablePath: path, browser: 'chrome' };
-        }
-    }
-    for (const path of WINDOWS_CANDIDATES.edge) {
-        if (await fileExists(path)) {
-            return { ok: true, executablePath: path, browser: 'edge' };
-        }
-    }
-    return {
-        ok: false,
-        error: {
-            code: 'BROWSER_NOT_FOUND',
-            message: 'No compatible local browser was found for headless fetch.'
-        }
-    };
-}
-async function defaultFileExists(path) {
-    try {
-        const { access } = await import('node:fs/promises');
-        await access(path);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}

package/dist/src/fetch/headless-fetch.d.ts

@@ -1,18 +0,0 @@
-import { type BrowserResolutionResult } from './browser-resolution.js';
-import type { WebFetchHeadlessResponse } from '../types.js';
-export declare function headlessFetch(url: string, { configuredPath, resolveBrowser, launchBrowser, now }?: {
-    configuredPath?: string;
-    resolveBrowser?: (options?: {
-        configuredPath?: string;
-    }) => Promise<BrowserResolutionResult>;
-    launchBrowser?: (options: {
-        executablePath: string;
-    }) => Promise<{
-        newContext: () => Promise<{
-            newPage: () => Promise<any>;
-            close: () => Promise<void>;
-        }>;
-        close: () => Promise<void>;
-    }>;
-    now?: () => number;
-}): Promise<WebFetchHeadlessResponse>;

package/dist/src/fetch/headless-fetch.js

@@ -1,87 +0,0 @@
-import { chromium } from 'playwright-core';
-import { extractReadableContentSafely } from '../extract/readability.js';
-import { resolveBrowserExecutable } from './browser-resolution.js';
-function cleanupRenderedText(text) {
-    let cleaned = text.replace(/(Show more)(\s+\1){1,}/gi, '$1');
-    cleaned = cleaned.replace(/(Privacy Terms)(\s+\1){1,}/gi, '$1');
-    cleaned = cleaned.replace(/\s+/g, ' ').trim();
-    return cleaned;
-}
-export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath }) => chromium.launch({ executablePath, headless: true }), now = () => Date.now() } = {}) {
-    const resolved = await resolveBrowser({ configuredPath });
-    if (!resolved.ok) {
-        return {
-            status: 'error',
-            url,
-            metadata: { method: 'headless', cacheHit: false },
-            error: resolved.error
-        };
-    }
-    let browser;
-    let context;
-    let page;
-    try {
-        browser = await launchBrowser({ executablePath: resolved.executablePath });
-        context = await browser.newContext();
-        page = await context.newPage();
-        const startedAt = now();
-        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
-        await page.waitForLoadState('load', { timeout: 10000 });
-        await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
-        const html = await page.content();
-        const finishedAt = now();
-        const extraction = extractReadableContentSafely(html);
-        const cleanedContent = {
-            ...extraction.content,
-            text: cleanupRenderedText(extraction.content.text)
-        };
-        if (!cleanedContent.text || cleanedContent.text.length < 40) {
-            return {
-                status: 'blocked',
-                url,
-                metadata: {
-                    method: 'headless',
-                    cacheHit: false,
-                    browser: resolved.browser,
-                    navigationMs: finishedAt - startedAt
-                },
-                error: {
-                    code: 'HEADLESS_EXTRACTION_WEAK',
-                    message: 'Rendered page did not produce enough readable content.'
-                }
-            };
-        }
-        return {
-            status: 'ok',
-            url,
-            content: cleanedContent,
-            metadata: {
-                method: 'headless',
-                cacheHit: false,
-                browser: resolved.browser,
-                navigationMs: finishedAt - startedAt,
-                truncated: cleanedContent.text.length >= 4000
-            }
-        };
-    }
-    catch (error) {
-        return {
-            status: 'error',
-            url,
-            metadata: {
-                method: 'headless',
-                cacheHit: false,
-                browser: resolved.browser
-            },
-            error: {
-                code: 'HEADLESS_NAVIGATION_FAILED',
-                message: error instanceof Error ? error.message : 'Unknown headless navigation failure.'
-            }
-        };
-    }
-    finally {
-        await page?.close?.().catch(() => undefined);
-        await context?.close?.().catch(() => undefined);
-        await browser?.close?.().catch(() => undefined);
-    }
-}

package/dist/src/fetch/http-fetch.d.ts

@@ -1,4 +0,0 @@
-import type { WebFetchResponse } from '../types.js';
-export declare function createHttpFetcher({ fetchImpl }?: {
-    fetchImpl?: typeof fetch;
-}): (url: string) => Promise<WebFetchResponse>;

package/dist/src/fetch/http-fetch.js

@@ -1,50 +0,0 @@
-import { extractReadableContentSafely } from '../extract/readability.js';
-function looksLikeScriptShell(html) {
-    const lower = html.toLowerCase();
-    return lower.includes('<script') && (lower.includes('id="app"') || lower.includes('id="root"'));
-}
-function isWeakHttpContent(options) {
-    const normalizedText = options.text.replace(/\s+/g, ' ').trim();
-    const normalizedHtml = options.html.replace(/\s+/g, ' ').trim();
-    const textLength = normalizedText.length;
-    const htmlLength = normalizedHtml.length;
-    const hasGenericShellMarker = /enable javascript|javascript required|please turn on javascript/i.test(options.html);
-    const veryShortBody = textLength > 0 && textLength < 120;
-    const lowDensity = htmlLength > 0 && textLength / htmlLength < 0.02;
-    return veryShortBody && (lowDensity || hasGenericShellMarker);
-}
-export function createHttpFetcher({ fetchImpl = fetch } = {}) {
-    return async function httpFetch(url) {
-        const response = await fetchImpl(url);
-        const contentType = response.headers.get('content-type') ?? '';
-        if (!contentType.includes('text/html')) {
-            return {
-                status: 'unsupported',
-                url: response.url,
-                metadata: { method: 'http', cacheHit: false, contentType }
-            };
-        }
-        const html = await response.text();
-        const extraction = extractReadableContentSafely(html);
-        const content = extraction.content;
-        if (looksLikeScriptShell(html) ||
-            content.text.length < 40 ||
-            isWeakHttpContent({ html, title: content.title, text: content.text })) {
-            return {
-                status: 'needs_headless',
-                url: response.url,
-                metadata: { method: 'http', cacheHit: false, contentType },
-                error: {
-                    code: 'WEAK_EXTRACTION',
-                    message: 'HTTP extraction was not reliable enough.'
-                }
-            };
-        }
-        return {
-            status: 'ok',
-            url: response.url,
-            content,
-            metadata: { method: 'http', cacheHit: false, contentType, truncated: content.text.length >= 4000 }
-        };
-    };
-}

package/dist/src/orchestration/index.d.ts

@@ -1,41 +0,0 @@
-import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
-export declare function createResearchWorkflow({ search, fetchPage, headlessFetch }?: {
-    search?: (input: {
-        query: string;
-    }) => Promise<WebSearchResponse>;
-    fetchPage?: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
-    headlessFetch?: (input: {
-        url: string;
-    }) => Promise<WebFetchHeadlessResponse>;
-}): {
-    run({ query }: {
-        query: string;
-    }): Promise<{
-        decision: {
-            action: "answer";
-            rationale: string;
-            approvedEvidence: import("./research-types.js").ResearchEvidence[];
-        };
-        evidence: import("./research-types.js").ResearchEvidence[];
-        workerPass: import("./research-types.js").ResearchWorkerResult;
-    } | {
-        decision: {
-            action: "escalate-headless";
-            rationale: string;
-            url: string;
-            approvedEvidence: import("./research-types.js").ResearchEvidence[];
-        };
-        evidence: import("./research-types.js").ResearchEvidence[];
-        workerPass: import("./research-types.js").ResearchWorkerResult;
-    } | {
-        decision: {
-            action: "research-again";
-            rationale: string;
-            followupQuery: string;
-        };
-        evidence: import("./research-types.js").ResearchEvidence[];
-        workerPass: import("./research-types.js").ResearchWorkerResult;
-    }>;
-};

package/dist/src/orchestration/index.js

@@ -1,9 +0,0 @@
-import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
-import { createWebFetchTool } from '../tools/web-fetch.js';
-import { createWebSearchTool } from '../tools/web-search.js';
-import { createResearchOrchestrator } from './research-orchestrator.js';
-import { createResearchWorker } from './research-worker.js';
-export function createResearchWorkflow({ search = createWebSearchTool(), fetchPage = createWebFetchTool(), headlessFetch = createWebFetchHeadlessTool() } = {}) {
-    const worker = createResearchWorker({ search, fetchPage });
-    return createResearchOrchestrator({ worker, headlessFetch });
-}

package/dist/src/orchestration/research-orchestrator.d.ts

@@ -1,43 +0,0 @@
-import type { WebFetchHeadlessResponse } from '../types.js';
-import type { ResearchEvidence, ResearchWorkerResult } from './research-types.js';
-export declare function createResearchOrchestrator({ worker, headlessFetch }: {
-    worker: {
-        run: (input: {
-            query: string;
-            maxSearchRounds: number;
-            maxFetches: number;
-        }) => Promise<ResearchWorkerResult>;
-    };
-    headlessFetch: (input: {
-        url: string;
-    }) => Promise<WebFetchHeadlessResponse>;
-}): {
-    run({ query }: {
-        query: string;
-    }): Promise<{
-        decision: {
-            action: "answer";
-            rationale: string;
-            approvedEvidence: ResearchEvidence[];
-        };
-        evidence: ResearchEvidence[];
-        workerPass: ResearchWorkerResult;
-    } | {
-        decision: {
-            action: "escalate-headless";
-            rationale: string;
-            url: string;
-            approvedEvidence: ResearchEvidence[];
-        };
-        evidence: ResearchEvidence[];
-        workerPass: ResearchWorkerResult;
-    } | {
-        decision: {
-            action: "research-again";
-            rationale: string;
-            followupQuery: string;
-        };
-        evidence: ResearchEvidence[];
-        workerPass: ResearchWorkerResult;
-    }>;
-};

package/dist/src/orchestration/research-orchestrator.js

@@ -1,87 +0,0 @@
-function sourceRank(sourceKind) {
-    switch (sourceKind) {
-        case 'official-docs':
-            return 0;
-        case 'official-api':
-            return 1;
-        case 'official-discussion':
-            return 2;
-        case 'issue-thread':
-            return 3;
-        case 'community':
-            return 4;
-        case 'package-page':
-            return 5;
-        default:
-            return 6;
-    }
-}
-function sortEvidence(evidence) {
-    return [...evidence].sort((left, right) => sourceRank(left.sourceKind) - sourceRank(right.sourceKind));
-}
-function strongEvidence(evidence) {
-    return evidence.filter((item) => item.sourceKind === 'official-docs' ||
-        item.sourceKind === 'official-api' ||
-        item.sourceKind === 'official-discussion');
-}
-function hasOfficialDocsOrApi(evidence) {
-    return evidence.some((item) => item.sourceKind === 'official-docs' || item.sourceKind === 'official-api');
-}
-function hasBotCheck(outcomes) {
-    return outcomes.some((outcome) => outcome.kind === 'bot-check');
-}
-function isHeadlessWorthTrying(pass, approvedEvidence) {
-    if (!pass.suggestedHeadlessUrl)
-        return false;
-    if (hasBotCheck(pass.lowValueOutcomes))
-        return false;
-    if (approvedEvidence.length >= 2 && hasOfficialDocsOrApi(approvedEvidence))
-        return false;
-    const candidate = pass.suggestedHeadlessUrl;
-    return !candidate.includes('npmjs.com/package/');
-}
-export function createResearchOrchestrator({ worker, headlessFetch }) {
-    return {
-        async run({ query }) {
-            const pass = await worker.run({ query, maxSearchRounds: 1, maxFetches: 3 });
-            const approvedEvidence = sortEvidence(pass.evidence.filter((item) => item.sourceKind !== 'package-page'));
-            const strong = strongEvidence(approvedEvidence);
-            const enoughEvidence = strong.length >= 2 && hasOfficialDocsOrApi(approvedEvidence);
-            if (enoughEvidence) {
-                const decision = {
-                    action: 'answer',
-                    rationale: 'Two strong sources with official support are enough to answer safely.',
-                    approvedEvidence
-                };
-                return { decision, evidence: approvedEvidence, workerPass: pass };
-            }
-            if (isHeadlessWorthTrying(pass, approvedEvidence)) {
-                const url = pass.suggestedHeadlessUrl;
-                await headlessFetch({ url });
-                const decision = {
-                    action: 'escalate-headless',
-                    rationale: 'One high-value page is worth a single orchestrator-approved headless retry.',
-                    url,
-                    approvedEvidence
-                };
-                return { decision, evidence: approvedEvidence, workerPass: pass };
-            }
-            const hasConcreteGap = pass.gaps.length > 0;
-            const onlyLowValueOutcomes = pass.lowValueOutcomes.length > 0 && pass.evidence.length === 0;
-            if (!hasConcreteGap || onlyLowValueOutcomes) {
-                const decision = {
-                    action: 'research-again',
-                    rationale: 'Current results did not justify more escalation; continue only with a more targeted pass.',
-                    followupQuery: query
-                };
-                return { decision, evidence: approvedEvidence, workerPass: pass };
-            }
-            const decision = {
-                action: 'research-again',
-                rationale: 'The first pass did not gather enough strong evidence to answer safely.',
-                followupQuery: query
-            };
-            return { decision, evidence: approvedEvidence, workerPass: pass };
-        }
-    };
-}

package/dist/src/orchestration/research-types.d.ts

@@ -1,41 +0,0 @@
-export type ResearchSourceKind = 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
-export type ResearchMethod = 'search' | 'http' | 'headless';
-export type ResearchEvidence = {
-    title: string;
-    url: string;
-    sourceKind: ResearchSourceKind;
-    method: Exclude<ResearchMethod, 'search'>;
-    summary: string;
-    supports: string[];
-};
-export type ResearchGap = {
-    kind: 'needs-more-evidence' | 'conflict' | 'fetch-failed';
-    message: string;
-};
-export type ResearchLowValueOutcome = {
-    kind: 'empty-search' | 'bot-check' | 'low-value-page' | 'duplicate-evidence';
-    url?: string;
-    message: string;
-};
-export type ResearchWorkerResult = {
-    searchQueries: string[];
-    evidence: ResearchEvidence[];
-    gaps: ResearchGap[];
-    lowValueOutcomes: ResearchLowValueOutcome[];
-    suggestedHeadlessUrl?: string;
-    exhaustedBudget: boolean;
-};
-export type ResearchOrchestratorDecision = {
-    action: 'answer';
-    rationale: string;
-    approvedEvidence: ResearchEvidence[];
-} | {
-    action: 'research-again';
-    rationale: string;
-    followupQuery: string;
-} | {
-    action: 'escalate-headless';
-    rationale: string;
-    url: string;
-    approvedEvidence: ResearchEvidence[];
-};

package/dist/src/orchestration/research-types.js

@@ -1 +0,0 @@
-export {};

package/dist/src/orchestration/research-worker.d.ts

@@ -1,16 +0,0 @@
-import type { WebFetchResponse, WebSearchResponse } from '../types.js';
-import type { ResearchWorkerResult } from './research-types.js';
-export declare function createResearchWorker({ search, fetchPage }: {
-    search: (input: {
-        query: string;
-    }) => Promise<WebSearchResponse>;
-    fetchPage: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
-}): {
-    run({ query, maxSearchRounds, maxFetches }: {
-        query: string;
-        maxSearchRounds: number;
-        maxFetches: number;
-    }): Promise<ResearchWorkerResult>;
-};