@demigodmode/pi-web-agent 0.2.1 → 0.2.2

package/dist/src/orchestration/research-orchestrator.js

@@ -0,0 +1,87 @@
+function sourceRank(sourceKind) {
+    switch (sourceKind) {
+        case 'official-docs':
+            return 0;
+        case 'official-api':
+            return 1;
+        case 'official-discussion':
+            return 2;
+        case 'issue-thread':
+            return 3;
+        case 'community':
+            return 4;
+        case 'package-page':
+            return 5;
+        default:
+            return 6;
+    }
+}
+function sortEvidence(evidence) {
+    return [...evidence].sort((left, right) => sourceRank(left.sourceKind) - sourceRank(right.sourceKind));
+}
+function strongEvidence(evidence) {
+    return evidence.filter((item) => item.sourceKind === 'official-docs' ||
+        item.sourceKind === 'official-api' ||
+        item.sourceKind === 'official-discussion');
+}
+function hasOfficialDocsOrApi(evidence) {
+    return evidence.some((item) => item.sourceKind === 'official-docs' || item.sourceKind === 'official-api');
+}
+function hasBotCheck(outcomes) {
+    return outcomes.some((outcome) => outcome.kind === 'bot-check');
+}
+function isHeadlessWorthTrying(pass, approvedEvidence) {
+    if (!pass.suggestedHeadlessUrl)
+        return false;
+    if (hasBotCheck(pass.lowValueOutcomes))
+        return false;
+    if (approvedEvidence.length >= 2 && hasOfficialDocsOrApi(approvedEvidence))
+        return false;
+    const candidate = pass.suggestedHeadlessUrl;
+    return !candidate.includes('npmjs.com/package/');
+}
+export function createResearchOrchestrator({ worker, headlessFetch }) {
+    return {
+        async run({ query }) {
+            const pass = await worker.run({ query, maxSearchRounds: 1, maxFetches: 3 });
+            const approvedEvidence = sortEvidence(pass.evidence.filter((item) => item.sourceKind !== 'package-page'));
+            const strong = strongEvidence(approvedEvidence);
+            const enoughEvidence = strong.length >= 2 && hasOfficialDocsOrApi(approvedEvidence);
+            if (enoughEvidence) {
+                const decision = {
+                    action: 'answer',
+                    rationale: 'Two strong sources with official support are enough to answer safely.',
+                    approvedEvidence
+                };
+                return { decision, evidence: approvedEvidence, workerPass: pass };
+            }
+            if (isHeadlessWorthTrying(pass, approvedEvidence)) {
+                const url = pass.suggestedHeadlessUrl;
+                await headlessFetch({ url });
+                const decision = {
+                    action: 'escalate-headless',
+                    rationale: 'One high-value page is worth a single orchestrator-approved headless retry.',
+                    url,
+                    approvedEvidence
+                };
+                return { decision, evidence: approvedEvidence, workerPass: pass };
+            }
+            const hasConcreteGap = pass.gaps.length > 0;
+            const onlyLowValueOutcomes = pass.lowValueOutcomes.length > 0 && pass.evidence.length === 0;
+            if (!hasConcreteGap || onlyLowValueOutcomes) {
+                const decision = {
+                    action: 'research-again',
+                    rationale: 'Current results did not justify more escalation; continue only with a more targeted pass.',
+                    followupQuery: query
+                };
+                return { decision, evidence: approvedEvidence, workerPass: pass };
+            }
+            const decision = {
+                action: 'research-again',
+                rationale: 'The first pass did not gather enough strong evidence to answer safely.',
+                followupQuery: query
+            };
+            return { decision, evidence: approvedEvidence, workerPass: pass };
+        }
+    };
+}

package/dist/src/orchestration/research-types.d.ts

@@ -0,0 +1,41 @@
+export type ResearchSourceKind = 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
+export type ResearchMethod = 'search' | 'http' | 'headless';
+export type ResearchEvidence = {
+    title: string;
+    url: string;
+    sourceKind: ResearchSourceKind;
+    method: Exclude<ResearchMethod, 'search'>;
+    summary: string;
+    supports: string[];
+};
+export type ResearchGap = {
+    kind: 'needs-more-evidence' | 'conflict' | 'fetch-failed';
+    message: string;
+};
+export type ResearchLowValueOutcome = {
+    kind: 'empty-search' | 'bot-check' | 'low-value-page' | 'duplicate-evidence';
+    url?: string;
+    message: string;
+};
+export type ResearchWorkerResult = {
+    searchQueries: string[];
+    evidence: ResearchEvidence[];
+    gaps: ResearchGap[];
+    lowValueOutcomes: ResearchLowValueOutcome[];
+    suggestedHeadlessUrl?: string;
+    exhaustedBudget: boolean;
+};
+export type ResearchOrchestratorDecision = {
+    action: 'answer';
+    rationale: string;
+    approvedEvidence: ResearchEvidence[];
+} | {
+    action: 'research-again';
+    rationale: string;
+    followupQuery: string;
+} | {
+    action: 'escalate-headless';
+    rationale: string;
+    url: string;
+    approvedEvidence: ResearchEvidence[];
+};

package/dist/src/orchestration/research-types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/orchestration/research-worker.d.ts

@@ -0,0 +1,16 @@
+import type { WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ResearchWorkerResult } from './research-types.js';
+export declare function createResearchWorker({ search, fetchPage }: {
+    search: (input: {
+        query: string;
+    }) => Promise<WebSearchResponse>;
+    fetchPage: (input: {
+        url: string;
+    }) => Promise<WebFetchResponse>;
+}): {
+    run({ query, maxSearchRounds, maxFetches }: {
+        query: string;
+        maxSearchRounds: number;
+        maxFetches: number;
+    }): Promise<ResearchWorkerResult>;
+};

package/dist/src/orchestration/research-worker.js

@@ -0,0 +1,131 @@
+function classifySource(url) {
+    if (url.includes('/docs/api/') || url.includes('/config/'))
+        return 'official-api';
+    if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide/'))
+        return 'official-docs';
+    if (url.includes('learn.microsoft.com'))
+        return 'official-docs';
+    if (url.includes('github.com/') && url.includes('/issues/'))
+        return 'issue-thread';
+    if (url.includes('npmjs.com/package/'))
+        return 'package-page';
+    return 'community';
+}
+function summarizeText(text, maxLength = 180) {
+    return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
+}
+function evidenceFromFetch(fetched, fallbackTitle) {
+    const content = fetched.content;
+    if (fetched.status !== 'ok' || !content)
+        return null;
+    const sourceKind = classifySource(fetched.url);
+    if (sourceKind === 'package-page') {
+        return null;
+    }
+    return {
+        title: content.title ?? fallbackTitle,
+        url: fetched.url,
+        sourceKind,
+        method: fetched.metadata.method,
+        summary: summarizeText(content.text),
+        supports: [summarizeText(content.text, 120)]
+    };
+}
+function lowValueOutcomeFromFetch(fetched) {
+    if (fetched.status !== 'ok' || !fetched.content)
+        return null;
+    if (classifySource(fetched.url) !== 'package-page')
+        return null;
+    return {
+        kind: 'low-value-page',
+        url: fetched.url,
+        message: 'Fetched page did not add strong research evidence.'
+    };
+}
+export function createResearchWorker({ search, fetchPage }) {
+    return {
+        async run({ query, maxSearchRounds, maxFetches }) {
+            const searchQueries = [query];
+            const evidence = [];
+            const gaps = [];
+            const lowValueOutcomes = [];
+            let suggestedHeadlessUrl;
+            if (maxSearchRounds <= 0 || maxFetches <= 0) {
+                return {
+                    searchQueries: [],
+                    evidence,
+                    gaps: [{ kind: 'needs-more-evidence', message: 'Research worker budget was zero.' }],
+                    lowValueOutcomes,
+                    suggestedHeadlessUrl,
+                    exhaustedBudget: true
+                };
+            }
+            const searchResult = await search({ query });
+            if (searchResult.status !== 'ok') {
+                return {
+                    searchQueries,
+                    evidence,
+                    gaps: [
+                        {
+                            kind: 'fetch-failed',
+                            message: searchResult.error?.message ?? 'Search failed during research worker pass.'
+                        }
+                    ],
+                    lowValueOutcomes,
+                    suggestedHeadlessUrl,
+                    exhaustedBudget: false
+                };
+            }
+            if (searchResult.results.length === 0) {
+                return {
+                    searchQueries,
+                    evidence,
+                    gaps,
+                    lowValueOutcomes: [
+                        {
+                            kind: 'empty-search',
+                            message: 'Search returned no results for this pass.'
+                        }
+                    ],
+                    suggestedHeadlessUrl,
+                    exhaustedBudget: false
+                };
+            }
+            const candidates = searchResult.results.slice(0, maxFetches);
+            for (const candidate of candidates) {
+                const fetched = await fetchPage({ url: candidate.url });
+                if (fetched.status === 'ok') {
+                    const parsedEvidence = evidenceFromFetch(fetched, candidate.title);
+                    if (parsedEvidence) {
+                        evidence.push(parsedEvidence);
+                        continue;
+                    }
+                    const lowValueOutcome = lowValueOutcomeFromFetch(fetched);
+                    if (lowValueOutcome) {
+                        lowValueOutcomes.push(lowValueOutcome);
+                    }
+                    continue;
+                }
+                if (fetched.status === 'needs_headless') {
+                    if (!suggestedHeadlessUrl) {
+                        suggestedHeadlessUrl = fetched.url;
+                    }
+                    gaps.push({ kind: 'fetch-failed', message: `HTTP fetch was weak for ${fetched.url}` });
+                    continue;
+                }
+                gaps.push({
+                    kind: 'fetch-failed',
+                    message: fetched.error?.message ?? `Fetch failed for ${candidate.url}`
+                });
+            }
+            return {
+                searchQueries,
+                evidence,
+                gaps,
+                lowValueOutcomes,
+                suggestedHeadlessUrl,
+                exhaustedBudget: false
+            };
+        }
+    };
+}

package/dist/src/search/duckduckgo.d.ts

@@ -0,0 +1,9 @@
+import type { SearchResult } from '../types.js';
+export type ParsedDuckDuckGoResults = {
+    results: SearchResult[];
+    noResults: boolean;
+    hasResultContainers: boolean;
+};
+export declare function buildSearchUrl(query: string): string;
+export declare function fetchDuckDuckGoHtml(query: string): Promise<string>;
+export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;

package/dist/src/search/duckduckgo.js

@@ -0,0 +1,52 @@
+import * as cheerio from 'cheerio';
+function normalizeDuckDuckGoUrl(rawUrl) {
+    try {
+        const absolute = rawUrl.startsWith('//') ? `https:${rawUrl}` : rawUrl;
+        const parsed = new URL(absolute);
+        const isDuckDuckGoRedirect = parsed.hostname === 'duckduckgo.com' && parsed.pathname === '/l/';
+        if (!isDuckDuckGoRedirect) {
+            return rawUrl;
+        }
+        const target = parsed.searchParams.get('uddg');
+        if (!target) {
+            return rawUrl;
+        }
+        return decodeURIComponent(target);
+    }
+    catch {
+        return rawUrl;
+    }
+}
+export function buildSearchUrl(query) {
+    const params = new URLSearchParams({ q: query });
+    return `https://html.duckduckgo.com/html/?${params.toString()}`;
+}
+export async function fetchDuckDuckGoHtml(query) {
+    const response = await fetch(buildSearchUrl(query));
+    if (!response.ok) {
+        throw new Error(`DuckDuckGo request failed with ${response.status}`);
+    }
+    return response.text();
+}
+export function parseDuckDuckGoResults(html) {
+    const $ = cheerio.load(html);
+    const resultContainers = $('.result');
+    const results = resultContainers
+        .map((_, element) => {
+        const title = $(element).find('.result__a').first().text().trim();
+        const url = normalizeDuckDuckGoUrl($(element).find('.result__a').first().attr('href')?.trim() ?? '');
+        const snippet = $(element).find('.result__snippet').first().text().trim();
+        return title && url ? { title, url, snippet } : null;
+    })
+        .get()
+        .filter((value) => value !== null);
+    const text = $.text().toLowerCase();
+    const noResults = text.includes('no results found') ||
+        text.includes('no more results') ||
+        text.includes('did not match any documents');
+    return {
+        results,
+        noResults,
+        hasResultContainers: resultContainers.length > 0
+    };
+}

package/dist/src/tools/web-explore.d.ts

@@ -0,0 +1,44 @@
+import type { ResearchEvidence } from '../orchestration/research-types.js';
+export declare function createWebExploreTool({ explore }?: {
+    explore?: {
+        run: (input: {
+            query: string;
+        }) => Promise<{
+            decision: {
+                action: 'answer' | 'research-again' | 'escalate-headless';
+            };
+            evidence: ResearchEvidence[];
+            workerPass: unknown;
+        }>;
+    } | ((input: {
+        query: string;
+    }) => Promise<{
+        decision: {
+            action: 'answer' | 'research-again' | 'escalate-headless';
+        };
+        evidence: ResearchEvidence[];
+        workerPass: unknown;
+    }>);
+}): ({ query }: {
+    query: string;
+}) => Promise<{
+    status: "error";
+    findings: never[];
+    sources: never[];
+    error: {
+        code: string;
+        message: string;
+    };
+    caveat?: undefined;
+    text?: undefined;
+} | {
+    status: "ok";
+    findings: string[];
+    sources: {
+        title: string;
+        url: string;
+    }[];
+    caveat: string | undefined;
+    text: string;
+    error?: undefined;
+}>;

package/dist/src/tools/web-explore.js

@@ -0,0 +1,50 @@
+import { createResearchWorkflow } from '../orchestration/index.js';
+function findingFromEvidence(evidence, index) {
+    if (evidence.summary.includes('Use channel')) {
+        return 'Use channel for branded Chrome or Edge when possible.';
+    }
+    if (evidence.summary.includes('use at your own risk') || evidence.summary.includes('risky')) {
+        return 'Treat executablePath as a fallback because Playwright documents it as use-at-your-own-risk.';
+    }
+    if (evidence.summary.includes('coverage.provider to v8') ||
+        evidence.summary.includes('@vitest/coverage-v8')) {
+        return 'Vitest coverage docs say to set coverage.provider to v8 and install @vitest/coverage-v8.';
+    }
+    return evidence.summary || `Finding ${index + 1}`;
+}
+function formatExploreText({ findings, sources, caveat }) {
+    const findingLines = findings.map((finding) => `- ${finding}`).join('\n');
+    const sourceLines = sources.map((source) => `- ${source.title}: ${source.url}`).join('\n');
+    const caveatBlock = caveat ? `\n\nCaveat\n${caveat}` : '';
+    return `Findings\n${findingLines}\n\nSources\n${sourceLines}${caveatBlock}`;
+}
+export function createWebExploreTool({ explore = createResearchWorkflow() } = {}) {
+    const runExplore = typeof explore === 'function' ? explore : explore.run.bind(explore);
+    return async function webExplore({ query }) {
+        const normalizedQuery = query.trim();
+        if (!normalizedQuery) {
+            return {
+                status: 'error',
+                findings: [],
+                sources: [],
+                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
+            };
+        }
+        const result = await runExplore({ query: normalizedQuery });
+        const findings = result.evidence.slice(0, 5).map(findingFromEvidence);
+        const sources = result.evidence.slice(0, 4).map((item) => ({
+            title: item.title,
+            url: item.url
+        }));
+        const caveat = result.decision.action === 'answer'
+            ? undefined
+            : 'Evidence is partial, so this answer is based on the strongest source found so far.';
+        return {
+            status: 'ok',
+            findings,
+            sources,
+            caveat,
+            text: formatExploreText({ findings, sources, caveat })
+        };
+    };
+}

package/dist/src/tools/web-fetch-headless.d.ts

@@ -0,0 +1,6 @@
+import type { WebFetchHeadlessResponse } from '../types.js';
+export declare function createWebFetchHeadlessTool({ fetchPage }?: {
+    fetchPage?: (url: string) => Promise<WebFetchHeadlessResponse>;
+}): ({ url }: {
+    url: string;
+}) => Promise<WebFetchHeadlessResponse>;

package/dist/src/tools/web-fetch-headless.js

@@ -0,0 +1,14 @@
+import { headlessFetch } from '../fetch/headless-fetch.js';
+export function createWebFetchHeadlessTool({ fetchPage = headlessFetch } = {}) {
+    return async function webFetchHeadless({ url }) {
+        if (!/^https?:\/\//.test(url)) {
+            return {
+                status: 'unsupported',
+                url,
+                metadata: { method: 'headless', cacheHit: false },
+                error: { code: 'UNSUPPORTED_URL', message: 'Only http and https URLs are supported.' }
+            };
+        }
+        return fetchPage(url);
+    };
+}

package/dist/src/tools/web-fetch.d.ts

@@ -0,0 +1,6 @@
+import type { WebFetchResponse } from '../types.js';
+export declare function createWebFetchTool({ fetchPage }?: {
+    fetchPage?: (url: string) => Promise<WebFetchResponse>;
+}): ({ url }: {
+    url: string;
+}) => Promise<WebFetchResponse>;

package/dist/src/tools/web-fetch.js

@@ -0,0 +1,14 @@
+import { createHttpFetcher } from '../fetch/http-fetch.js';
+export function createWebFetchTool({ fetchPage = createHttpFetcher() } = {}) {
+    return async function webFetch({ url }) {
+        if (!/^https?:\/\//.test(url)) {
+            return {
+                status: 'unsupported',
+                url,
+                metadata: { method: 'http', cacheHit: false },
+                error: { code: 'UNSUPPORTED_URL', message: 'Only http and https URLs are supported.' }
+            };
+        }
+        return fetchPage(url);
+    };
+}

package/dist/src/tools/web-search.d.ts

@@ -0,0 +1,10 @@
+import type { WebSearchResponse } from '../types.js';
+export declare function createWebSearchTool({ searchHtml, cache }?: {
+    searchHtml?: (query: string) => Promise<string>;
+    cache?: {
+        get(key: string): WebSearchResponse | undefined;
+        set(key: string, value: WebSearchResponse): void;
+    };
+}): ({ query }: {
+    query: string;
+}) => Promise<WebSearchResponse>;

package/dist/src/tools/web-search.js

@@ -0,0 +1,103 @@
+import { createCacheKey, createTtlCache } from '../cache/ttl-cache.js';
+import { fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
+function classifySearchFailure(error) {
+    const rawMessage = error instanceof Error ? error.message : 'Unknown search failure.';
+    const normalized = rawMessage.toLowerCase();
+    if (normalized.includes('blocked') ||
+        normalized.includes('rate limit') ||
+        normalized.includes('rate-limit') ||
+        normalized.includes('403') ||
+        normalized.includes('429') ||
+        normalized.includes('captcha') ||
+        normalized.includes('challenge')) {
+        return {
+            code: 'BLOCKED',
+            message: 'DuckDuckGo search appears to be blocked or rate limited.'
+        };
+    }
+    return {
+        code: 'FETCH_FAILED',
+        message: `DuckDuckGo search request failed: ${rawMessage}`
+    };
+}
+function htmlLooksBlocked(html) {
+    const normalized = html.toLowerCase();
+    return (normalized.includes('captcha') ||
+        normalized.includes('challenge') ||
+        normalized.includes('verify you are human') ||
+        normalized.includes('are you a robot') ||
+        normalized.includes('unusual traffic'));
+}
+export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache = createTtlCache({ ttlMs: 30_000 }) } = {}) {
+    return async function webSearch({ query }) {
+        const normalizedQuery = query.trim();
+        if (!normalizedQuery) {
+            return {
+                status: 'error',
+                results: [],
+                metadata: { backend: 'duckduckgo', cacheHit: false },
+                error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
+            };
+        }
+        const cacheKey = createCacheKey(['web_search', normalizedQuery]);
+        const cached = cache.get(cacheKey);
+        if (cached) {
+            return {
+                ...cached,
+                metadata: { ...cached.metadata, cacheHit: true }
+            };
+        }
+        try {
+            const html = await searchHtml(normalizedQuery);
+            const parsed = parseDuckDuckGoResults(html);
+            if (parsed.results.length > 0) {
+                const result = {
+                    status: 'ok',
+                    results: parsed.results,
+                    metadata: { backend: 'duckduckgo', cacheHit: false }
+                };
+                cache.set(cacheKey, result);
+                return result;
+            }
+            if (parsed.noResults) {
+                return {
+                    status: 'error',
+                    results: [],
+                    metadata: { backend: 'duckduckgo', cacheHit: false },
+                    error: {
+                        code: 'NO_RESULTS',
+                        message: 'DuckDuckGo returned no usable results for this query.'
+                    }
+                };
+            }
+            if (htmlLooksBlocked(html)) {
+                return {
+                    status: 'error',
+                    results: [],
+                    metadata: { backend: 'duckduckgo', cacheHit: false },
+                    error: {
+                        code: 'BLOCKED',
+                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
+                    }
+                };
+            }
+            return {
+                status: 'error',
+                results: [],
+                metadata: { backend: 'duckduckgo', cacheHit: false },
+                error: {
+                    code: 'PARSE_FAILED',
+                    message: 'DuckDuckGo returned a page, but it did not match the expected results format.'
+                }
+            };
+        }
+        catch (error) {
+            return {
+                status: 'error',
+                results: [],
+                metadata: { backend: 'duckduckgo', cacheHit: false },
+                error: classifySearchFailure(error)
+            };
+        }
+    };
+}

package/dist/src/types.d.ts

@@ -0,0 +1,48 @@
+export declare const TOOL_STATUSES: readonly ["ok", "needs_headless", "blocked", "unsupported", "error"];
+export type ToolStatus = (typeof TOOL_STATUSES)[number];
+export type SearchResult = {
+    title: string;
+    url: string;
+    snippet: string;
+};
+export type ToolError = {
+    code: string;
+    message: string;
+};
+export type SearchMetadata = {
+    backend: 'duckduckgo';
+    cacheHit: boolean;
+};
+export type FetchMetadata = {
+    method: 'http' | 'headless';
+    cacheHit: boolean;
+    contentType?: string;
+    truncated?: boolean;
+    browser?: 'configured' | 'chrome' | 'edge';
+    navigationMs?: number;
+};
+export type ExtractedContent = {
+    title?: string;
+    byline?: string;
+    text: string;
+};
+export type WebSearchResponse = {
+    status: 'ok' | 'error';
+    results: SearchResult[];
+    metadata: SearchMetadata;
+    error?: ToolError;
+};
+export type WebFetchResponse = {
+    status: ToolStatus;
+    url: string;
+    content?: ExtractedContent;
+    metadata: FetchMetadata;
+    error?: ToolError;
+};
+export type WebFetchHeadlessResponse = {
+    status: Exclude<ToolStatus, 'needs_headless'>;
+    url: string;
+    content?: ExtractedContent;
+    metadata: FetchMetadata;
+    error?: ToolError;
+};

package/dist/src/types.js

@@ -0,0 +1,7 @@
+export const TOOL_STATUSES = [
+    'ok',
+    'needs_headless',
+    'blocked',
+    'unsupported',
+    'error'
+];

package/dist/tests/cache/ttl-cache.test.d.ts

@@ -0,0 +1 @@
+export {};