@demigodmode/pi-web-agent 1.2.0 → 1.4.0

package/CHANGELOG.md

@@ -18,6 +18,36 @@ The format is intentionally simple and release-oriented.
 ### Breaking
 - None.
 
+## [1.4.0] - 2026-06-09
+### Added
+- Added evidence quality analysis for `web_explore`, including source diversity, unreadable source, bot-check, and possible conflict signals.
+
+### Changed
+- `web_explore` now uses source quality signals before deciding whether to answer, search again, or answer with a caveat.
+- Partial research caveats are now more specific when evidence is community-only, low-diversity, blocked, or cautionary.
+
+### Fixed
+- Nothing yet.
+
+### Breaking
+- None.
+
+## [1.3.0] - 2026-06-04
+### Added
+- Added direct URL handling in `web_explore` so linked pages are read before search results.
+- Added forum/thread source classification for Reddit-style discussions, forums, Stack Overflow, and GitHub issues/discussions.
+- Added Playwright-managed Chromium fallback when no local Chromium-family browser is detected.
+
+### Changed
+- Discussion-oriented queries now prefer forum/thread results over generic pages.
+- `/web-agent doctor` now reports the local-browser headless backend and managed Chromium fallback.
+
+### Fixed
+- Preserved direct/thread fetch gaps in bounded research results so unreadable thread sources get explicit caveats.
+
+### Breaking
+- None.
+
 ## [1.2.0] - 2026-06-01
 ### Added
 - Added backend provider and fallback editing to `/web-agent settings`.

package/README.md

@@ -8,7 +8,7 @@
 
 Most agent web tools blur search, fetch, browser rendering, and research into one vague thing. `pi-web-agent` exposes one public research tool, `web_explore`, and keeps search/fetch/headless work inside that bounded workflow.
 
-The point is keeping the model-facing boundary simple: ask `web_explore` to research a question, and it handles discovery, HTTP reads, targeted browser rendering, source ranking, and caveats internally.
+The point is keeping the model-facing boundary simple: ask `web_explore` to research a question, and it handles direct links, discovery, HTTP reads, targeted browser rendering, source ranking, and caveats internally.
 
 That sounds obvious, but a lot of agent tooling gets fuzzy right there. This package is meant to be stricter about what it actually did and more willing to say when a read was not good enough to trust.
 
@@ -20,9 +20,9 @@ Compatibility notice: current `pi-web-agent` requires Pi 0.74+ because Pi packag
 pi install npm:@demigodmode/pi-web-agent
 ```
 
-After installing, reload or restart Pi. Run `/web-agent` for the action menu, or `/web-agent doctor` to check whether the package loaded cleanly and whether headless rendering can find a browser.
+After installing, reload or restart Pi. Run `/web-agent` for the action menu, or `/web-agent doctor` to check whether the package loaded cleanly and which web backends are configured.
 
-Headless rendering currently requires a detectable Chromium-family browser: Chrome, Chromium, Edge, or Brave. Firefox/Safari-only systems can still use search and plain HTTP reads, but browser-rendered fallback pages need a supported Chromium-family browser for now.
+Headless rendering first tries a detectable Chromium-family browser: Chrome, Chromium, Edge, or Brave. If none is found, it falls back to Playwright-managed Chromium and still launches headless. Firefox/Safari-only systems can still use search and plain HTTP reads; browser-rendered fallback uses Chromium.
 
 Later on, update installed packages with:
 
@@ -109,7 +109,7 @@ Example:
 }
 ```
 
-Backend config is also supported. Defaults remain DuckDuckGo search, plain HTTP fetch, and local browser headless fallback.
+Backend config is also supported. Defaults remain DuckDuckGo search, plain HTTP fetch, and local-browser headless fallback with managed Chromium fallback configured.
 
 Backend settings can be changed from:
 

package/dist/backends/doctor.js

@@ -83,5 +83,8 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
     if (config.fetch.fallback) {
         lines.push(`fetch fallback: ${config.fetch.fallback}`);
     }
+    if (config.headless.provider === 'local-browser') {
+        lines.push('headless backend: local-browser (managed Chromium fallback configured)');
+    }
     return lines;
 }

package/dist/fetch/headless-fetch.d.ts

@@ -6,7 +6,8 @@ export declare function headlessFetch(url: string, { configuredPath, resolveBrow
         configuredPath?: string;
     }) => Promise<BrowserResolutionResult>;
     launchBrowser?: (options: {
-        executablePath: string;
+        executablePath?: string;
+        headless: true;
     }) => Promise<{
         newContext: () => Promise<{
             newPage: () => Promise<any>;

package/dist/fetch/headless-fetch.js

@@ -1,4 +1,4 @@
-import { chromium } from 'playwright-core';
+import { chromium } from 'playwright';
 import { extractReadableContentSafely } from '../extract/readability.js';
 import { resolveBrowserExecutable } from './browser-resolution.js';
 function cleanupRenderedText(text) {
@@ -7,9 +7,9 @@ function cleanupRenderedText(text) {
     cleaned = cleaned.replace(/\s+/g, ' ').trim();
     return cleaned;
 }
-export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath }) => chromium.launch({ executablePath, headless: true }), now = () => Date.now() } = {}) {
+export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless }) => chromium.launch(executablePath ? { executablePath, headless } : { headless }), now = () => Date.now() } = {}) {
     const resolved = await resolveBrowser({ configuredPath });
-    if (!resolved.ok) {
+    if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
         return {
             status: 'error',
             url,
@@ -17,11 +17,15 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
             error: resolved.error
         };
     }
+    const browserName = resolved.ok ? resolved.browser : 'chromium';
+    const launchOptions = resolved.ok
+        ? { executablePath: resolved.executablePath, headless: true }
+        : { headless: true };
     let browser;
     let context;
     let page;
     try {
-        browser = await launchBrowser({ executablePath: resolved.executablePath });
+        browser = await launchBrowser(launchOptions);
         context = await browser.newContext();
         page = await context.newPage();
         const startedAt = now();
@@ -42,7 +46,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
                 metadata: {
                     method: 'headless',
                     cacheHit: false,
-                    browser: resolved.browser,
+                    browser: browserName,
                     navigationMs: finishedAt - startedAt
                 },
                 error: {
@@ -58,7 +62,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
             metadata: {
                 method: 'headless',
                 cacheHit: false,
-                browser: resolved.browser,
+                browser: browserName,
                 navigationMs: finishedAt - startedAt,
                 truncated: cleanedContent.text.length >= 4000
             }
@@ -71,7 +75,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
             metadata: {
                 method: 'headless',
                 cacheHit: false,
-                browser: resolved.browser
+                browser: browserName
             },
             error: {
                 code: 'HEADLESS_NAVIGATION_FAILED',

package/dist/orchestration/answer-synthesizer.d.ts

@@ -1,7 +1,9 @@
 import type { ResearchEvidence } from './research-types.js';
-export declare function synthesizeAnswer({ evidence, partial }: {
+import type { EvidenceCaveatReason } from './evidence-quality.js';
+export declare function synthesizeAnswer({ evidence, partial, caveatReasons }: {
     evidence: ResearchEvidence[];
     partial: boolean;
+    caveatReasons?: EvidenceCaveatReason[];
 }): {
     findings: string[];
     caveat: string | undefined;

package/dist/orchestration/answer-synthesizer.js

@@ -1,7 +1,39 @@
 function normalizeSummary(summary) {
     return summary.replace(/\s+/g, ' ').trim();
 }
-export function synthesizeAnswer({ evidence, partial }) {
+function sentenceForReason(reason) {
+    switch (reason) {
+        case 'community-only':
+            return 'the strongest readable sources were mostly community/practical context';
+        case 'low-diversity':
+            return 'the source set was narrow';
+        case 'unreadable-direct-source':
+            return 'one or more linked sources could not be read reliably';
+        case 'unreadable-thread-source':
+            return 'one or more thread sources could not be read reliably';
+        case 'possible-conflict':
+            return 'readable sources include cautionary or possibly conflicting guidance';
+        case 'bot-check':
+            return 'some candidate sources showed bot-check or security verification pages';
+    }
+}
+function joinReasons(reasons) {
+    if (reasons.length === 0)
+        return '';
+    if (reasons.length === 1)
+        return reasons[0];
+    return `${reasons.slice(0, -1).join(', ')}, and ${reasons.at(-1)}`;
+}
+function caveatText(partial, caveatReasons = []) {
+    if (!partial)
+        return undefined;
+    const specificReasons = caveatReasons.map(sentenceForReason);
+    if (specificReasons.length > 0) {
+        return `Evidence is partial: ${joinReasons(specificReasons)}.`;
+    }
+    return 'Evidence is partial, so this answer is based on the strongest source found within the bounded research budget.';
+}
+export function synthesizeAnswer({ evidence, partial, caveatReasons = [] }) {
     const findings = evidence.slice(0, 5).map((item) => {
         const summary = normalizeSummary(item.summary);
         return item.sourceKind === 'community' || item.sourceKind === 'issue-thread'
@@ -10,8 +42,6 @@ export function synthesizeAnswer({ evidence, partial }) {
     });
     return {
         findings,
-        caveat: partial
-            ? 'Evidence is partial, so this answer is based on the strongest source found within the bounded research budget.'
-            : undefined
+        caveat: caveatText(partial, caveatReasons)
     };
 }

package/dist/orchestration/candidate-selector.d.ts

@@ -1,5 +1,6 @@
 import type { SearchResult } from '../types.js';
-export declare function selectCandidates({ results, seenUrls, maxCandidates }: {
+export declare function selectCandidates({ query, results, seenUrls, maxCandidates }: {
+    query?: string;
     results: SearchResult[];
     seenUrls: Set<string>;
     maxCandidates: number;

package/dist/orchestration/candidate-selector.js

@@ -1,16 +1,35 @@
-function candidateScore(result) {
+import { classifySourceProfile } from './source-profile.js';
+function wantsDiscussionSources(query = '') {
+    return /reddit|forum|forums|discussion|thread|comments|community|user experience|people recommend/i.test(query);
+}
+function candidateScore(result, query) {
     const url = result.url.toLowerCase();
-    if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide') || url.includes('learn.microsoft.com'))
+    const profile = classifySourceProfile(result.url);
+    const wantsThreads = wantsDiscussionSources(query);
+    if (profile.kind === 'official-docs')
         return 0;
-    if (url.includes('github.com/') && (url.includes('/issues/') || url.includes('/discussions/')))
+    if (profile.kind === 'official-api')
         return 1;
-    if (url.includes('github.com/'))
+    if (wantsThreads) {
+        if (profile.kind === 'forum-thread')
+            return 2;
+        if (profile.kind === 'issue-thread')
+            return 3;
+        if (url.includes('github.com/'))
+            return 4;
+        if (profile.kind === 'package-page')
+            return 6;
+        return 5;
+    }
+    if (profile.kind === 'issue-thread')
         return 2;
-    if (url.includes('npmjs.com/package/'))
-        return 4;
-    return 3;
+    if (url.includes('github.com/'))
+        return 3;
+    if (profile.kind === 'package-page')
+        return 5;
+    return 4;
 }
-export function selectCandidates({ results, seenUrls, maxCandidates }) {
+export function selectCandidates({ query, results, seenUrls, maxCandidates }) {
     const deduped = new Map();
     for (const result of results) {
         if (seenUrls.has(result.url))
@@ -19,6 +38,6 @@ export function selectCandidates({ results, seenUrls, maxCandidates }) {
             deduped.set(result.url, result);
     }
     return [...deduped.values()]
-        .sort((left, right) => candidateScore(left) - candidateScore(right))
+        .sort((left, right) => candidateScore(left, query) - candidateScore(right, query))
         .slice(0, maxCandidates);
 }

package/dist/orchestration/direct-url.d.ts

@@ -0,0 +1 @@
+export declare function extractDirectUrls(query: string): string[];

package/dist/orchestration/direct-url.js

@@ -0,0 +1,47 @@
+const TRACKING_PARAMS = new Set([
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_term',
+    'utm_content',
+    'utm_name',
+    'fbclid',
+    'gclid'
+]);
+function stripTrailingPunctuation(raw) {
+    let next = raw.trim();
+    while (/[),.;!?\]]$/.test(next)) {
+        const last = next.at(-1);
+        if (last === ')' && next.includes('(') && next.lastIndexOf('(') > next.lastIndexOf(')'))
+            break;
+        next = next.slice(0, -1);
+    }
+    return next;
+}
+function normalizeDirectUrl(raw) {
+    try {
+        const url = new URL(stripTrailingPunctuation(raw));
+        if (url.protocol !== 'http:' && url.protocol !== 'https:')
+            return undefined;
+        for (const key of [...url.searchParams.keys()]) {
+            if (TRACKING_PARAMS.has(key.toLowerCase())) {
+                url.searchParams.delete(key);
+            }
+        }
+        url.hash = '';
+        return url.toString().replace(/\/$/, '');
+    }
+    catch {
+        return undefined;
+    }
+}
+export function extractDirectUrls(query) {
+    const matches = query.match(/https?:\/\/\S+/gi) ?? [];
+    const urls = new Set();
+    for (const match of matches) {
+        const normalized = normalizeDirectUrl(match);
+        if (normalized)
+            urls.add(normalized);
+    }
+    return [...urls];
+}

package/dist/orchestration/evidence-quality.d.ts

@@ -0,0 +1,26 @@
+import type { ResearchEvidence, ResearchGap, ResearchLowValueOutcome } from './research-types.js';
+export type EvidenceCaveatReason = 'community-only' | 'low-diversity' | 'unreadable-direct-source' | 'unreadable-thread-source' | 'possible-conflict' | 'bot-check';
+export type EvidenceQualityReport = {
+    counts: {
+        total: number;
+        official: number;
+        community: number;
+        thread: number;
+        packagePage: number;
+        distinctHosts: number;
+    };
+    flags: {
+        hasOfficialEvidence: boolean;
+        hasOnlyCommunityEvidence: boolean;
+        hasLowDiversity: boolean;
+        hasUnreadableDirectSource: boolean;
+        hasUnreadableThreadSource: boolean;
+        hasPossibleConflict: boolean;
+    };
+    caveatReasons: EvidenceCaveatReason[];
+};
+export declare function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }: {
+    evidence: ResearchEvidence[];
+    gaps: ResearchGap[];
+    lowValueOutcomes: ResearchLowValueOutcome[];
+}): EvidenceQualityReport;

package/dist/orchestration/evidence-quality.js

@@ -0,0 +1,62 @@
+function hostname(url) {
+    try {
+        return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+    }
+    catch {
+        return url.toLowerCase();
+    }
+}
+function hasConflictMarkers(evidence) {
+    const combined = evidence
+        .flatMap((item) => [item.summary, ...item.supports])
+        .join('\n')
+        .toLowerCase();
+    const caution = /\bdeprecated\b|not recommended|use at your own risk|should not/.test(combined);
+    const positiveText = combined.replace(/not recommended/g, '');
+    const positive = /\brecommended\b/.test(positiveText);
+    return positive && caution;
+}
+function addReason(reasons, reason, enabled) {
+    if (enabled && !reasons.includes(reason))
+        reasons.push(reason);
+}
+export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
+    const official = evidence.filter((item) => item.sourceKind === 'official-docs' || item.sourceKind === 'official-api').length;
+    const community = evidence.filter((item) => item.sourceKind === 'community').length;
+    const thread = evidence.filter((item) => item.sourceKind === 'issue-thread' || item.sourceKind === 'official-discussion').length;
+    const packagePage = evidence.filter((item) => item.sourceKind === 'package-page').length;
+    const distinctHosts = new Set(evidence.map((item) => hostname(item.url))).size;
+    const hasOfficialEvidence = official > 0;
+    const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0;
+    const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1;
+    const hasUnreadableDirectSource = gaps.some((gap) => /Direct URL could not be read reliably/i.test(gap.message));
+    const hasUnreadableThreadSource = gaps.some((gap) => /Thread source could not be read reliably/i.test(gap.message));
+    const hasPossibleConflict = hasConflictMarkers(evidence);
+    const hasBotCheck = lowValueOutcomes.some((outcome) => outcome.kind === 'bot-check');
+    const caveatReasons = [];
+    addReason(caveatReasons, 'community-only', hasOnlyCommunityEvidence);
+    addReason(caveatReasons, 'low-diversity', hasLowDiversity);
+    addReason(caveatReasons, 'unreadable-direct-source', hasUnreadableDirectSource);
+    addReason(caveatReasons, 'unreadable-thread-source', hasUnreadableThreadSource);
+    addReason(caveatReasons, 'possible-conflict', hasPossibleConflict);
+    addReason(caveatReasons, 'bot-check', hasBotCheck);
+    return {
+        counts: {
+            total: evidence.length,
+            official,
+            community,
+            thread,
+            packagePage,
+            distinctHosts
+        },
+        flags: {
+            hasOfficialEvidence,
+            hasOnlyCommunityEvidence,
+            hasLowDiversity,
+            hasUnreadableDirectSource,
+            hasUnreadableThreadSource,
+            hasPossibleConflict
+        },
+        caveatReasons
+    };
+}

package/dist/orchestration/index.d.ts

@@ -23,6 +23,7 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
             fetchedPages: number;
             headlessAttempts: number;
             exhaustedBudget: boolean;
+            caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
         };
     }>;
 };

package/dist/orchestration/index.js

@@ -7,5 +7,9 @@ export function createResearchWorkflow({ backendConfig, search, fetchPage, headl
     const resolvedFetchPage = fetchPage ?? backends.fetchPage;
     const resolvedHeadlessFetch = headlessFetch ?? backends.headlessFetch;
     const worker = createResearchWorker({ search: resolvedSearch, fetchPage: resolvedFetchPage });
-    return createResearchOrchestrator({ worker, headlessFetch: resolvedHeadlessFetch });
+    return createResearchOrchestrator({
+        worker,
+        fetchDirect: resolvedFetchPage,
+        headlessFetch: resolvedHeadlessFetch
+    });
 }

package/dist/orchestration/research-orchestrator.d.ts

@@ -1,6 +1,7 @@
-import type { WebFetchHeadlessResponse } from '../types.js';
+import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
 import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
-export declare function createResearchOrchestrator({ worker, headlessFetch }: {
+import { type EvidenceCaveatReason } from './evidence-quality.js';
+export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
     worker: {
         run: (input: {
             query: string;
@@ -8,6 +9,9 @@ export declare function createResearchOrchestrator({ worker, headlessFetch }: {
             maxFetches: number;
         }) => Promise<ResearchWorkerResult>;
     };
+    fetchDirect?: (input: {
+        url: string;
+    }) => Promise<WebFetchResponse>;
     headlessFetch: (input: {
         url: string;
     }) => Promise<WebFetchHeadlessResponse>;
@@ -23,6 +27,7 @@ export declare function createResearchOrchestrator({ worker, headlessFetch }: {
             fetchedPages: number;
             headlessAttempts: number;
             exhaustedBudget: boolean;
+            caveatReasons: EvidenceCaveatReason[];
         };
     }>;
 };

package/dist/orchestration/research-orchestrator.js

@@ -1,23 +1,14 @@
 import { rankEvidence } from './evidence-ranker.js';
 import { planSearchQueries } from './query-planner.js';
+import { classifySourceProfile } from './source-profile.js';
+import { extractDirectUrls } from './direct-url.js';
 import { decideNextResearchStep } from './stop-decider.js';
+import { analyzeEvidenceQuality } from './evidence-quality.js';
 const DEFAULT_MAX_PASSES = 3;
 const DEFAULT_MAX_FETCHES_PER_PASS = 4;
 const DEFAULT_MAX_HEADLESS_ATTEMPTS = 2;
 function classifyEvidenceUrl(url) {
-    if (url.includes('/docs/api/') || url.includes('/config/'))
-        return 'official-api';
-    if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide/'))
-        return 'official-docs';
-    if (url.includes('github.com/vitest-dev/vitest') && url.includes('/docs/'))
-        return 'official-docs';
-    if (url.includes('learn.microsoft.com'))
-        return 'official-docs';
-    if (url.includes('github.com/') && url.includes('/issues/'))
-        return 'issue-thread';
-    if (url.includes('npmjs.com/package/'))
-        return 'package-page';
-    return 'community';
+    return classifySourceProfile(url).sourceKind;
 }
 function summarizeText(text, maxLength = 180) {
     return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
@@ -25,6 +16,20 @@ function summarizeText(text, maxLength = 180) {
 function isBotCheckContent({ title = '', text }) {
     return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
 }
+function evidenceFromFetch(result) {
+    if (result.status !== 'ok' || !result.content?.text.trim())
+        return null;
+    if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
+        return null;
+    return {
+        title: result.content.title ?? result.url,
+        url: result.url,
+        sourceKind: classifyEvidenceUrl(result.url),
+        method: result.metadata.method,
+        summary: summarizeText(result.content.text),
+        supports: [summarizeText(result.content.text, 120)]
+    };
+}
 function evidenceFromHeadless(result) {
     if (result.status !== 'ok' || !result.content?.text.trim())
         return null;
@@ -39,24 +44,38 @@ function evidenceFromHeadless(result) {
         supports: [summarizeText(result.content.text, 120)]
     };
 }
-function fallbackWorkerPass({ previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget }) {
+function combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget }) {
     return {
-        searchQueries: previousQueries,
-        evidence: [],
+        searchQueries: lastPass?.searchQueries ?? previousQueries,
+        evidence: lastPass?.evidence ?? [],
         gaps: allGaps,
         lowValueOutcomes: allLowValueOutcomes,
+        suggestedHeadlessUrl: lastPass?.suggestedHeadlessUrl,
         exhaustedBudget
     };
 }
-function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget }) {
+function directUnreadableMessage(url) {
+    return classifySourceProfile(url).kind === 'forum-thread'
+        ? `Thread source could not be read reliably: ${url}`
+        : `Direct URL could not be read reliably: ${url}`;
+}
+function shouldRetryDirectWithHeadless(result, evidence) {
+    if (result.status === 'needs_headless')
+        return true;
+    if (result.status !== 'ok' || evidence)
+        return false;
+    return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
+}
+function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [] }) {
     return {
         searchPasses: previousQueries.length,
         fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
         headlessAttempts,
-        exhaustedBudget
+        exhaustedBudget,
+        caveatReasons
     };
 }
-function decisionForAnswer(action, query, ranked) {
+function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
     if (action === 'answer') {
         return {
             action: 'answer',
@@ -66,11 +85,11 @@ function decisionForAnswer(action, query, ranked) {
     }
     return {
         action: 'research-again',
-        rationale: 'Research budget exhausted; answer with caveat.',
+        rationale: exhaustedBudget ? 'Research budget exhausted; answer with caveat.' : 'Evidence has quality caveats; answer with caveat.',
         followupQuery: query
     };
 }
-export function createResearchOrchestrator({ worker, headlessFetch }) {
+export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }) {
     return {
         async run({ query }) {
             const allEvidence = [];
@@ -80,6 +99,41 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
             const suggestedHeadlessUrls = [];
             let headlessAttempts = 0;
             let lastPass;
+            if (fetchDirect) {
+                for (const url of extractDirectUrls(query).slice(0, 3)) {
+                    const directResult = await fetchDirect({ url });
+                    const directEvidence = evidenceFromFetch(directResult);
+                    if (directEvidence) {
+                        allEvidence.push(directEvidence);
+                        continue;
+                    }
+                    if (shouldRetryDirectWithHeadless(directResult, directEvidence)) {
+                        if (headlessAttempts < DEFAULT_MAX_HEADLESS_ATTEMPTS) {
+                            headlessAttempts++;
+                            const headlessResult = await headlessFetch({ url: directResult.url });
+                            const headlessEvidence = evidenceFromHeadless(headlessResult);
+                            if (headlessEvidence) {
+                                allEvidence.push(headlessEvidence);
+                            }
+                            else {
+                                allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
+                            }
+                        }
+                        else {
+                            allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
+                        }
+                    }
+                    else if (directResult.status !== 'ok') {
+                        allGaps.push({
+                            kind: 'fetch-failed',
+                            message: directResult.error?.message ?? `Direct URL fetch failed for ${directResult.url}`
+                        });
+                    }
+                    else {
+                        allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
+                    }
+                }
+            }
             for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
                 const queries = planSearchQueries({
                     originalQuery: query,
@@ -101,13 +155,19 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
                     if (pass.suggestedHeadlessUrl)
                         suggestedHeadlessUrls.push(pass.suggestedHeadlessUrl);
                     const ranked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
+                    const quality = analyzeEvidenceQuality({
+                        evidence: ranked,
+                        gaps: allGaps,
+                        lowValueOutcomes: allLowValueOutcomes
+                    });
                     const decision = decideNextResearchStep({
                         evidence: ranked,
                         suggestedHeadlessUrls,
                         passIndex,
                         maxPasses: DEFAULT_MAX_PASSES,
                         headlessAttempts,
-                        maxHeadlessAttempts: DEFAULT_MAX_HEADLESS_ATTEMPTS
+                        maxHeadlessAttempts: DEFAULT_MAX_HEADLESS_ATTEMPTS,
+                        quality
                     });
                     if (decision.action === 'headless') {
                         headlessAttempts++;
@@ -116,25 +176,44 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
                         if (headlessEvidence) {
                             allEvidence.push(headlessEvidence);
                             const updatedRanked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
+                            const updatedQuality = analyzeEvidenceQuality({
+                                evidence: updatedRanked,
+                                gaps: allGaps,
+                                lowValueOutcomes: allLowValueOutcomes
+                            });
                             const updatedDecision = decideNextResearchStep({
                                 evidence: updatedRanked,
                                 suggestedHeadlessUrls: [],
                                 passIndex,
                                 maxPasses: DEFAULT_MAX_PASSES,
                                 headlessAttempts,
-                                maxHeadlessAttempts: DEFAULT_MAX_HEADLESS_ATTEMPTS
+                                maxHeadlessAttempts: DEFAULT_MAX_HEADLESS_ATTEMPTS,
+                                quality: updatedQuality
                             });
+                            const exhaustedBudget = updatedDecision.action !== 'answer' && passIndex + 1 >= DEFAULT_MAX_PASSES;
                             return {
-                                decision: decisionForAnswer(updatedDecision.action === 'answer' ? 'answer' : 'answer-with-caveat', query, updatedRanked),
+                                decision: decisionForAnswer({
+                                    action: updatedDecision.action === 'answer' ? 'answer' : 'answer-with-caveat',
+                                    query,
+                                    ranked: updatedRanked,
+                                    exhaustedBudget
+                                }),
                                 evidence: updatedRanked,
-                                workerPass: lastPass,
+                                workerPass: combinedWorkerPass({
+                                    lastPass,
+                                    previousQueries,
+                                    allGaps,
+                                    allLowValueOutcomes,
+                                    exhaustedBudget
+                                }),
                                 metadata: buildMetadata({
                                     previousQueries,
                                     allEvidence,
                                     allGaps,
                                     allLowValueOutcomes,
                                     headlessAttempts,
-                                    exhaustedBudget: updatedDecision.action !== 'answer'
+                                    exhaustedBudget,
+                                    caveatReasons: updatedQuality.caveatReasons
                                 })
                             };
                         }
@@ -146,52 +225,73 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
                                 approvedEvidence: ranked
                             },
                             evidence: ranked,
-                            workerPass: lastPass,
+                            workerPass: combinedWorkerPass({
+                                lastPass,
+                                previousQueries,
+                                allGaps,
+                                allLowValueOutcomes,
+                                exhaustedBudget: false
+                            }),
                             metadata: buildMetadata({
                                 previousQueries,
                                 allEvidence,
                                 allGaps,
                                 allLowValueOutcomes,
                                 headlessAttempts,
-                                exhaustedBudget: false
+                                exhaustedBudget: false,
+                                caveatReasons: quality.caveatReasons
                             })
                         };
                     }
                     if (decision.action === 'answer' || decision.action === 'answer-with-caveat') {
+                        const exhaustedBudget = decision.action === 'answer-with-caveat' && passIndex + 1 >= DEFAULT_MAX_PASSES;
                         return {
-                            decision: decisionForAnswer(decision.action, query, ranked),
+                            decision: decisionForAnswer({ action: decision.action, query, ranked, exhaustedBudget }),
                             evidence: ranked,
-                            workerPass: lastPass,
+                            workerPass: combinedWorkerPass({
+                                lastPass,
+                                previousQueries,
+                                allGaps,
+                                allLowValueOutcomes,
+                                exhaustedBudget
+                            }),
                             metadata: buildMetadata({
                                 previousQueries,
                                 allEvidence,
                                 allGaps,
                                 allLowValueOutcomes,
                                 headlessAttempts,
-                                exhaustedBudget: decision.action === 'answer-with-caveat'
+                                exhaustedBudget,
+                                caveatReasons: quality.caveatReasons
                             })
                         };
                     }
                 }
             }
             const ranked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
+            const quality = analyzeEvidenceQuality({
+                evidence: ranked,
+                gaps: allGaps,
+                lowValueOutcomes: allLowValueOutcomes
+            });
             return {
-                decision: decisionForAnswer('answer-with-caveat', query, ranked),
+                decision: decisionForAnswer({ action: 'answer-with-caveat', query, ranked, exhaustedBudget: true }),
                 evidence: ranked,
-                workerPass: lastPass ??
-                    fallbackWorkerPass({
-                        previousQueries,
-                        allGaps,
-                        allLowValueOutcomes,
-                        exhaustedBudget: true
-                    }),
+                workerPass: combinedWorkerPass({
+                    lastPass,
+                    previousQueries,
+                    allGaps,
+                    allLowValueOutcomes,
+                    exhaustedBudget: true
+                }),
                 metadata: buildMetadata({
                     previousQueries,
                     allEvidence,
                     allGaps,
                     allLowValueOutcomes,
                     headlessAttempts,
-                    exhaustedBudget: true
+                    exhaustedBudget: true,
+                    caveatReasons: quality.caveatReasons
                 })
             };
         }

package/dist/orchestration/research-worker.js

@@ -1,26 +1,20 @@
 import { selectCandidates } from './candidate-selector.js';
+import { classifySourceProfile } from './source-profile.js';
 function classifySource(url) {
-    if (url.includes('/docs/api/') || url.includes('/config/'))
-        return 'official-api';
-    if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide/'))
-        return 'official-docs';
-    if (url.includes('github.com/vitest-dev/vitest') && url.includes('/docs/'))
-        return 'official-docs';
-    if (url.includes('learn.microsoft.com'))
-        return 'official-docs';
-    if (url.includes('github.com/') && url.includes('/issues/'))
-        return 'issue-thread';
-    if (url.includes('npmjs.com/package/'))
-        return 'package-page';
-    return 'community';
+    return classifySourceProfile(url).sourceKind;
 }
 function summarizeText(text, maxLength = 180) {
     return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
 }
+function isBotCheckContent({ title = '', text }) {
+    return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
+}
 function evidenceFromFetch(fetched, fallbackTitle) {
     const content = fetched.content;
     if (fetched.status !== 'ok' || !content)
         return null;
+    if (isBotCheckContent({ title: content.title, text: content.text }))
+        return null;
     const sourceKind = classifySource(fetched.url);
     if (sourceKind === 'package-page') {
         return null;
@@ -37,6 +31,13 @@ function evidenceFromFetch(fetched, fallbackTitle) {
 function lowValueOutcomeFromFetch(fetched) {
     if (fetched.status !== 'ok' || !fetched.content)
         return null;
+    if (isBotCheckContent({ title: fetched.content.title, text: fetched.content.text })) {
+        return {
+            kind: 'bot-check',
+            url: fetched.url,
+            message: 'Fetched page showed a bot-check or security verification page.'
+        };
+    }
     if (classifySource(fetched.url) !== 'package-page')
         return null;
     return {
@@ -95,6 +96,7 @@ export function createResearchWorker({ search, fetchPage }) {
                 };
             }
             const candidates = selectCandidates({
+                query,
                 results: searchResult.results,
                 seenUrls: new Set(evidence.map((item) => item.url)),
                 maxCandidates: maxFetches

package/dist/orchestration/source-profile.d.ts

@@ -0,0 +1,8 @@
+import type { ResearchSourceKind } from './research-types.js';
+export type SourceProfileKind = 'official-docs' | 'official-api' | 'issue-thread' | 'forum-thread' | 'package-page' | 'community';
+export type SourceProfile = {
+    kind: SourceProfileKind;
+    sourceKind: ResearchSourceKind;
+    shouldPreferHeadlessWhenWeak: boolean;
+};
+export declare function classifySourceProfile(rawUrl: string): SourceProfile;

package/dist/orchestration/source-profile.js

@@ -0,0 +1,60 @@
+const COMMUNITY_FORUM_HOST_RE = /(^|\.)(community|forum|forums|discuss|discourse)\./;
+function profile(kind, sourceKind, shouldPreferHeadlessWhenWeak) {
+    return { kind, sourceKind, shouldPreferHeadlessWhenWeak };
+}
+function parseUrl(rawUrl) {
+    try {
+        return new URL(rawUrl);
+    }
+    catch {
+        return undefined;
+    }
+}
+function isOfficialApi(host, path) {
+    return ((host === 'playwright.dev' && path.startsWith('/docs/api/')) ||
+        (host === 'vitest.dev' && path.startsWith('/config/')));
+}
+function isOfficialDocs(host, path) {
+    return ((host === 'playwright.dev' && path.startsWith('/docs/')) ||
+        (host === 'vitest.dev' && path.startsWith('/guide/')) ||
+        (host === 'github.com' && path.startsWith('/vitest-dev/vitest/') && path.includes('/docs/')) ||
+        host === 'learn.microsoft.com');
+}
+function isIssueThread(host, path) {
+    return host === 'github.com' && (path.includes('/issues/') || path.includes('/discussions/'));
+}
+function hasForumThreadPath(path) {
+    return (path.includes('/forum/') ||
+        path.includes('/forums/') ||
+        path.includes('/t/') ||
+        path.includes('/topic/') ||
+        path.includes('/threads/'));
+}
+function isForumThread(host, path) {
+    return ((host === 'reddit.com' && path.includes('/comments/')) ||
+        (host === 'stackoverflow.com' && path.startsWith('/questions/')) ||
+        (COMMUNITY_FORUM_HOST_RE.test(`${host}.`) && hasForumThreadPath(path)));
+}
+export function classifySourceProfile(rawUrl) {
+    const parsed = parseUrl(rawUrl);
+    if (!parsed)
+        return profile('community', 'community', false);
+    const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
+    const path = parsed.pathname.toLowerCase();
+    if (isOfficialApi(host, path)) {
+        return profile('official-api', 'official-api', false);
+    }
+    if (isOfficialDocs(host, path)) {
+        return profile('official-docs', 'official-docs', false);
+    }
+    if (isIssueThread(host, path)) {
+        return profile('issue-thread', 'issue-thread', true);
+    }
+    if (isForumThread(host, path)) {
+        return profile('forum-thread', 'community', true);
+    }
+    if (host === 'npmjs.com' && path.startsWith('/package/')) {
+        return profile('package-page', 'package-page', false);
+    }
+    return profile('community', 'community', false);
+}

package/dist/orchestration/stop-decider.d.ts

@@ -1,4 +1,5 @@
 import type { ResearchEvidence } from './research-types.js';
+import type { EvidenceQualityReport } from './evidence-quality.js';
 export type ResearchStepDecision = {
     action: 'answer';
 } | {
@@ -9,11 +10,12 @@ export type ResearchStepDecision = {
     action: 'headless';
     url: string;
 };
-export declare function decideNextResearchStep({ evidence, suggestedHeadlessUrls, passIndex, maxPasses, headlessAttempts, maxHeadlessAttempts }: {
+export declare function decideNextResearchStep({ evidence, suggestedHeadlessUrls, passIndex, maxPasses, headlessAttempts, maxHeadlessAttempts, quality }: {
     evidence: ResearchEvidence[];
     suggestedHeadlessUrls: string[];
     passIndex: number;
     maxPasses: number;
     headlessAttempts: number;
     maxHeadlessAttempts: number;
+    quality?: EvidenceQualityReport;
 }): ResearchStepDecision;

package/dist/orchestration/stop-decider.js

@@ -1,7 +1,35 @@
 import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
-export function decideNextResearchStep({ evidence, suggestedHeadlessUrls, passIndex, maxPasses, headlessAttempts, maxHeadlessAttempts }) {
-    if (strongEvidenceCount(evidence) >= 2 && hasOfficialEvidence(evidence)) {
-        return { action: 'answer' };
+function activeCaveatReasons(evidence, quality) {
+    const reasons = quality?.caveatReasons ?? [];
+    if (!hasOfficialDocsAndApi(evidence))
+        return reasons;
+    return reasons.filter((reason) => reason !== 'low-diversity');
+}
+function hasQualityConcern(evidence, quality) {
+    return activeCaveatReasons(evidence, quality).length > 0;
+}
+function hasOfficialDocsAndApi(evidence) {
+    return evidence.some((item) => item.sourceKind === 'official-docs') &&
+        evidence.some((item) => item.sourceKind === 'official-api');
+}
+function shouldSearchForBetterQuality({ evidence, quality, passIndex, maxPasses }) {
+    if (!quality)
+        return false;
+    if (passIndex + 1 >= maxPasses)
+        return false;
+    if (quality.flags.hasOnlyCommunityEvidence)
+        return true;
+    if (quality.flags.hasLowDiversity && !hasOfficialDocsAndApi(evidence))
+        return true;
+    return false;
+}
+export function decideNextResearchStep({ evidence, suggestedHeadlessUrls, passIndex, maxPasses, headlessAttempts, maxHeadlessAttempts, quality }) {
+    const strongEnough = strongEvidenceCount(evidence) >= 2 && hasOfficialEvidence(evidence);
+    if (strongEnough && shouldSearchForBetterQuality({ evidence, quality, passIndex, maxPasses })) {
+        return { action: 'search-again' };
+    }
+    if (strongEnough) {
+        return hasQualityConcern(evidence, quality) ? { action: 'answer-with-caveat' } : { action: 'answer' };
     }
     const headlessUrl = suggestedHeadlessUrls.find((url) => !url.includes('npmjs.com/package/'));
     if (headlessUrl && headlessAttempts < maxHeadlessAttempts) {

package/dist/tools/web-explore.d.ts

@@ -39,6 +39,7 @@ export declare function createWebExploreTool({ explore }?: {
         fetchedPages: number;
         headlessAttempts: number;
         exhaustedBudget: boolean;
+        caveatReasons?: string[];
     };
     error?: import("../types.js").ToolError;
 }>;

package/dist/tools/web-explore.js

@@ -25,7 +25,8 @@ export function createWebExploreTool({ explore = createResearchWorkflow() } = {}
         }));
         const synthesized = synthesizeAnswer({
             evidence: result.evidence,
-            partial: result.decision.action !== 'answer'
+            partial: result.decision.action !== 'answer',
+            caveatReasons: result.metadata?.caveatReasons
         });
         const shaped = {
             status: 'ok',

package/dist/types.d.ts

@@ -68,6 +68,7 @@ export type WebExploreResponse = {
         fetchedPages: number;
         headlessAttempts: number;
         exhaustedBudget: boolean;
+        caveatReasons?: string[];
     };
     presentation?: PresentationEnvelope;
     error?: ToolError;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",
@@ -57,7 +57,7 @@
     "@mozilla/readability": "^0.6.0",
     "cheerio": "^1.1.0",
     "jsdom": "^26.0.0",
-    "playwright-core": "^1.54.0",
+    "playwright": "^1.60.0",
     "typebox": "^1.1.37"
   },
   "devDependencies": {