@demigodmode/pi-web-agent 1.11.0 → 1.13.0

package/dist/fetch/network-guard.js

@@ -0,0 +1,275 @@
+import { lookup as dnsLookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+export const BLOCKED_PRIVATE_ADDRESS = 'BLOCKED_PRIVATE_ADDRESS';
+export class BlockedAddressError extends Error {
+    host;
+    address;
+    code = BLOCKED_PRIVATE_ADDRESS;
+    constructor(host, address) {
+        super(`Blocked ${host}: resolves to private address ${address}. ` +
+            'Add it to backends.network.allowRanges if this is intended.');
+        this.host = host;
+        this.address = address;
+        this.name = 'BlockedAddressError';
+    }
+}
+/** No address could be verified, so we refuse rather than let something else resolve it. */
+export class UnverifiedDestinationError extends Error {
+    host;
+    code = BLOCKED_PRIVATE_ADDRESS;
+    constructor(host) {
+        super(`Blocked ${host}: could not verify its address before connecting.`);
+        this.host = host;
+        this.name = 'UnverifiedDestinationError';
+    }
+}
+export const UPSTREAM_PROXY_REFUSED = 'UPSTREAM_PROXY_REFUSED';
+/** The user's upstream proxy would not accept the approved IP. We never retry by hostname. */
+export class UpstreamProxyRefusedError extends Error {
+    host;
+    target;
+    status;
+    code = UPSTREAM_PROXY_REFUSED;
+    constructor(host, target, status) {
+        super(`Upstream proxy refused ${target} for ${host} (HTTP ${status}). ` +
+            'If it only accepts hostnames, set backends.network.trustProxyDns to trust it to enforce private-address restrictions.');
+        this.host = host;
+        this.target = target;
+        this.status = status;
+        this.name = 'UpstreamProxyRefusedError';
+    }
+}
+export function findGuardError(error) {
+    let current = error;
+    for (let depth = 0; depth < 5 && current; depth += 1) {
+        if (current instanceof BlockedAddressError ||
+            current instanceof UnverifiedDestinationError ||
+            current instanceof UpstreamProxyRefusedError) {
+            return current;
+        }
+        current = current.cause;
+    }
+    return undefined;
+}
+/** undici wraps connect errors as `TypeError: fetch failed` with the real error in `cause`. */
+export function findBlockedAddressError(error) {
+    let current = error;
+    for (let depth = 0; depth < 5 && current; depth += 1) {
+        if (current instanceof BlockedAddressError)
+            return current;
+        current = current.cause;
+    }
+    return undefined;
+}
+const IPV4_WIDTH = 32;
+const IPV6_WIDTH = 128;
+function ipv4ToBigInt(address) {
+    const parts = address.split('.');
+    if (parts.length !== 4)
+        return undefined;
+    let value = 0n;
+    for (const part of parts) {
+        if (!/^\d{1,3}$/.test(part))
+            return undefined;
+        const octet = Number(part);
+        if (octet > 255)
+            return undefined;
+        value = (value << 8n) | BigInt(octet);
+    }
+    return value;
+}
+function ipv6ToBigInt(address) {
+    let text = address.split('%')[0];
+    // Embedded IPv4 tail, e.g. ::ffff:127.0.0.1
+    if (text.includes('.')) {
+        const lastColon = text.lastIndexOf(':');
+        const v4 = ipv4ToBigInt(text.slice(lastColon + 1));
+        if (v4 === undefined)
+            return undefined;
+        const high = ((v4 >> 16n) & 0xffffn).toString(16);
+        const low = (v4 & 0xffffn).toString(16);
+        text = `${text.slice(0, lastColon + 1)}${high}:${low}`;
+    }
+    const halves = text.split('::');
+    if (halves.length > 2)
+        return undefined;
+    const head = halves[0] ? halves[0].split(':') : [];
+    const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+    const missing = 8 - head.length - tail.length;
+    if (halves.length === 1 && missing !== 0)
+        return undefined;
+    if (halves.length === 2 && missing < 1)
+        return undefined;
+    const groups = [...head, ...new Array(halves.length === 2 ? missing : 0).fill('0'), ...tail];
+    let value = 0n;
+    for (const group of groups) {
+        if (!/^[0-9a-f]{1,4}$/i.test(group))
+            return undefined;
+        value = (value << 16n) | BigInt(parseInt(group, 16));
+    }
+    return value;
+}
+const NAT64_WELL_KNOWN_NET = ipv6ToBigInt('64:ff9b::') >> 32n;
+const NAT64_LOCAL_NET = ipv6ToBigInt('64:ff9b:1::') >> 80n;
+const SIXTOFOUR_NET = ipv6ToBigInt('2002::') >> 112n;
+function parseAddress(address) {
+    const family = isIP(address.split('%')[0]);
+    if (family === 4) {
+        const value = ipv4ToBigInt(address);
+        return value === undefined ? undefined : { family: 4, value };
+    }
+    if (family === 6) {
+        const value = ipv6ToBigInt(address);
+        if (value === undefined)
+            return undefined;
+        // IPv4-mapped (::ffff:a.b.c.d): judge it as the IPv4 address it really is.
+        if (value >> 32n === 0xffffn)
+            return { family: 4, value: value & 0xffffffffn };
+        // IPv4-compatible (deprecated ::a.b.c.d), but not the special addresses :: and ::1.
+        if (value >> 32n === 0n && value > 1n)
+            return { family: 4, value };
+        // NAT64 (64:ff9b::/96 well-known, 64:ff9b:1::/48 local-use): IPv4 is the low 32 bits.
+        // DNS64 networks translate every IPv4-only host into these prefixes, so we cannot
+        // block the prefix outright -- judge the embedded address on its own merits instead.
+        if (value >> 32n === NAT64_WELL_KNOWN_NET)
+            return { family: 4, value: value & 0xffffffffn };
+        // Local-use NAT64 lets the operator pick a prefix shorter than /96; only the /96
+        // case has the IPv4 address in the low 32 bits, so only unwrap when bits 48-95
+        // (between the /48 prefix and the embedded address) are all zero.
+        if (value >> 80n === NAT64_LOCAL_NET && ((value >> 32n) & 0xffffffffffffn) === 0n) {
+            return { family: 4, value: value & 0xffffffffn };
+        }
+        // 6to4 (2002::/16): IPv4 is bits 16-48 from the top.
+        if (value >> 112n === SIXTOFOUR_NET)
+            return { family: 4, value: (value >> 80n) & 0xffffffffn };
+        return { family: 6, value };
+    }
+    return undefined;
+}
+export function parseCidr(text) {
+    const trimmed = text.trim();
+    const slash = trimmed.indexOf('/');
+    if (slash <= 0)
+        return undefined;
+    const prefixText = trimmed.slice(slash + 1);
+    if (!/^\d{1,3}$/.test(prefixText))
+        return undefined;
+    const prefix = Number(prefixText);
+    const address = parseAddress(trimmed.slice(0, slash));
+    if (!address)
+        return undefined;
+    // A mapped IPv6 CIDR would be ambiguous; require the plain family.
+    const family = isIP(trimmed.slice(0, slash));
+    const width = family === 4 ? IPV4_WIDTH : IPV6_WIDTH;
+    if (prefix > width)
+        return undefined;
+    const value = family === 4 ? ipv4ToBigInt(trimmed.slice(0, slash)) : ipv6ToBigInt(trimmed.slice(0, slash));
+    const shift = BigInt(width - prefix);
+    return { family, network: (value >> shift) << shift, prefix };
+}
+function cidrContains(cidr, address) {
+    if (cidr.family !== address.family)
+        return false;
+    const width = cidr.family === 4 ? IPV4_WIDTH : IPV6_WIDTH;
+    const shift = BigInt(width - cidr.prefix);
+    return address.value >> shift === cidr.network >> shift;
+}
+const BLOCKED_RANGES = [
+    '0.0.0.0/8',
+    '10.0.0.0/8',
+    '100.64.0.0/10',
+    '127.0.0.0/8',
+    '169.254.0.0/16',
+    '172.16.0.0/12',
+    '192.0.0.0/24',
+    '192.168.0.0/16',
+    '198.18.0.0/15',
+    '224.0.0.0/4',
+    '240.0.0.0/4',
+    '::1/128',
+    '::/128',
+    '64:ff9b:1::/48',
+    'fc00::/7',
+    'fe80::/10',
+    'fec0::/10',
+    'ff00::/8'
+].map((range) => parseCidr(range));
+/** Parses an allow list, dropping invalid entries and anything that allows every address. */
+export function usableAllowRanges(allowRanges = []) {
+    return allowRanges
+        .map((range) => parseCidr(range))
+        .filter((cidr) => cidr !== undefined && cidr.prefix > 0);
+}
+const DEFAULT_LOOKUP_TIMEOUT_MS = 10_000;
+const defaultLookup = (host) => dnsLookup(host, { all: true, verbatim: true });
+function stripBrackets(host) {
+    return host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host;
+}
+export function createNetworkGuard(config = {}, { lookup = defaultLookup, lookupTimeoutMs = DEFAULT_LOOKUP_TIMEOUT_MS } = {}) {
+    const allow = usableAllowRanges(config.allowRanges);
+    function isBlockedAddress(address) {
+        const parsed = parseAddress(address);
+        if (!parsed)
+            return true; // fail closed
+        if (allow.some((cidr) => cidrContains(cidr, parsed)))
+            return false;
+        return BLOCKED_RANGES.some((cidr) => cidrContains(cidr, parsed));
+    }
+    async function resolveHost(rawHost) {
+        const host = stripBrackets(rawHost.toLowerCase()).replace(/\.+$/, '');
+        if (host === 'localhost' || host.endsWith('.localhost')) {
+            return isBlockedAddress('127.0.0.1')
+                ? { status: 'blocked', host, address: '127.0.0.1' }
+                : { status: 'allowed', host, addresses: ['127.0.0.1'] };
+        }
+        if (isIP(host)) {
+            return isBlockedAddress(host)
+                ? { status: 'blocked', host, address: host }
+                : { status: 'allowed', host, addresses: [host] };
+        }
+        let answers;
+        let timer;
+        try {
+            answers = await Promise.race([
+                lookup(host),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve(undefined), lookupTimeoutMs);
+                    timer.unref?.();
+                })
+            ]);
+        }
+        catch {
+            return { status: 'unresolved', host };
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        if (!answers || answers.length === 0)
+            return { status: 'unresolved', host };
+        // Any private answer blocks: which address a connection would pick is not ours to control.
+        const blocked = answers.find((entry) => isBlockedAddress(entry.address));
+        if (blocked)
+            return { status: 'blocked', host, address: blocked.address };
+        return { status: 'allowed', host, addresses: answers.map((entry) => entry.address) };
+    }
+    async function checkHost(rawHost) {
+        const resolution = await resolveHost(rawHost);
+        if (resolution.status === 'blocked') {
+            return { allowed: false, host: resolution.host, address: resolution.address };
+        }
+        return resolution.status === 'unresolved' ? { allowed: true, unresolved: true } : { allowed: true };
+    }
+    async function assertUrlAllowed(url) {
+        let parsed;
+        try {
+            parsed = new URL(url);
+        }
+        catch {
+            return; // not a URL; the tools reject it with UNSUPPORTED_URL
+        }
+        const verdict = await checkHost(parsed.hostname);
+        if (!verdict.allowed)
+            throw new BlockedAddressError(verdict.host, verdict.address);
+    }
+    return { isBlockedAddress, resolveHost, checkHost, assertUrlAllowed };
+}

package/dist/orchestration/answer-synthesizer.js

@@ -15,6 +15,8 @@ function sentenceForReason(reason) {
             return 'readable sources include cautionary or possibly conflicting guidance';
         case 'bot-check':
             return 'some candidate sources showed bot-check or security verification pages';
+        case 'partial-search-coverage':
+            return 'some search backends were unavailable, so results may be incomplete';
     }
 }
 function joinReasons(reasons) {

package/dist/orchestration/evidence-quality.d.ts

@@ -1,5 +1,5 @@
 import type { ResearchEvidence, ResearchGap, ResearchLowValueOutcome } from './research-types.js';
-export type EvidenceCaveatReason = 'community-only' | 'low-diversity' | 'unreadable-direct-source' | 'unreadable-thread-source' | 'possible-conflict' | 'bot-check';
+export type EvidenceCaveatReason = 'community-only' | 'low-diversity' | 'unreadable-direct-source' | 'unreadable-thread-source' | 'possible-conflict' | 'bot-check' | 'partial-search-coverage';
 export type EvidenceQualityReport = {
     counts: {
         total: number;
@@ -20,8 +20,9 @@ export type EvidenceQualityReport = {
     };
     caveatReasons: EvidenceCaveatReason[];
 };
-export declare function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }: {
+export declare function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes, partialSearchCoverage }: {
     evidence: ResearchEvidence[];
     gaps: ResearchGap[];
     lowValueOutcomes: ResearchLowValueOutcome[];
+    partialSearchCoverage?: boolean;
 }): EvidenceQualityReport;

package/dist/orchestration/evidence-quality.js

@@ -20,7 +20,7 @@ function addReason(reasons, reason, enabled) {
     if (enabled && !reasons.includes(reason))
         reasons.push(reason);
 }
-export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
+export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes, partialSearchCoverage = false }) {
     const official = evidence.filter((item) => item.sourceKind === 'official-docs' || item.sourceKind === 'official-api').length;
     const community = evidence.filter((item) => item.sourceKind === 'community').length;
     const thread = evidence.filter((item) => item.sourceKind === 'issue-thread' || item.sourceKind === 'official-discussion').length;
@@ -41,6 +41,7 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
     addReason(caveatReasons, 'unreadable-thread-source', hasUnreadableThreadSource);
     addReason(caveatReasons, 'possible-conflict', hasPossibleConflict);
     addReason(caveatReasons, 'bot-check', hasBotCheck);
+    addReason(caveatReasons, 'partial-search-coverage', partialSearchCoverage);
     return {
         counts: {
             total: evidence.length,

package/dist/orchestration/index.d.ts

@@ -1,16 +1,12 @@
 import type { BackendConfig } from '../backends/config.js';
-import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
 export declare function createResearchWorkflow({ backendConfig, search, fetchPage, headlessFetch }?: {
     backendConfig?: BackendConfig;
     search?: (input: {
         query: string;
     }) => Promise<WebSearchResponse>;
-    fetchPage?: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
-    headlessFetch?: (input: {
-        url: string;
-    }) => Promise<WebFetchHeadlessResponse>;
+    fetchPage?: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
+    headlessFetch?: (input: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;
 }): {
     run({ query }: {
         query: string;
@@ -19,6 +15,7 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
         evidence: import("./research-types.js").ResearchEvidence[];
         workerPass: import("./research-types.js").ResearchWorkerResult;
         metadata: {
+            attempts?: import("../types.js").Attempt[] | undefined;
             searchPasses: number;
             fetchedPages: number;
             headlessAttempts: number;
@@ -27,5 +24,27 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
             fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
             fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
         };
+        terminalFailure?: undefined;
+    } | {
+        decision: import("./research-types.js").ResearchOrchestratorDecision;
+        evidence: never[];
+        workerPass: import("./research-types.js").ResearchWorkerResult;
+        metadata: {
+            attempts?: import("../types.js").Attempt[] | undefined;
+            searchPasses: number;
+            fetchedPages: number;
+            headlessAttempts: number;
+            exhaustedBudget: boolean;
+            caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
+            fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
+            fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
+        };
+        terminalFailure: {
+            code: string;
+            message: string;
+        };
     }>;
+} & {
+    /** Releases the backend set this workflow created. Injected capabilities are left alone. */
+    close(): Promise<void>;
 };

package/dist/orchestration/index.js

@@ -2,14 +2,21 @@ import { createBackendSet } from '../backends/factory.js';
 import { createResearchOrchestrator } from './research-orchestrator.js';
 import { createResearchWorker } from './research-worker.js';
 export function createResearchWorkflow({ backendConfig, search, fetchPage, headlessFetch } = {}) {
-    const backends = createBackendSet(backendConfig);
+    // Only build (and own) a backend set when something wasn't injected.
+    const backends = search && fetchPage && headlessFetch ? undefined : createBackendSet(backendConfig);
     const resolvedSearch = search ?? backends.search;
     const resolvedFetchPage = fetchPage ?? backends.fetchPage;
     const resolvedHeadlessFetch = headlessFetch ?? backends.headlessFetch;
     const worker = createResearchWorker({ search: resolvedSearch, fetchPage: resolvedFetchPage });
-    return createResearchOrchestrator({
+    const orchestrator = createResearchOrchestrator({
         worker,
         fetchDirect: resolvedFetchPage,
         headlessFetch: resolvedHeadlessFetch
     });
+    return Object.assign(orchestrator, {
+        /** Releases the backend set this workflow created. Injected capabilities are left alone. */
+        async close() {
+            await backends?.close?.();
+        }
+    });
 }

package/dist/orchestration/research-orchestrator.d.ts

@@ -1,4 +1,4 @@
-import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
+import type { Attempt, ResearchFetchInput, SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
 import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
 import { type EvidenceCaveatReason } from './evidence-quality.js';
 export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
@@ -9,12 +9,8 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
             maxFetches: number;
         }) => Promise<ResearchWorkerResult>;
     };
-    fetchDirect?: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
-    headlessFetch: (input: {
-        url: string;
-    }) => Promise<WebFetchHeadlessResponse>;
+    fetchDirect?: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
+    headlessFetch: (input: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;
 }): {
     run({ query }: {
         query: string;
@@ -23,6 +19,7 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
         evidence: ResearchEvidence[];
         workerPass: ResearchWorkerResult;
         metadata: {
+            attempts?: Attempt[] | undefined;
             searchPasses: number;
             fetchedPages: number;
             headlessAttempts: number;
@@ -31,5 +28,24 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
             fanoutProviders: SearchProviderName[] | undefined;
             fanoutSkipped: SearchProviderName[] | undefined;
         };
+        terminalFailure?: undefined;
+    } | {
+        decision: ResearchOrchestratorDecision;
+        evidence: never[];
+        workerPass: ResearchWorkerResult;
+        metadata: {
+            attempts?: Attempt[] | undefined;
+            searchPasses: number;
+            fetchedPages: number;
+            headlessAttempts: number;
+            exhaustedBudget: boolean;
+            caveatReasons: EvidenceCaveatReason[];
+            fanoutProviders: SearchProviderName[] | undefined;
+            fanoutSkipped: SearchProviderName[] | undefined;
+        };
+        terminalFailure: {
+            code: string;
+            message: string;
+        };
     }>;
 };

package/dist/orchestration/research-orchestrator.js

@@ -1,28 +1,30 @@
+import { failureOf, isTerminalFailure } from '../backends/failure.js';
 import { rankEvidence } from './evidence-ranker.js';
 import { planSearchQueries } from './query-planner.js';
 import { classifySourceProfile } from './source-profile.js';
 import { extractDirectUrls } from './direct-url.js';
 import { decideNextResearchStep } from './stop-decider.js';
 import { analyzeEvidenceQuality } from './evidence-quality.js';
+import { selectRelevantExcerpt } from '../extract/section-selector.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
 const DEFAULT_MAX_PASSES = 3;
 const DEFAULT_MAX_FETCHES_PER_PASS = 4;
 const DEFAULT_MAX_HEADLESS_ATTEMPTS = 2;
 function classifyEvidenceUrl(url) {
     return classifySourceProfile(url).sourceKind;
 }
-function summarizeText(text, maxLength = 180) {
-    return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
-}
 function isReaderMethod(method) {
     return method === 'github' || method === 'pdf' || method === 'youtube';
 }
-function isBotCheckContent({ title = '', text }) {
-    return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
+function isBotCheckContent({ title = '', text, botCheck }) {
+    if (botCheck)
+        return true;
+    return hasBotCheckContent(`${title}\n${text}`);
 }
-function evidenceFromFetch(result) {
+function evidenceFromFetch(result, query) {
     if (result.status !== 'ok' || !result.content?.text.trim())
         return null;
-    if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
+    if (isBotCheckContent({ title: result.content.title, text: result.content.text, botCheck: result.content.botCheck }))
         return null;
     if (isReaderMethod(result.metadata.method)) {
         return {
@@ -39,22 +41,22 @@ function evidenceFromFetch(result) {
         url: result.url,
         sourceKind: classifyEvidenceUrl(result.url),
         method: result.metadata.method,
-        summary: summarizeText(result.content.text),
-        supports: [summarizeText(result.content.text, 120)]
+        summary: selectRelevantExcerpt(result.content.text, query, 180),
+        supports: [selectRelevantExcerpt(result.content.text, query, 120)]
     };
 }
-function evidenceFromHeadless(result) {
+function evidenceFromHeadless(result, query) {
     if (result.status !== 'ok' || !result.content?.text.trim())
         return null;
-    if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
+    if (isBotCheckContent({ title: result.content.title, text: result.content.text, botCheck: result.content.botCheck }))
         return null;
     return {
         title: result.content.title ?? result.url,
         url: result.url,
         sourceKind: classifyEvidenceUrl(result.url),
         method: 'headless',
-        summary: summarizeText(result.content.text),
-        supports: [summarizeText(result.content.text, 120)]
+        summary: selectRelevantExcerpt(result.content.text, query, 180),
+        supports: [selectRelevantExcerpt(result.content.text, query, 120)]
     };
 }
 function combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget }) {
@@ -79,7 +81,7 @@ function shouldRetryDirectWithHeadless(result, evidence) {
         return false;
     return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
 }
-function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped }) {
+function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped, attempts }) {
     return {
         searchPasses: previousQueries.length,
         fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
@@ -87,7 +89,8 @@ function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutco
         exhaustedBudget,
         caveatReasons,
         fanoutProviders,
-        fanoutSkipped
+        fanoutSkipped,
+        ...(attempts && attempts.length > 0 ? { attempts } : {})
     };
 }
 function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
@@ -114,26 +117,38 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
             const suggestedHeadlessUrls = [];
             let headlessAttempts = 0;
             let lastPass;
+            let searchCoveragePartial = false;
             const fanoutProvidersSeen = new Set();
             const fanoutSkippedSeen = new Set();
+            // Search and fetch attempts from every pass and direct URL, for verbose provenance.
+            const runAttempts = [];
             function fanoutSnapshot() {
                 const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
                 const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
-                return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
+                return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined, attempts: [...runAttempts] };
             }
             if (fetchDirect) {
                 for (const url of extractDirectUrls(query).slice(0, 3)) {
-                    const directResult = await fetchDirect({ url });
-                    const directEvidence = evidenceFromFetch(directResult);
+                    const directResult = await fetchDirect({ url, query });
+                    if (directResult.metadata.attempts)
+                        runAttempts.push(...directResult.metadata.attempts);
+                    const directEvidence = evidenceFromFetch(directResult, query);
                     if (directEvidence) {
                         allEvidence.push(directEvidence);
                         continue;
                     }
+                    if (isTerminalFailure(failureOf(directResult))) {
+                        allGaps.push({
+                            kind: 'fetch-failed',
+                            message: directResult.error?.message ?? `Direct URL fetch failed for ${directResult.url}`
+                        });
+                        continue;
+                    }
                     if (shouldRetryDirectWithHeadless(directResult, directEvidence)) {
                         if (headlessAttempts < DEFAULT_MAX_HEADLESS_ATTEMPTS) {
                             headlessAttempts++;
-                            const headlessResult = await headlessFetch({ url: directResult.url });
-                            const headlessEvidence = evidenceFromHeadless(headlessResult);
+                            const headlessResult = await headlessFetch({ url: directResult.url, query });
+                            const headlessEvidence = evidenceFromHeadless(headlessResult, query);
                             if (headlessEvidence) {
                                 allEvidence.push(headlessEvidence);
                             }
@@ -161,7 +176,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                 const quality = analyzeEvidenceQuality({
                     evidence: ranked,
                     gaps: allGaps,
-                    lowValueOutcomes: allLowValueOutcomes
+                    lowValueOutcomes: allLowValueOutcomes,
+                    partialSearchCoverage: searchCoveragePartial
                 });
                 return {
                     decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
@@ -200,6 +216,21 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                         maxFetches: DEFAULT_MAX_FETCHES_PER_PASS
                     });
                     lastPass = pass;
+                    if (pass.searchAttempts)
+                        runAttempts.push(...pass.searchAttempts);
+                    if (pass.fetchAttempts)
+                        runAttempts.push(...pass.fetchAttempts);
+                    if (pass.searchCoveragePartial)
+                        searchCoveragePartial = true;
+                    if (pass.terminalFailure) {
+                        return {
+                            decision: decisionForAnswer({ action: 'answer-with-caveat', query, ranked: [], exhaustedBudget: false }),
+                            evidence: [],
+                            workerPass: combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget: false }),
+                            metadata: buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget: false, ...fanoutSnapshot() }),
+                            terminalFailure: pass.terminalFailure
+                        };
+                    }
                     pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
                     pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
                     allEvidence.push(...pass.evidence);
@@ -211,7 +242,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     const quality = analyzeEvidenceQuality({
                         evidence: ranked,
                         gaps: allGaps,
-                        lowValueOutcomes: allLowValueOutcomes
+                        lowValueOutcomes: allLowValueOutcomes,
+                        partialSearchCoverage: searchCoveragePartial
                     });
                     const decision = decideNextResearchStep({
                         evidence: ranked,
@@ -224,15 +256,16 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     });
                     if (decision.action === 'headless') {
                         headlessAttempts++;
-                        const headlessResult = await headlessFetch({ url: decision.url });
-                        const headlessEvidence = evidenceFromHeadless(headlessResult);
+                        const headlessResult = await headlessFetch({ url: decision.url, query });
+                        const headlessEvidence = evidenceFromHeadless(headlessResult, query);
                         if (headlessEvidence) {
                             allEvidence.push(headlessEvidence);
                             const updatedRanked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
                             const updatedQuality = analyzeEvidenceQuality({
                                 evidence: updatedRanked,
                                 gaps: allGaps,
-                                lowValueOutcomes: allLowValueOutcomes
+                                lowValueOutcomes: allLowValueOutcomes,
+                                partialSearchCoverage: searchCoveragePartial
                             });
                             const updatedDecision = decideNextResearchStep({
                                 evidence: updatedRanked,
@@ -328,7 +361,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
             const quality = analyzeEvidenceQuality({
                 evidence: ranked,
                 gaps: allGaps,
-                lowValueOutcomes: allLowValueOutcomes
+                lowValueOutcomes: allLowValueOutcomes,
+                partialSearchCoverage: searchCoveragePartial
             });
             return {
                 decision: decisionForAnswer({ action: 'answer-with-caveat', query, ranked, exhaustedBudget: true }),