@demigodmode/pi-web-agent 1.12.0 → 1.13.0

package/CHANGELOG.md

@@ -18,6 +18,21 @@ The format is intentionally simple and release-oriented.
 ### Breaking
 - None.
 
+## [1.13.0] - 2026-09-24
+### Added
+- `web_explore` now uses the research question to pick relevant sections from long HTML pages instead of stopping at the first 4,000 characters. It can reach an answer buried later in a page, whether the page came through HTTP, headless browsing, or Firecrawl. Direct `web_fetch` output is unchanged. (#36)
+
+### Changed
+- You.com search and `/web-agent doctor` now use the documented `/v1/search` endpoint. Web and news results appear in the usual search list, with web results first. Result descriptions provide the snippets when present. (#60)
+
+### Fixed
+- Research still excludes recognized bot-check pages when their verification message sits outside the section chosen for the question. That page cannot be used as evidence just because another section looks relevant. (#36)
+- `/web-agent doctor` no longer repeats the default DuckDuckGo and HTTP backends after listing them in the config summary. It still shows checks and warnings for backends that need them. (#64)
+- On pages with several `<article>` sections and no `<main>`, query-based reads now search across the articles. A match in the first article no longer hides an answer in a later one. (#71)
+
+### Breaking
+- None.
+
 ## [1.12.0] - 2026-09-17
 ### Added
 - A network allow list, under Settings → Backends → Network allow list. web_explore now refuses private and local addresses when the link came from the model or a page it read, so if you genuinely want it to read something on your own network (an internal docs site, a service on localhost), add that range here as a CIDR like `10.0.0.0/24`. Entries are checked when you save, and ones that would allow everything (`0.0.0.0/0`, `::/0`) are rejected, since that would quietly turn the protection off. It is also the fix if every fetch suddenly fails: some proxy apps run in fake-IP mode and make every site look like it lives in `198.18.0.0/15`, and adding that range gets you going again. (#53)

package/README.md

@@ -34,6 +34,8 @@ One public tool, `web_explore`, that does bounded web research for Pi: search, f
 - **Six search backends.** DuckDuckGo (keyless default), SearXNG, Brave, You.com, Exa, Tavily.
 - **Optional search fanout.** Query several backends at once, dedupe, and rank pages that more than one provider agreed on to the top. Off by default; flip it to `on` or `auto`.
 - **Honest by default.** Weak, narrow, blocked, or cautionary evidence gets flagged instead of dressed up as confidence.
+- **Safe with untrusted pages.** Links the model picks, or finds on a page it read, can't reach localhost, your LAN, or cloud metadata endpoints. That includes redirects and everything a headless page loads. Addresses you configure yourself aren't affected, and an allow list covers the private ranges you do want.
+- **Fallback that knows why.** Rate-limited or misconfigured backends get skipped for a while, flaky ones get one retry, and answers say when some search backends were unavailable.
 - **Bounded output.** `compact` / `preview` / `verbose` transcript modes.
 - **Zero-config to start.** Runs keyless out of the box (DuckDuckGo search, local browser, the built-in readers). Opt into hosted backends, fallback, search fanout, and per-tool output modes through config when you want more control.
 

package/dist/backends/doctor.js

@@ -1,3 +1,4 @@
+import { normalizeYouComResults, YOUCOM_SEARCH_URL } from '../search/youcom.js';
 function withTimeout(timeoutMs) {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), timeoutMs);
@@ -13,7 +14,7 @@ function braveDoctorUrl() {
     return url.toString();
 }
 function youcomDoctorBody() {
-    return JSON.stringify({ query: 'pi-web-agent-doctor', max_results: 1 });
+    return JSON.stringify({ query: 'pi-web-agent-doctor', count: 1 });
 }
 function exaDoctorBody() {
     return JSON.stringify({ query: 'pi-web-agent-doctor', numResults: 1 });
@@ -42,10 +43,7 @@ function firecrawlDoctorBody(options = {}) {
 }
 export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs = 3_000 } = {}) {
     const lines = [];
-    if (config.search.provider === 'duckduckgo') {
-        lines.push('search backend: duckduckgo');
-    }
-    else if (config.search.provider === 'brave') {
+    if (config.search.provider === 'brave') {
         const apiKey = process.env.PI_WEB_AGENT_BRAVE_API_KEY;
         if (!apiKey?.trim()) {
             lines.push('search backend: brave warning (missing PI_WEB_AGENT_BRAVE_API_KEY)');
@@ -86,7 +84,7 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
         else {
             const timeout = withTimeout(timeoutMs);
             try {
-                const response = await fetchImpl('https://api.you.com/v1/agents/search', {
+                const response = await fetchImpl(YOUCOM_SEARCH_URL, {
                     method: 'POST',
                     headers: {
                         Accept: 'application/json',
@@ -100,8 +98,8 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
                     lines.push(`search backend: youcom warning (HTTP ${response.status})`);
                 }
                 else {
-                    const json = (await response.json());
-                    lines.push(Array.isArray(json.results)
+                    const normalized = normalizeYouComResults(await response.json());
+                    lines.push(normalized && (normalized.rawCount === 0 || normalized.results.length > 0)
                         ? 'search backend: youcom ok'
                         : 'search backend: youcom warning (unexpected response)');
                 }
@@ -186,23 +184,26 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
             }
         }
     }
-    else if (!config.search.baseUrl) {
-        lines.push('search backend: searxng warning (missing baseUrl)');
-    }
-    else {
-        const timeout = withTimeout(timeoutMs);
-        try {
-            const response = await fetchImpl(searxngDoctorUrl(config.search.baseUrl, config.search.options), { signal: timeout.signal });
-            const json = (await response.json());
-            lines.push(response.ok && Array.isArray(json.results)
-                ? 'search backend: searxng ok'
-                : 'search backend: searxng warning (unexpected response)');
-        }
-        catch (error) {
-            lines.push(`search backend: searxng warning (${message(error)})`);
+    else if (config.search.provider === 'searxng') {
+        const baseUrl = config.search.baseUrl;
+        if (!baseUrl) {
+            lines.push('search backend: searxng warning (missing baseUrl)');
         }
-        finally {
-            timeout.done();
+        else {
+            const timeout = withTimeout(timeoutMs);
+            try {
+                const response = await fetchImpl(searxngDoctorUrl(baseUrl, config.search.options), { signal: timeout.signal });
+                const json = (await response.json());
+                lines.push(response.ok && Array.isArray(json.results)
+                    ? 'search backend: searxng ok'
+                    : 'search backend: searxng warning (unexpected response)');
+            }
+            catch (error) {
+                lines.push(`search backend: searxng warning (${message(error)})`);
+            }
+            finally {
+                timeout.done();
+            }
         }
     }
     if (config.search.fallback) {
@@ -241,13 +242,10 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
             }
         }
     }
-    if (config.fetch.provider === 'http') {
-        lines.push('fetch backend: http');
-    }
-    else if (!config.fetch.baseUrl) {
+    if (config.fetch.provider !== 'http' && !config.fetch.baseUrl) {
         lines.push('fetch backend: firecrawl warning (missing baseUrl)');
     }
-    else {
+    else if (config.fetch.provider !== 'http') {
         const timeout = withTimeout(timeoutMs);
         try {
             const headers = { 'content-type': 'application/json' };

package/dist/backends/factory.d.ts

@@ -11,18 +11,14 @@ import { type ProviderHealth } from './provider-health.js';
 import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
 import { createWebFetchTool } from '../tools/web-fetch.js';
 import { createWebSearchTool } from '../tools/web-search.js';
-import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
 import { type BackendConfig, type ProxyConfig } from './config.js';
 export type BackendSet = {
     search: (input: {
         query: string;
     }) => Promise<WebSearchResponse>;
-    fetchPage: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
-    headlessFetch: (input: {
-        url: string;
-    }) => Promise<WebFetchHeadlessResponse>;
+    fetchPage: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
+    headlessFetch: (input: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;
     /** Releases the guard proxy and its agents. Idempotent; never starts the proxy. */
     close: () => Promise<void>;
 };

package/dist/backends/factory.js

@@ -241,18 +241,18 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
         // chainSearch only falls back on non-terminal failures, so a terminal result or bad_request never reaches keyless Tavily.
         search = chainSearch([search, guarded('tavily', createTavilySearch({ keyless: true, fetchImpl }), 'tavily-keyless')], policyDeps);
     }
-    const httpFetch = createHttpFetch({ fetchPage: createHttpFetcher({ fetchImpl: targetFetch }) });
+    const httpFetcher = createHttpFetcher({ fetchImpl: targetFetch });
+    const httpFetch = createHttpFetch({ fetchPage: ({ url, query }) => httpFetcher(url, query) });
+    const firecrawlFetcher = config.fetch.baseUrl
+        ? createFirecrawlFetch({
+            baseUrl: config.fetch.baseUrl,
+            apiKey: config.fetch.apiKey ?? process.env.PI_WEB_AGENT_FIRECRAWL_API_KEY,
+            options: config.fetch.options,
+            fetchImpl
+        })
+        : invalidFirecrawlFetch();
     const fetchPage = config.fetch.provider === 'firecrawl'
-        ? withFetchPolicy(config.fetch.baseUrl
-            ? createHttpFetch({
-                fetchPage: createFirecrawlFetch({
-                    baseUrl: config.fetch.baseUrl,
-                    apiKey: config.fetch.apiKey ?? process.env.PI_WEB_AGENT_FIRECRAWL_API_KEY,
-                    options: config.fetch.options,
-                    fetchImpl
-                })
-            })
-            : createHttpFetch({ fetchPage: invalidFirecrawlFetch() }), config.fetch.fallback === 'http' ? httpFetch : undefined, policyDeps)
+        ? withFetchPolicy(createHttpFetch({ fetchPage: ({ url, query }) => firecrawlFetcher(url, query) }), config.fetch.fallback === 'http' ? httpFetch : undefined, policyDeps)
         : httpFetch;
     const fetchPageWithReaders = createSpecialContentResolver({
         readers: [
@@ -262,7 +262,7 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
         ],
         fallback: fetchPage
     });
-    const headlessPage = (url) => headlessFetch(url, { guard: networkGuard, guardProxy: getGuardProxy });
+    const headlessPage = ({ url, query }) => headlessFetch(url, { query, guard: networkGuard, guardProxy: getGuardProxy });
     return {
         search,
         fetchPage: withTargetGuard(fetchPageWithReaders, networkGuard, config.fetch.provider === 'firecrawl' ? 'firecrawl' : 'http'),

package/dist/backends/fallback-policy.d.ts

@@ -1,4 +1,4 @@
-import type { SearchProviderName, WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ResearchFetchInput, SearchProviderName, WebFetchResponse, WebSearchResponse } from '../types.js';
 import type { ProviderHealth } from './provider-health.js';
 export declare const RETRY_BASE_MS = 500;
 export declare const RETRY_JITTER_MS = 250;
@@ -11,9 +11,7 @@ export type PolicyDeps = {
 type Search = (input: {
     query: string;
 }) => Promise<WebSearchResponse>;
-type FetchPage = (input: {
-    url: string;
-}) => Promise<WebFetchResponse>;
+type FetchPage = (input: ResearchFetchInput) => Promise<WebFetchResponse>;
 /**
  * One provider under the #55 policy: skip when cooling down or disabled,
  * retry exactly once on transient, record state. Never falls back itself.

package/dist/backends/provider-failure.js

@@ -88,7 +88,6 @@ export function classifyHttpFailure(provider, parts, now = Date.now()) {
     }
     else if (provider === 'youcom') {
         // Source: https://you.com/docs/api-reference/search/v1-search (429 UNVERIFIED -> default)
-        // Documented for /v1/search; the client currently calls /v1/agents/search (#60).
         if (status === 402)
             kind = 'quota_exhausted';
         else if (status === 403)

package/dist/extract/bot-check.d.ts

@@ -0,0 +1 @@
+export declare function hasBotCheckContent(source: string, format?: 'html' | 'markdown' | 'text'): boolean;

package/dist/extract/bot-check.js

@@ -0,0 +1,10 @@
+const BOT_CHECK_RE = /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i;
+export function hasBotCheckContent(source, format = 'text') {
+    const visibleSource = format === 'html'
+        ? source
+            .replace(/<!--[\s\S]*?-->/g, ' ')
+            .replace(/<(script|style|noscript|svg|template)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
+            .replace(/<[^>]+>/g, ' ')
+        : source;
+    return BOT_CHECK_RE.test(visibleSource);
+}

package/dist/extract/readability.d.ts

@@ -6,3 +6,7 @@ export type SafeReadableExtraction = {
 };
 export declare function extractReadableContent(html: string, maxLength?: number): ExtractedContent;
 export declare function extractReadableContentSafely(html: string, maxLength?: number): SafeReadableExtraction;
+/** Research-only extraction: choose relevant content before the usual text cap. */
+export declare function extractReadableContentForQuery(html: string, query: string, maxLength?: number): SafeReadableExtraction & {
+    omitted: boolean;
+};

package/dist/extract/readability.js

@@ -1,5 +1,6 @@
 import { Readability } from '@mozilla/readability';
 import { JSDOM, VirtualConsole } from 'jsdom';
+import { selectRelevantContent } from './section-selector.js';
 export function extractReadableContent(html, maxLength = 4000) {
     let stylesheetError;
     const virtualConsole = new VirtualConsole();
@@ -57,14 +58,25 @@ function extractPreferredSection(html) {
     const mainMatch = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
     if (mainMatch)
         return mainMatch[1];
-    const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
-    if (articleMatch)
-        return articleMatch[1];
+    const articles = html.match(/<article\b[^>]*>/gi) ?? [];
+    if (articles.length === 1) {
+        const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
+        if (articleMatch)
+            return articleMatch[1];
+    }
     const bodyMatch = html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
     if (bodyMatch)
         return bodyMatch[1];
     return html;
 }
+function extractPreferredDomSection(document) {
+    const main = document.querySelector('main');
+    const articles = document.querySelectorAll('article');
+    const region = main ?? (articles.length === 1 ? articles[0] : document.body);
+    const cleanedRegion = region.cloneNode(true);
+    cleanedRegion.querySelectorAll('script, style, noscript, svg, template').forEach((element) => element.remove());
+    return cleanedRegion.outerHTML;
+}
 function extractFallbackText(html, maxLength) {
     const title = extractTitle(html);
     let section = extractPreferredSection(html);
@@ -97,3 +109,55 @@ export function extractReadableContentSafely(html, maxLength = 4000) {
         };
     }
 }
+/** Research-only extraction: choose relevant content before the usual text cap. */
+export function extractReadableContentForQuery(html, query, maxLength = 4000) {
+    let stylesheetError;
+    const virtualConsole = new VirtualConsole();
+    virtualConsole.on('jsdomError', (error) => {
+        if (!stylesheetError && error.message.includes('Could not parse CSS stylesheet')) {
+            stylesheetError = error;
+        }
+    });
+    try {
+        const dom = new JSDOM(html, { url: 'https://example.com', virtualConsole });
+        if (stylesheetError)
+            throw stylesheetError;
+        const preferredRegion = extractPreferredDomSection(dom.window.document);
+        const article = new Readability(dom.window.document).parse();
+        const selected = selectRelevantContent({
+            source: article?.content ?? dom.window.document.body.innerHTML,
+            format: 'html', query, maxLength
+        });
+        const preferredSelection = selectRelevantContent({
+            source: preferredRegion,
+            format: 'html', query, maxLength
+        });
+        const querySelection = preferredSelection.matched ? preferredSelection : selected;
+        return {
+            mode: 'readability',
+            omitted: querySelection.omitted,
+            content: {
+                title: article?.title ?? (dom.window.document.title || undefined),
+                byline: article?.byline || undefined,
+                text: querySelection.text,
+                ...(querySelection.anchor ? { sectionAnchor: querySelection.anchor } : {})
+            }
+        };
+    }
+    catch {
+        let region = extractPreferredSection(html);
+        for (const tag of ['script', 'style', 'noscript', 'svg', 'template']) {
+            region = stripTagContent(region, tag);
+        }
+        const selected = selectRelevantContent({ source: region, format: 'html', query, maxLength });
+        return {
+            mode: 'fallback',
+            omitted: selected.omitted,
+            content: {
+                title: extractTitle(html),
+                text: selected.text,
+                ...(selected.anchor ? { sectionAnchor: selected.anchor } : {})
+            }
+        };
+    }
+}

package/dist/extract/section-selector.d.ts

@@ -0,0 +1,14 @@
+export type SelectionFormat = 'html' | 'markdown' | 'text';
+export type SectionSelection = {
+    text: string;
+    anchor?: string;
+    omitted: boolean;
+    matched: boolean;
+};
+export declare function selectRelevantContent({ source, format, query, maxLength }: {
+    source: string;
+    format: SelectionFormat;
+    query: string;
+    maxLength?: number;
+}): SectionSelection;
+export declare function selectRelevantExcerpt(text: string, query: string, maxLength: number): string;

package/dist/extract/section-selector.js

@@ -0,0 +1,233 @@
+import { JSDOM } from 'jsdom';
+const STOPWORDS = new Set([
+    'about', 'after', 'from', 'have', 'into', 'that', 'their', 'there', 'these',
+    'this', 'what', 'when', 'where', 'which', 'with', 'would', 'your', 'https', 'http'
+]);
+function clean(text) {
+    return text.replace(/\s+/g, ' ').trim();
+}
+function queryTerms(query) {
+    const withoutUrls = query.replace(/https?:\/\/\S+/gi, ' ');
+    return [...new Set((withoutUrls.toLowerCase().match(/[\p{L}\p{N}]{3,}/gu) ?? [])
+            .filter((term) => !STOPWORDS.has(term)))];
+}
+function scoreText(text, terms, allowPartialMatch = false) {
+    const normalized = text.toLowerCase();
+    const words = new Set(normalized.match(/[\p{L}\p{N}]{3,}/gu) ?? []);
+    return terms.reduce((score, term) => score + Number(words.has(term) ||
+        (/\p{Script=Han}/u.test(term) && normalized.includes(term)) ||
+        (allowPartialMatch && normalized.includes(term))), 0);
+}
+function splitLong(text, maxLength) {
+    const normalized = clean(text);
+    if (normalized.length <= maxLength)
+        return normalized ? [normalized] : [];
+    const words = normalized.split(' ');
+    const chunks = [];
+    let chunk = '';
+    for (let word of words) {
+        if (word.length > maxLength) {
+            if (chunk)
+                chunks.push(chunk);
+            while (word.length > maxLength) {
+                chunks.push(word.slice(0, maxLength));
+                word = word.slice(maxLength);
+            }
+            chunk = word;
+            continue;
+        }
+        if (chunk && chunk.length + word.length + 1 > maxLength) {
+            chunks.push(chunk);
+            chunk = word;
+        }
+        else {
+            chunk = chunk ? `${chunk} ${word}` : word;
+        }
+    }
+    if (chunk)
+        chunks.push(chunk);
+    return chunks;
+}
+function htmlSections(source) {
+    const document = new JSDOM(source).window.document;
+    const main = document.querySelector('main');
+    const articles = document.querySelectorAll('article');
+    const root = main ?? (articles.length === 1 ? articles[0] : document.body);
+    root.querySelectorAll('script, style, noscript, svg, template').forEach((element) => element.remove());
+    const chromeTags = new Set(['nav', 'aside', 'header', 'footer']);
+    const chromeRoles = new Set(['navigation', 'complementary', 'banner', 'contentinfo']);
+    root.querySelectorAll('nav, aside, header, footer, [role], [aria-label]').forEach((element) => {
+        const roles = (element.getAttribute('role') ?? '').toLowerCase().split(/\s+/);
+        const label = element.getAttribute('aria-label') ?? '';
+        if (chromeTags.has(element.tagName.toLowerCase()) || roles.some((role) => chromeRoles.has(role)) || /\bbreadcrumbs?\b/i.test(label)) {
+            element.remove();
+        }
+    });
+    const sections = [{ blocks: [] }];
+    let current = sections[0];
+    let pendingAnchor;
+    const contentSelector = 'h1, h2, h3, h4, h5, h6, p, li, pre, blockquote, dt, dd, div, a[name]';
+    for (const element of root.querySelectorAll(contentSelector)) {
+        const tag = element.tagName.toLowerCase();
+        if (tag === 'a') {
+            pendingAnchor = element.getAttribute('name') ?? element.id ?? pendingAnchor;
+            continue;
+        }
+        if (/^h[1-6]$/.test(tag)) {
+            const heading = clean(element.textContent ?? '');
+            if (!heading)
+                continue;
+            const anchor = element.id || element.querySelector('[id], a[name]')?.getAttribute('id') ||
+                element.querySelector('a[name]')?.getAttribute('name') || pendingAnchor || undefined;
+            current = { heading, anchor, blocks: [] };
+            sections.push(current);
+            pendingAnchor = undefined;
+            continue;
+        }
+        if (tag === 'div' && element.querySelector(contentSelector))
+            continue;
+        if (element.closest('li, blockquote') !== element && element.parentElement?.closest('li, blockquote'))
+            continue;
+        const text = clean(element.textContent ?? '');
+        if (text) {
+            if (!current.anchor && pendingAnchor)
+                current.anchor = pendingAnchor;
+            current.blocks.push(text);
+            pendingAnchor = undefined;
+        }
+    }
+    if (sections.every((section) => section.blocks.length === 0)) {
+        const text = clean(root.textContent ?? '');
+        if (text)
+            return [{ blocks: [text] }];
+    }
+    return sections.filter((section) => section.heading || section.blocks.length);
+}
+function markdownSections(source) {
+    const sections = [{ blocks: [] }];
+    let current = sections[0];
+    let paragraph = [];
+    const flush = () => {
+        if (paragraph.length)
+            current.blocks.push(clean(paragraph.join(' ')));
+        paragraph = [];
+    };
+    for (const line of source.split(/\r?\n/)) {
+        const heading = line.match(/^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$/);
+        if (heading) {
+            flush();
+            current = { heading: clean(heading[1]), blocks: [] };
+            sections.push(current);
+        }
+        else if (!line.trim()) {
+            flush();
+        }
+        else {
+            paragraph.push(line.trim());
+        }
+    }
+    flush();
+    return sections.filter((section) => section.heading || section.blocks.length);
+}
+function plainSections(source) {
+    return source.split(/\n\s*\n/).map((text) => ({ blocks: [clean(text)] }))
+        .filter((section) => section.blocks[0]);
+}
+function renderSections(sections) {
+    return sections.flatMap((section) => [section.heading, ...section.blocks].filter(Boolean)).join('\n\n');
+}
+function makeWindows(sections, terms, maxLength) {
+    const windows = [];
+    const windowLimit = Math.max(1, Math.min(1200, maxLength));
+    for (const section of sections) {
+        const fullHeading = section.heading ?? '';
+        if (fullHeading.length + 2 > windowLimit) {
+            const heading = fullHeading.slice(0, windowLimit);
+            windows.push({
+                text: heading,
+                anchor: section.anchor,
+                score: scoreText(heading, terms) * 3,
+                index: windows.length
+            });
+            for (const block of section.blocks) {
+                const allowPartialMatch = /^\p{L}[\p{L}\p{N}]*$/u.test(block) && block.length > windowLimit;
+                for (const chunk of splitLong(block, windowLimit)) {
+                    windows.push({
+                        text: chunk,
+                        anchor: section.anchor,
+                        score: scoreText(chunk, terms, allowPartialMatch),
+                        index: windows.length
+                    });
+                }
+            }
+            continue;
+        }
+        const heading = fullHeading;
+        const headingLength = heading ? heading.length + 2 : 0;
+        const bodyLimit = Math.max(0, windowLimit - headingLength);
+        const chunks = bodyLimit > 0 ? section.blocks.flatMap((block) => {
+            const allowPartialMatch = /^\p{L}[\p{L}\p{N}]*$/u.test(block) && block.length > bodyLimit;
+            return splitLong(block, bodyLimit).map((text) => ({ text, allowPartialMatch }));
+        }) : [];
+        if (chunks.length === 0 && heading)
+            chunks.push({ text: '', allowPartialMatch: false });
+        let body = '';
+        let bodyAllowsPartialMatch = false;
+        const add = () => {
+            const text = clean([heading, body].filter(Boolean).join('\n\n'));
+            windows.push({ text, anchor: section.anchor, score: scoreText(heading, terms) * 3 + scoreText(body, terms, bodyAllowsPartialMatch), index: windows.length });
+            body = '';
+            bodyAllowsPartialMatch = false;
+        };
+        for (const chunk of chunks) {
+            if (body && body.length + chunk.text.length + 2 > bodyLimit)
+                add();
+            body = body ? `${body}\n\n${chunk.text}` : chunk.text;
+            bodyAllowsPartialMatch = !body.includes('\n\n') && chunk.allowPartialMatch;
+        }
+        if (body || heading)
+            add();
+    }
+    return windows;
+}
+export function selectRelevantContent({ source, format, query, maxLength = 4000 }) {
+    const sections = format === 'html' ? htmlSections(source) : format === 'markdown'
+        ? markdownSections(source) : plainSections(source);
+    const fullText = renderSections(sections);
+    if (maxLength <= 0) {
+        return { text: '', omitted: fullText.length > 0, matched: false };
+    }
+    const terms = queryTerms(query);
+    const windows = makeWindows(sections, terms, maxLength);
+    const ranked = windows.filter((window) => window.score > 0)
+        .sort((a, b) => b.score - a.score || a.index - b.index);
+    if (ranked.length === 0) {
+        return { text: fullText.slice(0, maxLength), omitted: fullText.length > maxLength, matched: false };
+    }
+    const selected = [];
+    let remaining = maxLength;
+    for (const window of ranked) {
+        const separatorLength = selected.length ? 2 : 0;
+        if (window.text.length + separatorLength > remaining)
+            continue;
+        selected.push(window);
+        remaining -= window.text.length + separatorLength;
+    }
+    selected.sort((a, b) => a.index - b.index);
+    const text = selected.map((window) => window.text).join('\n\n');
+    return {
+        text,
+        anchor: ranked.find((window) => selected.includes(window))?.anchor,
+        omitted: selected.length < windows.length,
+        matched: true
+    };
+}
+export function selectRelevantExcerpt(text, query, maxLength) {
+    const terms = queryTerms(query);
+    const chunks = text.split(/\n\s*\n/).flatMap((paragraph) => splitLong(paragraph, maxLength));
+    if (chunks.length === 0)
+        return '';
+    const best = chunks.map((chunk, index) => ({ chunk, index, score: scoreText(chunk, terms) }))
+        .sort((a, b) => b.score - a.score || a.index - b.index)[0];
+    return best.chunk.slice(0, maxLength);
+}

package/dist/fetch/firecrawl-fetch.d.ts

@@ -5,4 +5,4 @@ export declare function createFirecrawlFetcher({ baseUrl, apiKey, options, fetch
     apiKey?: string;
     options?: FirecrawlOptions;
     fetchImpl?: typeof fetch;
-}): (url: string) => Promise<WebFetchResponse>;
+}): (url: string, query?: string) => Promise<WebFetchResponse>;

package/dist/fetch/firecrawl-fetch.js

@@ -1,4 +1,6 @@
 import { classifyHttpFailure, readResponseParts } from '../backends/provider-failure.js';
+import { selectRelevantContent } from '../extract/section-selector.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
 function buildScrapeUrl(baseUrl) {
     return new URL('/v1/scrape', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`).toString();
 }
@@ -6,7 +8,7 @@ function errorMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
 export function createFirecrawlFetcher({ baseUrl, apiKey, options, fetchImpl = fetch }) {
-    return async function firecrawlFetch(url) {
+    return async function firecrawlFetch(url, query) {
         const failed = (message, failure) => ({
             status: 'error',
             url,
@@ -56,11 +58,9 @@ export function createFirecrawlFetcher({ baseUrl, apiKey, options, fetchImpl = f
                 httpStatus: response.status
             });
         }
-        const text = typeof parsed.data?.markdown === 'string'
-            ? parsed.data.markdown
-            : typeof parsed.data?.html === 'string'
-                ? parsed.data.html
-                : '';
+        const markdown = typeof parsed.data?.markdown === 'string' ? parsed.data.markdown : undefined;
+        const html = typeof parsed.data?.html === 'string' ? parsed.data.html : undefined;
+        const text = markdown ?? html ?? '';
         const resolvedUrl = typeof parsed.data?.metadata?.sourceURL === 'string'
             ? parsed.data.metadata.sourceURL
             : url;
@@ -75,11 +75,20 @@ export function createFirecrawlFetcher({ baseUrl, apiKey, options, fetchImpl = f
                 error: { code: 'WEAK_EXTRACTION', message: 'Firecrawl did not return useful page text.' }
             };
         }
+        const selection = query
+            ? selectRelevantContent({ source: text, format: markdown !== undefined ? 'markdown' : 'html', query })
+            : undefined;
+        const selectedText = selection?.text ?? text;
         return {
             status: 'ok',
             url: resolvedUrl,
-            content: { title, text },
-            metadata: { method: 'firecrawl', cacheHit: false, truncated: text.length >= 4000 }
+            content: {
+                title,
+                text: selectedText,
+                ...(hasBotCheckContent(text, markdown !== undefined ? 'markdown' : 'html') ? { botCheck: true } : {}),
+                ...(selection?.anchor ? { sectionAnchor: selection.anchor } : {})
+            },
+            metadata: { method: 'firecrawl', cacheHit: false, truncated: selection?.omitted ?? text.length >= 4000 }
         };
     };
 }

package/dist/fetch/headless-fetch.d.ts

@@ -8,8 +8,9 @@ export type BrowserProxyOptions = {
     password?: string;
     bypass?: string;
 };
-export declare function headlessFetch(url: string, { configuredPath, proxy, guard, guardProxy, resolveBrowser, launchBrowser, now }?: {
+export declare function headlessFetch(url: string, { configuredPath, query, proxy, guard, guardProxy, resolveBrowser, launchBrowser, now }?: {
     configuredPath?: string;
+    query?: string;
     /** Only used without a guard. With a guard, Chromium always goes through the guard proxy, which chains upstream itself. */
     proxy?: BrowserProxyOptions;
     guard?: NetworkGuard;

package/dist/fetch/headless-fetch.js

@@ -1,5 +1,6 @@
 import { chromium } from 'playwright';
-import { extractReadableContentSafely } from '../extract/readability.js';
+import { extractReadableContentForQuery, extractReadableContentSafely } from '../extract/readability.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
 import { resolveBrowserExecutable } from './browser-resolution.js';
 import { BLOCKED_HEADER } from './guard-proxy.js';
 import { BLOCKED_PRIVATE_ADDRESS, BlockedAddressError, UPSTREAM_PROXY_REFUSED } from './network-guard.js';
@@ -30,7 +31,7 @@ function hostnameOf(url) {
 function normalizeHost(host) {
     return host.replace(/^\[|\]$/g, '').toLowerCase().replace(/\.+$/, '');
 }
-export async function headlessFetch(url, { configuredPath, proxy, guard, guardProxy, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless, proxy }) => chromium.launch(executablePath ? { executablePath, headless, ...(proxy ? { proxy } : {}) } : { headless, ...(proxy ? { proxy } : {}) }), now = () => Date.now() } = {}) {
+export async function headlessFetch(url, { configuredPath, query, proxy, guard, guardProxy, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless, proxy }) => chromium.launch(executablePath ? { executablePath, headless, ...(proxy ? { proxy } : {}) } : { headless, ...(proxy ? { proxy } : {}) }), now = () => Date.now() } = {}) {
     if (guard) {
         const hostname = hostnameOf(url);
         if (!guardProxy) {
@@ -199,12 +200,16 @@ export async function headlessFetch(url, { configuredPath, proxy, guard, guardPr
         const html = await page.content();
         const finishedAt = now();
         const blockedSubresources = subresourceRefusals();
-        const extraction = extractReadableContentSafely(html);
+        const baselineExtraction = extractReadableContentSafely(html);
+        const queryExtraction = query ? extractReadableContentForQuery(html, query) : undefined;
+        const extraction = queryExtraction ?? baselineExtraction;
+        const cleanedBaselineText = cleanupRenderedText(baselineExtraction.content.text);
         const cleanedContent = {
             ...extraction.content,
-            text: cleanupRenderedText(extraction.content.text)
+            text: cleanupRenderedText(extraction.content.text),
+            ...(hasBotCheckContent(html, 'html') ? { botCheck: true } : {})
         };
-        if (!cleanedContent.text || cleanedContent.text.length < 40) {
+        if (!cleanedBaselineText || cleanedBaselineText.length < 40) {
             return {
                 status: 'blocked',
                 url,
@@ -230,7 +235,7 @@ export async function headlessFetch(url, { configuredPath, proxy, guard, guardPr
                 cacheHit: false,
                 browser: browserName,
                 navigationMs: finishedAt - startedAt,
-                truncated: cleanedContent.text.length >= 4000,
+                truncated: queryExtraction?.omitted ?? cleanedContent.text.length >= 4000,
                 ...(blockedSubresources > 0 ? { blockedSubresources } : {})
             }
         };

package/dist/fetch/http-fetch.d.ts

@@ -1,4 +1,4 @@
 import type { WebFetchResponse } from '../types.js';
 export declare function createHttpFetcher({ fetchImpl }?: {
     fetchImpl?: typeof fetch;
-}): (url: string) => Promise<WebFetchResponse>;
+}): (url: string, query?: string) => Promise<WebFetchResponse>;

package/dist/fetch/http-fetch.js

@@ -1,4 +1,5 @@
-import { extractReadableContentSafely } from '../extract/readability.js';
+import { extractReadableContentForQuery, extractReadableContentSafely } from '../extract/readability.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
 import { findGuardError } from './network-guard.js';
 function looksLikeScriptShell(html) {
     const lower = html.toLowerCase();
@@ -15,7 +16,7 @@ function isWeakHttpContent(options) {
     return veryShortBody && (lowDensity || hasGenericShellMarker);
 }
 export function createHttpFetcher({ fetchImpl = fetch } = {}) {
-    return async function httpFetch(url) {
+    return async function httpFetch(url, query) {
         let response;
         try {
             response = await fetchImpl(url);
@@ -40,11 +41,16 @@ export function createHttpFetcher({ fetchImpl = fetch } = {}) {
             };
         }
         const html = await response.text();
-        const extraction = extractReadableContentSafely(html);
-        const content = extraction.content;
+        const baselineExtraction = extractReadableContentSafely(html);
+        const queryExtraction = query ? extractReadableContentForQuery(html, query) : undefined;
+        const extraction = queryExtraction ?? baselineExtraction;
+        const content = {
+            ...extraction.content,
+            ...(hasBotCheckContent(html, 'html') ? { botCheck: true } : {})
+        };
         if (looksLikeScriptShell(html) ||
-            content.text.length < 40 ||
-            isWeakHttpContent({ html, title: content.title, text: content.text })) {
+            baselineExtraction.content.text.length < 40 ||
+            isWeakHttpContent({ html, title: baselineExtraction.content.title, text: baselineExtraction.content.text })) {
             return {
                 status: 'needs_headless',
                 url: response.url,
@@ -59,7 +65,7 @@ export function createHttpFetcher({ fetchImpl = fetch } = {}) {
             status: 'ok',
             url: response.url,
             content,
-            metadata: { method: 'http', cacheHit: false, contentType, truncated: content.text.length >= 4000 }
+            metadata: { method: 'http', cacheHit: false, contentType, truncated: queryExtraction?.omitted ?? content.text.length >= 4000 }
         };
     };
 }

package/dist/orchestration/index.d.ts

@@ -1,16 +1,12 @@
 import type { BackendConfig } from '../backends/config.js';
-import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
 export declare function createResearchWorkflow({ backendConfig, search, fetchPage, headlessFetch }?: {
     backendConfig?: BackendConfig;
     search?: (input: {
         query: string;
     }) => Promise<WebSearchResponse>;
-    fetchPage?: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
-    headlessFetch?: (input: {
-        url: string;
-    }) => Promise<WebFetchHeadlessResponse>;
+    fetchPage?: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
+    headlessFetch?: (input: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;
 }): {
     run({ query }: {
         query: string;

package/dist/orchestration/research-orchestrator.d.ts

@@ -1,4 +1,4 @@
-import type { Attempt, SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
+import type { Attempt, ResearchFetchInput, SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
 import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
 import { type EvidenceCaveatReason } from './evidence-quality.js';
 export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
@@ -9,12 +9,8 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
             maxFetches: number;
         }) => Promise<ResearchWorkerResult>;
     };
-    fetchDirect?: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
-    headlessFetch: (input: {
-        url: string;
-    }) => Promise<WebFetchHeadlessResponse>;
+    fetchDirect?: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
+    headlessFetch: (input: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;
 }): {
     run({ query }: {
         query: string;

package/dist/orchestration/research-orchestrator.js

@@ -5,25 +5,26 @@ import { classifySourceProfile } from './source-profile.js';
 import { extractDirectUrls } from './direct-url.js';
 import { decideNextResearchStep } from './stop-decider.js';
 import { analyzeEvidenceQuality } from './evidence-quality.js';
+import { selectRelevantExcerpt } from '../extract/section-selector.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
 const DEFAULT_MAX_PASSES = 3;
 const DEFAULT_MAX_FETCHES_PER_PASS = 4;
 const DEFAULT_MAX_HEADLESS_ATTEMPTS = 2;
 function classifyEvidenceUrl(url) {
     return classifySourceProfile(url).sourceKind;
 }
-function summarizeText(text, maxLength = 180) {
-    return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
-}
 function isReaderMethod(method) {
     return method === 'github' || method === 'pdf' || method === 'youtube';
 }
-function isBotCheckContent({ title = '', text }) {
-    return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
+function isBotCheckContent({ title = '', text, botCheck }) {
+    if (botCheck)
+        return true;
+    return hasBotCheckContent(`${title}\n${text}`);
 }
-function evidenceFromFetch(result) {
+function evidenceFromFetch(result, query) {
     if (result.status !== 'ok' || !result.content?.text.trim())
         return null;
-    if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
+    if (isBotCheckContent({ title: result.content.title, text: result.content.text, botCheck: result.content.botCheck }))
         return null;
     if (isReaderMethod(result.metadata.method)) {
         return {
@@ -40,22 +41,22 @@ function evidenceFromFetch(result) {
         url: result.url,
         sourceKind: classifyEvidenceUrl(result.url),
         method: result.metadata.method,
-        summary: summarizeText(result.content.text),
-        supports: [summarizeText(result.content.text, 120)]
+        summary: selectRelevantExcerpt(result.content.text, query, 180),
+        supports: [selectRelevantExcerpt(result.content.text, query, 120)]
     };
 }
-function evidenceFromHeadless(result) {
+function evidenceFromHeadless(result, query) {
     if (result.status !== 'ok' || !result.content?.text.trim())
         return null;
-    if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
+    if (isBotCheckContent({ title: result.content.title, text: result.content.text, botCheck: result.content.botCheck }))
         return null;
     return {
         title: result.content.title ?? result.url,
         url: result.url,
         sourceKind: classifyEvidenceUrl(result.url),
         method: 'headless',
-        summary: summarizeText(result.content.text),
-        supports: [summarizeText(result.content.text, 120)]
+        summary: selectRelevantExcerpt(result.content.text, query, 180),
+        supports: [selectRelevantExcerpt(result.content.text, query, 120)]
     };
 }
 function combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget }) {
@@ -128,10 +129,10 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
             }
             if (fetchDirect) {
                 for (const url of extractDirectUrls(query).slice(0, 3)) {
-                    const directResult = await fetchDirect({ url });
+                    const directResult = await fetchDirect({ url, query });
                     if (directResult.metadata.attempts)
                         runAttempts.push(...directResult.metadata.attempts);
-                    const directEvidence = evidenceFromFetch(directResult);
+                    const directEvidence = evidenceFromFetch(directResult, query);
                     if (directEvidence) {
                         allEvidence.push(directEvidence);
                         continue;
@@ -146,8 +147,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     if (shouldRetryDirectWithHeadless(directResult, directEvidence)) {
                         if (headlessAttempts < DEFAULT_MAX_HEADLESS_ATTEMPTS) {
                             headlessAttempts++;
-                            const headlessResult = await headlessFetch({ url: directResult.url });
-                            const headlessEvidence = evidenceFromHeadless(headlessResult);
+                            const headlessResult = await headlessFetch({ url: directResult.url, query });
+                            const headlessEvidence = evidenceFromHeadless(headlessResult, query);
                             if (headlessEvidence) {
                                 allEvidence.push(headlessEvidence);
                             }
@@ -255,8 +256,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
                     });
                     if (decision.action === 'headless') {
                         headlessAttempts++;
-                        const headlessResult = await headlessFetch({ url: decision.url });
-                        const headlessEvidence = evidenceFromHeadless(headlessResult);
+                        const headlessResult = await headlessFetch({ url: decision.url, query });
+                        const headlessEvidence = evidenceFromHeadless(headlessResult, query);
                         if (headlessEvidence) {
                             allEvidence.push(headlessEvidence);
                             const updatedRanked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));

package/dist/orchestration/research-worker.d.ts

@@ -1,12 +1,10 @@
-import type { WebFetchResponse, WebSearchResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchResponse, WebSearchResponse } from '../types.js';
 import type { ResearchWorkerResult } from './research-types.js';
 export declare function createResearchWorker({ search, fetchPage }: {
     search: (input: {
         query: string;
     }) => Promise<WebSearchResponse>;
-    fetchPage: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
+    fetchPage: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
 }): {
     run({ query, maxSearchRounds, maxFetches }: {
         query: string;

package/dist/orchestration/research-worker.js

@@ -1,23 +1,24 @@
 import { failureOf, isTerminalFailure } from '../backends/failure.js';
 import { selectCandidates } from './candidate-selector.js';
 import { classifySourceProfile } from './source-profile.js';
+import { selectRelevantExcerpt } from '../extract/section-selector.js';
+import { hasBotCheckContent } from '../extract/bot-check.js';
 function classifySource(url) {
     return classifySourceProfile(url).sourceKind;
 }
-function summarizeText(text, maxLength = 180) {
-    return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
-}
 function isReaderMethod(method) {
     return method === 'github' || method === 'pdf' || method === 'youtube';
 }
-function isBotCheckContent({ title = '', text }) {
-    return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
+function isBotCheckContent({ title = '', text, botCheck }) {
+    if (botCheck)
+        return true;
+    return hasBotCheckContent(`${title}\n${text}`);
 }
-function evidenceFromFetch(fetched, fallbackTitle) {
+function evidenceFromFetch(fetched, fallbackTitle, query) {
     const content = fetched.content;
     if (fetched.status !== 'ok' || !content)
         return null;
-    if (isBotCheckContent({ title: content.title, text: content.text }))
+    if (isBotCheckContent({ title: content.title, text: content.text, botCheck: content.botCheck }))
         return null;
     // A successful reader read with usable text is primary content, exempt from the
     // package-page filter below.
@@ -40,14 +41,14 @@ function evidenceFromFetch(fetched, fallbackTitle) {
         url: fetched.url,
         sourceKind,
         method: fetched.metadata.method,
-        summary: summarizeText(content.text),
-        supports: [summarizeText(content.text, 120)]
+        summary: selectRelevantExcerpt(content.text, query, 180),
+        supports: [selectRelevantExcerpt(content.text, query, 120)]
     };
 }
 function lowValueOutcomeFromFetch(fetched) {
     if (fetched.status !== 'ok' || !fetched.content)
         return null;
-    if (isBotCheckContent({ title: fetched.content.title, text: fetched.content.text })) {
+    if (isBotCheckContent({ title: fetched.content.title, text: fetched.content.text, botCheck: fetched.content.botCheck })) {
         return {
             kind: 'bot-check',
             url: fetched.url,
@@ -145,11 +146,11 @@ export function createResearchWorker({ search, fetchPage }) {
             });
             const fetchAttempts = [];
             for (const candidate of candidates) {
-                const fetched = await fetchPage({ url: candidate.url });
+                const fetched = await fetchPage({ url: candidate.url, query });
                 if (fetched.metadata.attempts)
                     fetchAttempts.push(...fetched.metadata.attempts);
                 if (fetched.status === 'ok') {
-                    const parsedEvidence = evidenceFromFetch(fetched, candidate.title);
+                    const parsedEvidence = evidenceFromFetch(fetched, candidate.title, query);
                     if (parsedEvidence) {
                         evidence.push(parsedEvidence);
                         continue;

package/dist/readers/resolver.d.ts

@@ -1,12 +1,8 @@
-import type { WebFetchResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchResponse } from '../types.js';
 import type { SpecialContentReader } from './types.js';
 type ResolverDeps = {
     readers: SpecialContentReader[];
-    fallback: (input: {
-        url: string;
-    }) => Promise<WebFetchResponse>;
+    fallback: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
 };
-export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input: {
-    url: string;
-}) => Promise<WebFetchResponse>;
+export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input: ResearchFetchInput) => Promise<WebFetchResponse>;
 export {};

package/dist/search/youcom.d.ts

@@ -1,3 +1,6 @@
+import { type Normalized } from './json-provider.js';
+export declare const YOUCOM_SEARCH_URL = "https://ydc-index.io/v1/search";
+export declare function normalizeYouComResults(json: unknown): Normalized | undefined;
 export declare function createYouComSearchTool({ apiKey, fetchImpl }: {
     apiKey?: string;
     fetchImpl?: typeof fetch;

package/dist/search/youcom.js

@@ -1,5 +1,40 @@
-import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
-const YOUCOM_SEARCH_URL = 'https://api.you.com/v1/agents/search';
+import { createJsonSearchProvider } from './json-provider.js';
+export const YOUCOM_SEARCH_URL = 'https://ydc-index.io/v1/search';
+function normalizeResult(item, snippet) {
+    if (!item || typeof item !== 'object')
+        return [];
+    const result = item;
+    return typeof result.title === 'string' && typeof result.url === 'string'
+        ? [{ title: result.title, url: result.url, snippet: typeof result.description === 'string' ? result.description : snippet }]
+        : [];
+}
+export function normalizeYouComResults(json) {
+    if (!json || typeof json !== 'object')
+        return undefined;
+    const response = json;
+    if (!response.results || typeof response.results !== 'object' || Array.isArray(response.results))
+        return undefined;
+    const sections = response.results;
+    if (sections.web === undefined && sections.news === undefined)
+        return undefined;
+    if (sections.web !== undefined && !Array.isArray(sections.web))
+        return undefined;
+    if (sections.news !== undefined && !Array.isArray(sections.news))
+        return undefined;
+    const web = sections.web ?? [];
+    const news = sections.news ?? [];
+    return {
+        rawCount: web.length + news.length,
+        results: [
+            ...web.flatMap((item) => {
+                const snippets = item && typeof item === 'object' ? item.snippets : undefined;
+                const snippet = Array.isArray(snippets) ? snippets.find((value) => typeof value === 'string') ?? '' : '';
+                return normalizeResult(item, snippet);
+            }),
+            ...news.flatMap((item) => normalizeResult(item, ''))
+        ]
+    };
+}
 export function createYouComSearchTool({ apiKey, fetchImpl = fetch }) {
     return createJsonSearchProvider({
         name: 'youcom',
@@ -12,9 +47,9 @@ export function createYouComSearchTool({ apiKey, fetchImpl = fetch }) {
             init: {
                 method: 'POST',
                 headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'X-API-Key': apiKey ?? '' },
-                body: JSON.stringify({ query, max_results: 10 })
+                body: JSON.stringify({ query, count: 10 })
             }
         }),
-        normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'snippet')
+        normalize: normalizeYouComResults
     });
 }

package/dist/tools/web-fetch-headless.d.ts

@@ -1,6 +1,4 @@
-import type { WebFetchHeadlessResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchHeadlessResponse } from '../types.js';
 export declare function createWebFetchHeadlessTool({ fetchPage }?: {
-    fetchPage?: (url: string) => Promise<WebFetchHeadlessResponse>;
-}): ({ url }: {
-    url: string;
-}) => Promise<WebFetchHeadlessResponse>;
+    fetchPage?: (input: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;
+}): ({ url, query }: ResearchFetchInput) => Promise<WebFetchHeadlessResponse>;

package/dist/tools/web-fetch-headless.js

@@ -1,7 +1,7 @@
 import { headlessFetch } from '../fetch/headless-fetch.js';
 import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
-export function createWebFetchHeadlessTool({ fetchPage = headlessFetch } = {}) {
-    return async function webFetchHeadless({ url }) {
+export function createWebFetchHeadlessTool({ fetchPage = ({ url, query }) => headlessFetch(url, { query }) } = {}) {
+    return async function webFetchHeadless({ url, query }) {
         if (!/^https?:\/\//.test(url)) {
             const result = {
                 status: 'unsupported',
@@ -14,7 +14,7 @@ export function createWebFetchHeadlessTool({ fetchPage = headlessFetch } = {}) {
                 presentation: buildFetchPresentation(result)
             };
         }
-        const result = await fetchPage(url);
+        const result = await fetchPage({ url, ...(query ? { query } : {}) });
         return {
             ...result,
             presentation: buildFetchPresentation(result)

package/dist/tools/web-fetch.d.ts

@@ -1,6 +1,4 @@
-import type { WebFetchResponse } from '../types.js';
+import type { ResearchFetchInput, WebFetchResponse } from '../types.js';
 export declare function createWebFetchTool({ fetchPage }?: {
-    fetchPage?: (url: string) => Promise<WebFetchResponse>;
-}): ({ url }: {
-    url: string;
-}) => Promise<WebFetchResponse>;
+    fetchPage?: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
+}): ({ url, query }: ResearchFetchInput) => Promise<WebFetchResponse>;

package/dist/tools/web-fetch.js

@@ -1,7 +1,7 @@
 import { createHttpFetcher } from '../fetch/http-fetch.js';
 import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
-export function createWebFetchTool({ fetchPage = createHttpFetcher() } = {}) {
-    return async function webFetch({ url }) {
+export function createWebFetchTool({ fetchPage = ({ url, query }) => createHttpFetcher()(url, query) } = {}) {
+    return async function webFetch({ url, query }) {
         if (!/^https?:\/\//.test(url)) {
             const result = {
                 status: 'unsupported',
@@ -14,7 +14,7 @@ export function createWebFetchTool({ fetchPage = createHttpFetcher() } = {}) {
                 presentation: buildFetchPresentation(result)
             };
         }
-        const result = await fetchPage(url);
+        const result = await fetchPage({ url, ...(query ? { query } : {}) });
         return {
             ...result,
             presentation: buildFetchPresentation(result)

package/dist/types.d.ts

@@ -61,6 +61,10 @@ export type SearchMetadata = {
     coverage?: SearchCoverage;
 };
 export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
+export type ResearchFetchInput = {
+    url: string;
+    query?: string;
+};
 export type FetchMetadata = {
     method: FetchMethod;
     cacheHit: boolean;
@@ -78,6 +82,10 @@ export type ExtractedContent = {
     title?: string;
     byline?: string;
     text: string;
+    /** A bot or security verification marker appeared in the source before query selection. */
+    botCheck?: boolean;
+    /** Anchor of the section chosen for research; direct fetches do not set this. */
+    sectionAnchor?: string;
 };
 export type WebSearchResponse = {
     status: 'ok' | 'error';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",