@demigodmode/pi-web-agent 1.11.0 → 1.13.0

package/dist/commands/web-agent-config.js

@@ -1,5 +1,6 @@
 import { DEFAULT_BACKEND_CONFIG, isValidProxyUrl, mergeBackendConfigLayers, stripProxyCredentials, validateBackendConfig, usableSearchProviders } from '../backends/config.js';
 import { checkBackendHealth } from '../backends/doctor.js';
+import { parseCidr } from '../fetch/network-guard.js';
 import { DynamicBorder, getSettingsListTheme } from '@earendil-works/pi-coding-agent';
 import { Container, Input, SelectList, SettingsList, Text } from '@earendil-works/pi-tui';
 import { DEFAULT_PRESENTATION_CONFIG, mergePresentationConfigLayers, resolvePresentationMode } from '../presentation/config.js';
@@ -32,7 +33,10 @@ function cloneBackendConfig(config) {
             options: config.fetch.options ? { ...config.fetch.options } : undefined
         },
         headless: { ...config.headless },
-        proxy: config.proxy ? { ...config.proxy } : undefined
+        proxy: config.proxy ? { ...config.proxy } : undefined,
+        network: config.network
+            ? { ...config.network, ...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {}) }
+            : undefined
     };
 }
 function sameJson(left, right) {
@@ -50,6 +54,22 @@ export function validateBackendUrl(value) {
         return { ok: false, message: 'Invalid URL. Include http:// or https://.' };
     }
 }
+function splitRanges(value) {
+    return value
+        .split(',')
+        .map((range) => range.trim())
+        .filter(Boolean);
+}
+export function validateAllowRanges(value) {
+    for (const range of splitRanges(value)) {
+        const cidr = parseCidr(range);
+        if (!cidr)
+            return { ok: false, message: `Not a valid CIDR range: ${range}` };
+        if (cidr.prefix === 0)
+            return { ok: false, message: `${range} allows every address. List specific ranges instead.` };
+    }
+    return { ok: true, value: splitRanges(value).join(', ') };
+}
 /**
  * The extension re-applies this patch on load, so `pending` here means the
  * patch could not be written (read-only install, permissions, or a dependency
@@ -140,7 +160,7 @@ function buildPresentationSettingsItems(scope, config) {
         }))
     ];
 }
-export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChange) {
+export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChange, validate = validateBackendUrl) {
     return (currentValue, done) => {
         onOpenChange?.(true);
         const initialValue = currentValue && currentValue !== 'not set' ? currentValue : placeholderUrl;
@@ -164,7 +184,7 @@ export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChang
                 finish('');
                 return;
             }
-            const validated = validateBackendUrl(value);
+            const validated = validate(value);
             if (!validated.ok) {
                 showError(validated.message);
                 return;
@@ -276,6 +296,20 @@ function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange
             label: 'Proxy URL',
             currentValue: backends.proxy ? stripProxyCredentials(backends.proxy.url) : 'not set',
             submenu: createBackendUrlEditor(theme, 'HTTP proxy URL', 'http://127.0.0.1:7890', onUrlEditorOpenChange)
+        },
+        {
+            id: 'backend:network:allowRanges',
+            label: 'Network allow list',
+            currentValue: backends.network?.allowRanges?.length ? backends.network.allowRanges.join(', ') : 'not set',
+            submenu: createBackendUrlEditor(theme, 'Private ranges to allow, comma separated CIDRs, e.g. 198.18.0.0/15', 
+            // No prefilled example: pressing enter on an empty list must not quietly add an exception.
+            '', onUrlEditorOpenChange, validateAllowRanges)
+        },
+        {
+            id: 'backend:network:trustProxyDns',
+            label: 'Trust the upstream proxy to enforce private-address restrictions',
+            currentValue: backends.network?.trustProxyDns ? 'on' : 'off',
+            values: ['off', 'on']
         }
     ];
 }
@@ -456,6 +490,25 @@ export function applySettingsValue(state, id, newValue) {
             delete currentBackends.proxy;
         }
     }
+    if (id === 'backend:network:allowRanges') {
+        const ranges = splitRanges(newValue);
+        const next = { ...currentBackends.network };
+        if (ranges.length > 0) {
+            next.allowRanges = ranges;
+        }
+        else {
+            delete next.allowRanges;
+        }
+        if (Object.keys(next).length > 0) {
+            currentBackends.network = next;
+        }
+        else {
+            delete currentBackends.network;
+        }
+    }
+    if (id === 'backend:network:trustProxyDns') {
+        currentBackends.network = { ...currentBackends.network, trustProxyDns: newValue === 'on' };
+    }
     nextDrafts[nextScope] = currentDraft;
     nextBackendDrafts[nextScope] = currentBackends;
     return {
@@ -531,6 +584,23 @@ export function collapseBackendConfigToOverride(config, inheritedConfig) {
             override.proxy = { url: '' };
         }
     }
+    if (!sameJson(config.network, inheritedConfig.network)) {
+        if (config.network) {
+            override.network = {
+                ...config.network,
+                ...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {})
+            };
+            // Cleared at this scope: an explicit empty list overrides the parent's.
+            if (!config.network.allowRanges && inheritedConfig.network?.allowRanges)
+                override.network.allowRanges = [];
+        }
+        else if (inheritedConfig.network) {
+            override.network = {
+                ...(inheritedConfig.network.allowRanges ? { allowRanges: [] } : {}),
+                ...(inheritedConfig.network.trustProxyDns ? { trustProxyDns: false } : {})
+            };
+        }
+    }
     return override;
 }
 export function handleSettingsShortcut(data) {
@@ -753,6 +823,8 @@ export function registerWebAgentConfigCommands(pi, deps = {}) {
                     `typebox: ${typeboxOk ? 'ok' : 'missing'}`,
                     formatJitiCompatLine(checkCompat()),
                     formatBackendSummary(backendConfig),
+                    `network allow list: ${backendConfig.network?.allowRanges?.length ? backendConfig.network.allowRanges.join(', ') : 'none'}`,
+                    `trust upstream proxy for private addresses: ${backendConfig.network?.trustProxyDns ? 'on' : 'off'}`,
                     backendIssues.length > 0 ? `backend config: warning\n${backendIssues.join('\n')}` : 'backend config: ok',
                     ...backendHealth
                 ];

package/dist/extension.js

@@ -59,20 +59,64 @@ export default function extension(pi) {
     registerWebAgentConfigCommands(pi);
     const injectedWebExplore = pi.__webExploreTool;
     let cachedBackendKey;
+    let cachedWorkflow;
     let cachedWebExplore;
+    // Runs in flight per workflow, so a replaced workflow is never closed mid-run.
+    const activeRuns = new Map();
+    const retiring = new Set();
+    const closeWorkflow = (workflow) => {
+        retiring.delete(workflow);
+        void Promise.resolve(workflow.close?.()).catch(() => undefined);
+    };
+    const retire = (workflow) => {
+        if ((activeRuns.get(workflow) ?? 0) === 0) {
+            closeWorkflow(workflow);
+        }
+        else {
+            retiring.add(workflow);
+        }
+    };
+    const leased = (workflow) => ({
+        run: async (input) => {
+            activeRuns.set(workflow, (activeRuns.get(workflow) ?? 0) + 1);
+            try {
+                return await workflow.run(input);
+            }
+            finally {
+                const remaining = (activeRuns.get(workflow) ?? 1) - 1;
+                if (remaining > 0) {
+                    activeRuns.set(workflow, remaining);
+                }
+                else {
+                    activeRuns.delete(workflow);
+                    if (retiring.has(workflow))
+                        closeWorkflow(workflow);
+                }
+            }
+        }
+    });
     async function getConfiguredWebExplore() {
         if (injectedWebExplore)
             return injectedWebExplore;
         const backendConfig = await getEffectiveBackendConfig(pi);
         const backendKey = JSON.stringify(backendConfig);
         if (!cachedWebExplore || cachedBackendKey !== backendKey) {
+            if (cachedWorkflow)
+                retire(cachedWorkflow);
             cachedBackendKey = backendKey;
-            cachedWebExplore = createWebExploreTool({
-                explore: createResearchWorkflow({ backendConfig })
-            });
+            cachedWorkflow = createResearchWorkflow({ backendConfig });
+            cachedWebExplore = createWebExploreTool({ explore: leased(cachedWorkflow) });
         }
         return cachedWebExplore;
     }
+    pi.on('session_shutdown', async () => {
+        const workflow = cachedWorkflow;
+        cachedWorkflow = undefined;
+        cachedWebExplore = undefined;
+        cachedBackendKey = undefined;
+        if (workflow)
+            retire(workflow);
+    });
     pi.on('session_start', async (_event, ctx) => {
         try {
             const notice = await getUpdateChangelogNotice();

package/dist/extract/bot-check.d.ts

@@ -0,0 +1 @@
+export declare function hasBotCheckContent(source: string, format?: 'html' | 'markdown' | 'text'): boolean;

package/dist/extract/bot-check.js

@@ -0,0 +1,10 @@
+const BOT_CHECK_RE = /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i;
+export function hasBotCheckContent(source, format = 'text') {
+    const visibleSource = format === 'html'
+        ? source
+            .replace(/<!--[\s\S]*?-->/g, ' ')
+            .replace(/<(script|style|noscript|svg|template)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ')
+            .replace(/<[^>]+>/g, ' ')
+        : source;
+    return BOT_CHECK_RE.test(visibleSource);
+}

package/dist/extract/readability.d.ts

@@ -6,3 +6,7 @@ export type SafeReadableExtraction = {
 };
 export declare function extractReadableContent(html: string, maxLength?: number): ExtractedContent;
 export declare function extractReadableContentSafely(html: string, maxLength?: number): SafeReadableExtraction;
+/** Research-only extraction: choose relevant content before the usual text cap. */
+export declare function extractReadableContentForQuery(html: string, query: string, maxLength?: number): SafeReadableExtraction & {
+    omitted: boolean;
+};

package/dist/extract/readability.js

@@ -1,5 +1,6 @@
 import { Readability } from '@mozilla/readability';
 import { JSDOM, VirtualConsole } from 'jsdom';
+import { selectRelevantContent } from './section-selector.js';
 export function extractReadableContent(html, maxLength = 4000) {
     let stylesheetError;
     const virtualConsole = new VirtualConsole();
@@ -57,14 +58,25 @@ function extractPreferredSection(html) {
     const mainMatch = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
     if (mainMatch)
         return mainMatch[1];
-    const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
-    if (articleMatch)
-        return articleMatch[1];
+    const articles = html.match(/<article\b[^>]*>/gi) ?? [];
+    if (articles.length === 1) {
+        const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
+        if (articleMatch)
+            return articleMatch[1];
+    }
     const bodyMatch = html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
     if (bodyMatch)
         return bodyMatch[1];
     return html;
 }
+function extractPreferredDomSection(document) {
+    const main = document.querySelector('main');
+    const articles = document.querySelectorAll('article');
+    const region = main ?? (articles.length === 1 ? articles[0] : document.body);
+    const cleanedRegion = region.cloneNode(true);
+    cleanedRegion.querySelectorAll('script, style, noscript, svg, template').forEach((element) => element.remove());
+    return cleanedRegion.outerHTML;
+}
 function extractFallbackText(html, maxLength) {
     const title = extractTitle(html);
     let section = extractPreferredSection(html);
@@ -97,3 +109,55 @@ export function extractReadableContentSafely(html, maxLength = 4000) {
         };
     }
 }
+/** Research-only extraction: choose relevant content before the usual text cap. */
+export function extractReadableContentForQuery(html, query, maxLength = 4000) {
+    let stylesheetError;
+    const virtualConsole = new VirtualConsole();
+    virtualConsole.on('jsdomError', (error) => {
+        if (!stylesheetError && error.message.includes('Could not parse CSS stylesheet')) {
+            stylesheetError = error;
+        }
+    });
+    try {
+        const dom = new JSDOM(html, { url: 'https://example.com', virtualConsole });
+        if (stylesheetError)
+            throw stylesheetError;
+        const preferredRegion = extractPreferredDomSection(dom.window.document);
+        const article = new Readability(dom.window.document).parse();
+        const selected = selectRelevantContent({
+            source: article?.content ?? dom.window.document.body.innerHTML,
+            format: 'html', query, maxLength
+        });
+        const preferredSelection = selectRelevantContent({
+            source: preferredRegion,
+            format: 'html', query, maxLength
+        });
+        const querySelection = preferredSelection.matched ? preferredSelection : selected;
+        return {
+            mode: 'readability',
+            omitted: querySelection.omitted,
+            content: {
+                title: article?.title ?? (dom.window.document.title || undefined),
+                byline: article?.byline || undefined,
+                text: querySelection.text,
+                ...(querySelection.anchor ? { sectionAnchor: querySelection.anchor } : {})
+            }
+        };
+    }
+    catch {
+        let region = extractPreferredSection(html);
+        for (const tag of ['script', 'style', 'noscript', 'svg', 'template']) {
+            region = stripTagContent(region, tag);
+        }
+        const selected = selectRelevantContent({ source: region, format: 'html', query, maxLength });
+        return {
+            mode: 'fallback',
+            omitted: selected.omitted,
+            content: {
+                title: extractTitle(html),
+                text: selected.text,
+                ...(selected.anchor ? { sectionAnchor: selected.anchor } : {})
+            }
+        };
+    }
+}

package/dist/extract/section-selector.d.ts

@@ -0,0 +1,14 @@
+export type SelectionFormat = 'html' | 'markdown' | 'text';
+export type SectionSelection = {
+    text: string;
+    anchor?: string;
+    omitted: boolean;
+    matched: boolean;
+};
+export declare function selectRelevantContent({ source, format, query, maxLength }: {
+    source: string;
+    format: SelectionFormat;
+    query: string;
+    maxLength?: number;
+}): SectionSelection;
+export declare function selectRelevantExcerpt(text: string, query: string, maxLength: number): string;

package/dist/extract/section-selector.js

@@ -0,0 +1,233 @@
+import { JSDOM } from 'jsdom';
+const STOPWORDS = new Set([
+    'about', 'after', 'from', 'have', 'into', 'that', 'their', 'there', 'these',
+    'this', 'what', 'when', 'where', 'which', 'with', 'would', 'your', 'https', 'http'
+]);
+function clean(text) {
+    return text.replace(/\s+/g, ' ').trim();
+}
+function queryTerms(query) {
+    const withoutUrls = query.replace(/https?:\/\/\S+/gi, ' ');
+    return [...new Set((withoutUrls.toLowerCase().match(/[\p{L}\p{N}]{3,}/gu) ?? [])
+            .filter((term) => !STOPWORDS.has(term)))];
+}
+function scoreText(text, terms, allowPartialMatch = false) {
+    const normalized = text.toLowerCase();
+    const words = new Set(normalized.match(/[\p{L}\p{N}]{3,}/gu) ?? []);
+    return terms.reduce((score, term) => score + Number(words.has(term) ||
+        (/\p{Script=Han}/u.test(term) && normalized.includes(term)) ||
+        (allowPartialMatch && normalized.includes(term))), 0);
+}
+function splitLong(text, maxLength) {
+    const normalized = clean(text);
+    if (normalized.length <= maxLength)
+        return normalized ? [normalized] : [];
+    const words = normalized.split(' ');
+    const chunks = [];
+    let chunk = '';
+    for (let word of words) {
+        if (word.length > maxLength) {
+            if (chunk)
+                chunks.push(chunk);
+            while (word.length > maxLength) {
+                chunks.push(word.slice(0, maxLength));
+                word = word.slice(maxLength);
+            }
+            chunk = word;
+            continue;
+        }
+        if (chunk && chunk.length + word.length + 1 > maxLength) {
+            chunks.push(chunk);
+            chunk = word;
+        }
+        else {
+            chunk = chunk ? `${chunk} ${word}` : word;
+        }
+    }
+    if (chunk)
+        chunks.push(chunk);
+    return chunks;
+}
+function htmlSections(source) {
+    const document = new JSDOM(source).window.document;
+    const main = document.querySelector('main');
+    const articles = document.querySelectorAll('article');
+    const root = main ?? (articles.length === 1 ? articles[0] : document.body);
+    root.querySelectorAll('script, style, noscript, svg, template').forEach((element) => element.remove());
+    const chromeTags = new Set(['nav', 'aside', 'header', 'footer']);
+    const chromeRoles = new Set(['navigation', 'complementary', 'banner', 'contentinfo']);
+    root.querySelectorAll('nav, aside, header, footer, [role], [aria-label]').forEach((element) => {
+        const roles = (element.getAttribute('role') ?? '').toLowerCase().split(/\s+/);
+        const label = element.getAttribute('aria-label') ?? '';
+        if (chromeTags.has(element.tagName.toLowerCase()) || roles.some((role) => chromeRoles.has(role)) || /\bbreadcrumbs?\b/i.test(label)) {
+            element.remove();
+        }
+    });
+    const sections = [{ blocks: [] }];
+    let current = sections[0];
+    let pendingAnchor;
+    const contentSelector = 'h1, h2, h3, h4, h5, h6, p, li, pre, blockquote, dt, dd, div, a[name]';
+    for (const element of root.querySelectorAll(contentSelector)) {
+        const tag = element.tagName.toLowerCase();
+        if (tag === 'a') {
+            pendingAnchor = element.getAttribute('name') ?? element.id ?? pendingAnchor;
+            continue;
+        }
+        if (/^h[1-6]$/.test(tag)) {
+            const heading = clean(element.textContent ?? '');
+            if (!heading)
+                continue;
+            const anchor = element.id || element.querySelector('[id], a[name]')?.getAttribute('id') ||
+                element.querySelector('a[name]')?.getAttribute('name') || pendingAnchor || undefined;
+            current = { heading, anchor, blocks: [] };
+            sections.push(current);
+            pendingAnchor = undefined;
+            continue;
+        }
+        if (tag === 'div' && element.querySelector(contentSelector))
+            continue;
+        if (element.closest('li, blockquote') !== element && element.parentElement?.closest('li, blockquote'))
+            continue;
+        const text = clean(element.textContent ?? '');
+        if (text) {
+            if (!current.anchor && pendingAnchor)
+                current.anchor = pendingAnchor;
+            current.blocks.push(text);
+            pendingAnchor = undefined;
+        }
+    }
+    if (sections.every((section) => section.blocks.length === 0)) {
+        const text = clean(root.textContent ?? '');
+        if (text)
+            return [{ blocks: [text] }];
+    }
+    return sections.filter((section) => section.heading || section.blocks.length);
+}
+function markdownSections(source) {
+    const sections = [{ blocks: [] }];
+    let current = sections[0];
+    let paragraph = [];
+    const flush = () => {
+        if (paragraph.length)
+            current.blocks.push(clean(paragraph.join(' ')));
+        paragraph = [];
+    };
+    for (const line of source.split(/\r?\n/)) {
+        const heading = line.match(/^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$/);
+        if (heading) {
+            flush();
+            current = { heading: clean(heading[1]), blocks: [] };
+            sections.push(current);
+        }
+        else if (!line.trim()) {
+            flush();
+        }
+        else {
+            paragraph.push(line.trim());
+        }
+    }
+    flush();
+    return sections.filter((section) => section.heading || section.blocks.length);
+}
+function plainSections(source) {
+    return source.split(/\n\s*\n/).map((text) => ({ blocks: [clean(text)] }))
+        .filter((section) => section.blocks[0]);
+}
+function renderSections(sections) {
+    return sections.flatMap((section) => [section.heading, ...section.blocks].filter(Boolean)).join('\n\n');
+}
+function makeWindows(sections, terms, maxLength) {
+    const windows = [];
+    const windowLimit = Math.max(1, Math.min(1200, maxLength));
+    for (const section of sections) {
+        const fullHeading = section.heading ?? '';
+        if (fullHeading.length + 2 > windowLimit) {
+            const heading = fullHeading.slice(0, windowLimit);
+            windows.push({
+                text: heading,
+                anchor: section.anchor,
+                score: scoreText(heading, terms) * 3,
+                index: windows.length
+            });
+            for (const block of section.blocks) {
+                const allowPartialMatch = /^\p{L}[\p{L}\p{N}]*$/u.test(block) && block.length > windowLimit;
+                for (const chunk of splitLong(block, windowLimit)) {
+                    windows.push({
+                        text: chunk,
+                        anchor: section.anchor,
+                        score: scoreText(chunk, terms, allowPartialMatch),
+                        index: windows.length
+                    });
+                }
+            }
+            continue;
+        }
+        const heading = fullHeading;
+        const headingLength = heading ? heading.length + 2 : 0;
+        const bodyLimit = Math.max(0, windowLimit - headingLength);
+        const chunks = bodyLimit > 0 ? section.blocks.flatMap((block) => {
+            const allowPartialMatch = /^\p{L}[\p{L}\p{N}]*$/u.test(block) && block.length > bodyLimit;
+            return splitLong(block, bodyLimit).map((text) => ({ text, allowPartialMatch }));
+        }) : [];
+        if (chunks.length === 0 && heading)
+            chunks.push({ text: '', allowPartialMatch: false });
+        let body = '';
+        let bodyAllowsPartialMatch = false;
+        const add = () => {
+            const text = clean([heading, body].filter(Boolean).join('\n\n'));
+            windows.push({ text, anchor: section.anchor, score: scoreText(heading, terms) * 3 + scoreText(body, terms, bodyAllowsPartialMatch), index: windows.length });
+            body = '';
+            bodyAllowsPartialMatch = false;
+        };
+        for (const chunk of chunks) {
+            if (body && body.length + chunk.text.length + 2 > bodyLimit)
+                add();
+            body = body ? `${body}\n\n${chunk.text}` : chunk.text;
+            bodyAllowsPartialMatch = !body.includes('\n\n') && chunk.allowPartialMatch;
+        }
+        if (body || heading)
+            add();
+    }
+    return windows;
+}
+export function selectRelevantContent({ source, format, query, maxLength = 4000 }) {
+    const sections = format === 'html' ? htmlSections(source) : format === 'markdown'
+        ? markdownSections(source) : plainSections(source);
+    const fullText = renderSections(sections);
+    if (maxLength <= 0) {
+        return { text: '', omitted: fullText.length > 0, matched: false };
+    }
+    const terms = queryTerms(query);
+    const windows = makeWindows(sections, terms, maxLength);
+    const ranked = windows.filter((window) => window.score > 0)
+        .sort((a, b) => b.score - a.score || a.index - b.index);
+    if (ranked.length === 0) {
+        return { text: fullText.slice(0, maxLength), omitted: fullText.length > maxLength, matched: false };
+    }
+    const selected = [];
+    let remaining = maxLength;
+    for (const window of ranked) {
+        const separatorLength = selected.length ? 2 : 0;
+        if (window.text.length + separatorLength > remaining)
+            continue;
+        selected.push(window);
+        remaining -= window.text.length + separatorLength;
+    }
+    selected.sort((a, b) => a.index - b.index);
+    const text = selected.map((window) => window.text).join('\n\n');
+    return {
+        text,
+        anchor: ranked.find((window) => selected.includes(window))?.anchor,
+        omitted: selected.length < windows.length,
+        matched: true
+    };
+}
+export function selectRelevantExcerpt(text, query, maxLength) {
+    const terms = queryTerms(query);
+    const chunks = text.split(/\n\s*\n/).flatMap((paragraph) => splitLong(paragraph, maxLength));
+    if (chunks.length === 0)
+        return '';
+    const best = chunks.map((chunk, index) => ({ chunk, index, score: scoreText(chunk, terms) }))
+        .sort((a, b) => b.score - a.score || a.index - b.index)[0];
+    return best.chunk.slice(0, maxLength);
+}

package/dist/fetch/destination-policy.d.ts

@@ -0,0 +1,32 @@
+import { BlockedAddressError, UnverifiedDestinationError, type NetworkGuard } from './network-guard.js';
+export type Destination = {
+    action: 'connect';
+    host: string;
+    address: string;
+} | {
+    action: 'delegate';
+    host: string;
+} | {
+    action: 'refuse';
+    error: BlockedAddressError | UnverifiedDestinationError;
+};
+export type DestinationMode = {
+    /** An upstream proxy is configured. */
+    upstream: boolean;
+    /**
+     * The user explicitly trusts the upstream proxy to enforce private-address
+     * restrictions. Only meaningful with an upstream.
+     */
+    trustProxyDns?: boolean;
+};
+/**
+ * Turns one resolution into where a connection may go (spec revision 2).
+ *
+ * - blocked answers are refused in every mode
+ * - by default the connection goes to an address we checked, never a hostname
+ *   something else will resolve
+ * - with a trusted upstream the hostname is delegated, which also covers names
+ *   only that proxy can resolve
+ * - anything else unresolvable is refused
+ */
+export declare function decideDestination(host: string, guard: NetworkGuard, mode: DestinationMode): Promise<Destination>;

package/dist/fetch/destination-policy.js

@@ -0,0 +1,24 @@
+import { BlockedAddressError, UnverifiedDestinationError } from './network-guard.js';
+/**
+ * Turns one resolution into where a connection may go (spec revision 2).
+ *
+ * - blocked answers are refused in every mode
+ * - by default the connection goes to an address we checked, never a hostname
+ *   something else will resolve
+ * - with a trusted upstream the hostname is delegated, which also covers names
+ *   only that proxy can resolve
+ * - anything else unresolvable is refused
+ */
+export async function decideDestination(host, guard, mode) {
+    const resolution = await guard.resolveHost(host);
+    if (resolution.status === 'blocked') {
+        return { action: 'refuse', error: new BlockedAddressError(resolution.host, resolution.address) };
+    }
+    if (mode.upstream && mode.trustProxyDns) {
+        return { action: 'delegate', host: resolution.host };
+    }
+    if (resolution.status === 'unresolved') {
+        return { action: 'refuse', error: new UnverifiedDestinationError(resolution.host) };
+    }
+    return { action: 'connect', host: resolution.host, address: resolution.addresses[0] };
+}

package/dist/fetch/firecrawl-fetch.d.ts

@@ -5,4 +5,4 @@ export declare function createFirecrawlFetcher({ baseUrl, apiKey, options, fetch
     apiKey?: string;
     options?: FirecrawlOptions;
     fetchImpl?: typeof fetch;
-}): (url: string) => Promise<WebFetchResponse>;
+}): (url: string, query?: string) => Promise<WebFetchResponse>;