@taurgis/bonsai 1.0.5 → 3.0.0

package/dist/sites/coveo.js

@@ -0,0 +1,33 @@
+export const ALLOWED_DOC_HOSTS = new Set(['help.salesforce.com', 'developer.salesforce.com']);
+export function isAllowedDocHost(hostname) {
+    return ALLOWED_DOC_HOSTS.has(hostname.toLowerCase());
+}
+/**
+ * Coveo indexes Help articles under an internal `/help_doccontent` URL; rewrite it to the
+ * canonical `/s/articleView` page a human (or our fetcher) can actually open.
+ */
+export function normalizeHelpDocContentUrl(rawUrl) {
+    let url;
+    try {
+        url = new URL(rawUrl);
+    }
+    catch {
+        return rawUrl;
+    }
+    if (url.hostname.toLowerCase() !== 'help.salesforce.com')
+        return rawUrl;
+    if (!url.pathname.toLowerCase().startsWith('/help_doccontent'))
+        return rawUrl;
+    const id = url.searchParams.get('id');
+    if (!id)
+        return rawUrl;
+    const articleUrl = new URL('https://help.salesforce.com/s/articleView');
+    articleUrl.searchParams.set('id', id.endsWith('.htm') ? id : `${id}.htm`);
+    articleUrl.searchParams.set('type', '5');
+    for (const key of ['release', 'language']) {
+        const value = url.searchParams.get(key);
+        if (value)
+            articleUrl.searchParams.set(key, value);
+    }
+    return articleUrl.toString();
+}

package/dist/sites/index.js

@@ -2,10 +2,9 @@ import { salesforce } from './salesforce/index.js';
 import { salesforceDeveloper } from './salesforce-developer/index.js';
 // The known documentation sites. A plain constant, not a registry: these ship
 // with the CLI, so there is nothing to register at runtime. Sites with custom
-// fetch/search behavior live in their own module (e.g. ./salesforce). Salesforce Help and
+// fetch behavior live in their own module (e.g. ./salesforce). Salesforce Help and
 // Salesforce Developer are separate modules — different domains, different extraction.
 export const SITES = [
-    { id: 'react', name: 'React', domains: ['react.dev', 'legacy.reactjs.org'] },
     salesforce,
     salesforceDeveloper,
     // TanStack docs embed a GitHub "edit this page" link, so capture resolves the page's source

package/dist/sites/salesforce/fetch-page.js

@@ -1,5 +1,5 @@
 import { fetchSalesforceDoc } from '../salesforce-doc-fetch.js';
-import { normalizeHelpDocContentUrl } from './coveo.js';
+import { normalizeHelpDocContentUrl } from '../coveo.js';
 // Containers a Help article renders into (inside shadow DOM); the shared fetcher pierces
 // shadow roots to find them, falling back to the page body when none match.
 const CONTENT_SELECTORS = [

package/dist/sites/salesforce/index.js

@@ -1,16 +1,11 @@
 import { fetchSalesforcePage } from './fetch-page.js';
-import { searchSalesforce } from './browser-search.js';
-// Scoped to help.salesforce.com — an LWR Experience site with light-DOM article content and
-// Coveo (HTCommunity) search. developer.salesforce.com is intentionally NOT routed here: its
-// API-reference docs render inside shadow DOM with tabbed/code content that needs a different
-// extraction strategy, so it belongs in its own module when built — not bolted onto this one.
-// (developer.salesforce.com URLs that appear in Help search results are still surfaced; see
-// ALLOWED_DOC_HOSTS in coveo.ts.)
+// Scoped to help.salesforce.com — an LWR Experience site with light-DOM article content.
+// developer.salesforce.com is intentionally NOT routed here: its API-reference docs render
+// inside shadow DOM with tabbed/code content that needs a different extraction strategy.
 export const salesforce = {
     id: 'salesforce',
     name: 'Salesforce Help',
     domains: ['help.salesforce.com'],
     defaults: { rendered: true },
     fetchPage: fetchSalesforcePage,
-    search: searchSalesforce,
 };

package/dist/sites/salesforce-developer/fetch-page.js

@@ -2,8 +2,10 @@ import { fetchSalesforceDoc } from '../salesforce-doc-fetch.js';
 // Containers a developer doc renders into. doc-content-layout / doc-amf-reference are the
 // web-component hosts for guide pages and API-reference (AMF) pages respectively.
 const CONTENT_SELECTORS = [
-    'doc-content-layout',
+    // API-reference pages: doc-amf-reference holds the method/type tables. doc-content-layout only
+    // carries the shared deprecation banner and must not win selector priority on those pages.
     'doc-amf-reference',
+    'doc-content-layout',
     '.markdown-content',
     'main article',
     'article',

package/dist/sites/salesforce-developer/index.js

@@ -1,8 +1,7 @@
 import { fetchDeveloperPage } from './fetch-page.js';
 // developer.salesforce.com docs (guides + API reference) render client-side inside web
 // components, so content lives in shadow DOM — a fundamentally different extraction strategy
-// than help.salesforce.com (see ../salesforce). No `search`: developer docs use a separate
-// search backend, not Help's Coveo HTCommunity hub.
+// than help.salesforce.com (see ../salesforce).
 export const salesforceDeveloper = {
     id: 'salesforce-developer',
     name: 'Salesforce Developer',

package/dist/sites/salesforce-doc-fetch.d.ts

@@ -2,6 +2,17 @@ import type { SiteFetchResult } from './types.js';
 /** True when the extracted text is an error/not-found/loading shell rather than real content. */
 export declare function looksLikeSalesforceError(text: string): boolean;
 export declare function stripBoilerplate(markdown: string): string;
+/** Parses the Salesforce private content API JSON payload. */
+export declare function parseDocsApiPayload(raw: string): {
+    content: string;
+    title: string;
+} | null;
+/** Keeps whichever Markdown source carries more structural content (DOM vs content API HTML). */
+export declare function preferRicherMarkdown(domMarkdown: string, apiHtml: string, domTitle: string, apiTitle: string): {
+    markdown: string;
+    title: string;
+    usedApi: boolean;
+};
 export interface SalesforceDocOptions {
     allowedHost: string;
     contentSelectors: string[];

package/dist/sites/salesforce-doc-fetch.js

@@ -1,17 +1,22 @@
-import { openCdpPage, waitForLoad, waitForContentReady } from '../lib/research/browser.js';
+import { openCdpPage, waitForLoad, ResponseCapture, } from '../lib/research/browser.js';
 import { checkDnsSafety } from '../lib/research/fetcher.js';
 import { htmlToMarkdown } from '../lib/research/markdown.js';
+import { contentMetrics, contentRicherThan } from '../lib/research/regression.js';
+import { SALESFORCE_SHADOW_DOM_HELPERS, buildPriorityContainerPickBlock, pollSalesforceContentReady, } from './salesforce-dom-probe.js';
 // Both help.salesforce.com and developer.salesforce.com are LWR sites that render doc content
 // client-side inside web components (shadow DOM). document.documentElement.outerHTML captures
 // only the "Loading…" shell, and Readability prunes the structured result to a single block —
 // so this shared fetcher serializes the shadow DOM and converts the content container directly.
 const TIMEOUT_MS = 45_000;
 const SETTLE_MS = 1_500;
-const MIN_CONTENT_CHARS = 100;
+/** Minimum text on the priority content host before wait/capture proceed (shared threshold). */
+const MIN_CONTAINER_CHARS = 100;
 // Below this a capture is treated as a shell and retried (a fresh reload of the heavy page).
 const SUBSTANTIAL_CHARS = 300;
 const MAX_ATTEMPTS = 2;
 const BODY_LIMIT_BYTES = 6 * 1024 * 1024;
+/** Grace period after DOM settle for late /docs/get_document_content/ responses on guide pages. */
+const DOCS_API_GRACE_MS = 500;
 const ACCEPT_CONSENT_EXPRESSION = "(() => { const b = document.querySelector('#onetrust-accept-btn-handler'); if (b) b.click(); })()";
 const ERROR_PATTERNS = [
     /page\s*(not\s*found|doesn'?t\s*exist|can'?t\s*be\s*found)/i,
@@ -66,6 +71,33 @@ export function stripBoilerplate(markdown) {
         .replace(/\n{3,}/g, '\n\n')
         .trim();
 }
+const DOCS_CONTENT_API = '/docs/get_document_content/';
+/** Parses the Salesforce private content API JSON payload. */
+export function parseDocsApiPayload(raw) {
+    if (!raw?.trim())
+        return null;
+    try {
+        const payload = JSON.parse(raw);
+        if (!payload.content)
+            return null;
+        return { content: payload.content, title: payload.title || payload.id || '' };
+    }
+    catch {
+        return null;
+    }
+}
+/** Keeps whichever Markdown source carries more structural content (DOM vs content API HTML). */
+export function preferRicherMarkdown(domMarkdown, apiHtml, domTitle, apiTitle) {
+    const apiMarkdown = stripBoilerplate(htmlToMarkdown(apiHtml));
+    if (contentRicherThan(contentMetrics(apiMarkdown), contentMetrics(domMarkdown))) {
+        return {
+            markdown: apiMarkdown,
+            title: apiTitle || domTitle,
+            usedApi: true,
+        };
+    }
+    return { markdown: domMarkdown, title: domTitle, usedApi: false };
+}
 /**
  * In-page capture. Expands collapsed "Show" sections, inlines dx-code-block source (its code
  * lives in a `code-block` attribute, not text), strips chrome, then picks the richest content
@@ -73,22 +105,9 @@ export function stripBoilerplate(markdown) {
  * Help pages (which lack those components) and load-bearing on Developer API-reference pages.
  */
 function buildCaptureScript(contentSelectors, extraRemove) {
+    const containerPick = buildPriorityContainerPickBlock(contentSelectors, MIN_CONTAINER_CHARS);
     return `(async () => {
-    // Collects every descendant element across open shadow roots AND light DOM, including the
-    // root's own shadow root (so it works both document-wide and scoped to a single component).
-    const deepElements = (root) => {
-      const out = [];
-      const visit = (node) => {
-        if (node.shadowRoot) visit(node.shadowRoot);
-        for (const child of Array.from(node.children || [])) {
-          out.push(child);
-          visit(child);
-        }
-      };
-      visit(root);
-      return out;
-    };
-
+    ${SALESFORCE_SHADOW_DOM_HELPERS}
     // Collapse controls vary by page: <anypoint-button class="toggle-button">, <dx-button>, and
     // older <* class="complex-toggle">. They all render as a button-like element whose entire
     // label is just "Show"/"Hide", so match on that shape rather than a fixed tag list.
@@ -229,34 +248,12 @@ function buildCaptureScript(contentSelectors, extraRemove) {
     };
     const serialize = (el) => { const root = document.createElement('div'); append(el, root, 0); return root.innerHTML; };
 
-    // Pick the content container by selector PRIORITY, not by max text: the selectors are ordered
-    // specific→general, so the first one with substantial content is the tight article/doc wrapper.
-    // Picking the largest match instead pulls in broader page chrome (nav tree, breadcrumb, chat
-    // widget) that surrounds the article. innerText is used because it pierces shadow roots
-    // (textContent does not), so a real container is measured rather than scoring 0.
-    const SELECTORS = ${JSON.stringify(contentSelectors)};
-    const MIN_PICK_CHARS = 200;
-    let container = null;
-    for (const sel of SELECTORS) {
-      let best = null, bestLen = 0;
-      for (const el of deepElements(document)) {
-        let matched; try { matched = el.matches && el.matches(sel); } catch (e) { matched = false; }
-        if (!matched) continue;
-        const len = (el.innerText || '').trim().length;
-        if (len > bestLen) { best = el; bestLen = len; }
-      }
-      if (best && bestLen >= MIN_PICK_CHARS) { container = best; break; }
-    }
-    container = container || document.body || document.documentElement;
+    ${containerPick}
     const title = (document.querySelector('h1')?.textContent || document.title || '').trim();
     return { html: serialize(container), title };
   })()`;
 }
-/**
- * Renders a Salesforce LWR doc page, serializes its shadow-DOM content container, and converts
- * it to Markdown. Validates the host before any network access.
- */
-export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, removeSelectors = [] }) {
+function parseSalesforceDocUrl(url, allowedHost) {
     let parsed;
     try {
         parsed = new URL(url);
@@ -267,6 +264,74 @@ export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, r
     if (parsed.hostname.toLowerCase() !== allowedHost) {
         throw new Error(`Refusing to fetch host "${parsed.hostname}" (expected ${allowedHost}).`);
     }
+    return parsed;
+}
+function assertReadableSalesforceContent(detailedMarkdown, url) {
+    if (detailedMarkdown.length < MIN_CONTAINER_CHARS || looksLikeSalesforceError(detailedMarkdown)) {
+        throw new Error(`Salesforce returned no readable content for ${url} (page may be missing, gated, or still loading).`);
+    }
+}
+async function captureSalesforceAttempt(page, url, captureScript, contentSelectors) {
+    const docsCapture = new ResponseCapture(page, [
+        {
+            key: 'docs-content',
+            test: (request) => request.url.includes(DOCS_CONTENT_API),
+            accept: (body) => parseDocsApiPayload(body) !== null,
+        },
+    ]);
+    await page.client.send('Page.navigate', { url }, page.sessionId);
+    await waitForLoad(page.client, page.sessionId, TIMEOUT_MS, 0).catch(() => { });
+    await page.client
+        .send('Runtime.evaluate', { expression: ACCEPT_CONSENT_EXPRESSION }, page.sessionId)
+        .catch(() => { });
+    await pollSalesforceContentReady(page, contentSelectors, MIN_CONTAINER_CHARS, TIMEOUT_MS);
+    await new Promise((r) => setTimeout(r, SETTLE_MS));
+    const docsPayload = parseDocsApiPayload((await docsCapture.waitFor('docs-content', DOCS_API_GRACE_MS)) ?? '');
+    const docsApiHtml = docsPayload?.content ?? '';
+    const docsApiTitle = docsPayload?.title ?? '';
+    const result = await page.client.send('Runtime.evaluate', { expression: captureScript, awaitPromise: true, returnByValue: true }, page.sessionId);
+    const captured = result?.result?.value;
+    const html = typeof captured?.html === 'string' ? captured.html : '';
+    const title = captured?.title || '';
+    if (Buffer.byteLength(html) > BODY_LIMIT_BYTES) {
+        throw new Error(`Page exceeded body size limit (${Buffer.byteLength(html)} bytes).`);
+    }
+    let detailedMarkdown = stripBoilerplate(htmlToMarkdown(html));
+    let usedDocsApi = false;
+    if (docsApiHtml) {
+        const picked = preferRicherMarkdown(detailedMarkdown, docsApiHtml, title, docsApiTitle);
+        detailedMarkdown = picked.markdown;
+        if (picked.usedApi) {
+            usedDocsApi = true;
+        }
+        return { html, title: picked.title, detailedMarkdown, usedDocsApi };
+    }
+    return { html, title, detailedMarkdown, usedDocsApi };
+}
+function buildSalesforceFetchResult(url, html, title, detailedMarkdown, qualityNotes) {
+    return {
+        fetchResult: {
+            contentType: 'text/html',
+            etag: null,
+            lastModified: null,
+            finalUrl: url,
+            responseSize: Buffer.byteLength(html),
+            content: html,
+        },
+        extraction: {
+            title: title || url,
+            detailedMarkdown,
+            confidence: confidenceFor(detailedMarkdown.length),
+            qualityNotes,
+        },
+    };
+}
+/**
+ * Renders a Salesforce LWR doc page, serializes its shadow-DOM content container, and converts
+ * it to Markdown. Validates the host before any network access.
+ */
+export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, removeSelectors = [] }) {
+    const parsed = parseSalesforceDocUrl(url, allowedHost);
     await checkDnsSafety(parsed.hostname);
     const captureScript = buildCaptureScript(contentSelectors, removeSelectors);
     const page = await openCdpPage();
@@ -274,46 +339,22 @@ export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, r
         let html = '';
         let title = '';
         let detailedMarkdown = '';
-        // The heavy api-console renders intermittently slowly; reload and retry if a capture comes
-        // back shell-thin rather than caching an empty page.
+        let usedDocsApi = false;
+        const qualityNotes = ['extracted from rendered shadow-DOM content container'];
         for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
-            await page.client.send('Page.navigate', { url }, page.sessionId);
-            await waitForLoad(page.client, page.sessionId, TIMEOUT_MS, 0).catch(() => { });
-            await page.client
-                .send('Runtime.evaluate', { expression: ACCEPT_CONSENT_EXPRESSION }, page.sessionId)
-                .catch(() => { });
-            await waitForContentReady(page, contentSelectors, MIN_CONTENT_CHARS, TIMEOUT_MS);
-            await new Promise((r) => setTimeout(r, SETTLE_MS));
-            const result = await page.client.send('Runtime.evaluate', { expression: captureScript, awaitPromise: true, returnByValue: true }, page.sessionId);
-            const captured = result?.result?.value;
-            html = typeof captured?.html === 'string' ? captured.html : '';
-            title = captured?.title || '';
-            if (Buffer.byteLength(html) > BODY_LIMIT_BYTES) {
-                throw new Error(`Page exceeded body size limit (${Buffer.byteLength(html)} bytes).`);
-            }
-            detailedMarkdown = stripBoilerplate(htmlToMarkdown(html));
+            const captured = await captureSalesforceAttempt(page, url, captureScript, contentSelectors);
+            html = captured.html;
+            title = captured.title;
+            detailedMarkdown = captured.detailedMarkdown;
+            usedDocsApi = usedDocsApi || captured.usedDocsApi;
             if (detailedMarkdown.length >= SUBSTANTIAL_CHARS)
                 break;
         }
-        if (detailedMarkdown.length < MIN_CONTENT_CHARS || looksLikeSalesforceError(detailedMarkdown)) {
-            throw new Error(`Salesforce returned no readable content for ${url} (page may be missing, gated, or still loading).`);
+        assertReadableSalesforceContent(detailedMarkdown, url);
+        if (usedDocsApi) {
+            qualityNotes.push('article body from /docs/get_document_content/ API');
         }
-        return {
-            fetchResult: {
-                contentType: 'text/html',
-                etag: null,
-                lastModified: null,
-                finalUrl: url,
-                responseSize: Buffer.byteLength(html),
-                content: html,
-            },
-            extraction: {
-                title: title || url,
-                detailedMarkdown,
-                confidence: confidenceFor(detailedMarkdown.length),
-                qualityNotes: ['extracted from rendered shadow-DOM content container'],
-            },
-        };
+        return buildSalesforceFetchResult(url, html, title, detailedMarkdown, qualityNotes);
     }
     finally {
         await page.close();

package/dist/sites/salesforce-dom-probe.d.ts

@@ -0,0 +1,16 @@
+import type { CdpPage } from '../lib/research/browser.js';
+/** Shared in-page helpers for traversing Salesforce LWR open shadow roots. */
+export declare const SALESFORCE_SHADOW_DOM_HELPERS = "\nconst deepElements = (root) => {\n  const out = [];\n  const visit = (node) => {\n    if (node.shadowRoot) visit(node.shadowRoot);\n    for (const child of Array.from(node.children || [])) {\n      out.push(child);\n      visit(child);\n    }\n  };\n  visit(root);\n  return out;\n};\n\nconst deepText = (root) => {\n  let txt = '';\n  const visit = (node) => {\n    if (node.nodeType === 3) {\n      txt += node.textContent || '';\n    } else if (node.nodeType === 1 || node.nodeType === 9 || node.nodeType === 11) {\n      const tag = node.tagName ? node.tagName.toLowerCase() : '';\n      if (tag !== 'style' && tag !== 'script') {\n        if (node.shadowRoot) visit(node.shadowRoot);\n        for (const child of Array.from(node.childNodes || [])) visit(child);\n      }\n    }\n  };\n  visit(root);\n  return txt;\n};\n\nconst textLenFor = (el) =>\n  Math.max((el.innerText || '').trim().length, deepText(el).trim().length);\n";
+/**
+ * In-page block: pick the highest-priority content container that exists on the page.
+ * Does not fall through to lower-priority selectors when a higher-priority host is present
+ * (e.g. doc-content-layout banner vs doc-amf-reference table).
+ */
+export declare function buildPriorityContainerPickBlock(selectors: string[], minChars: number, containerVar?: string): string;
+/** CDP expression polled until the priority content host reaches minChars and the body settles. */
+export declare function buildContentReadyProbeExpression(selectors: string[], minChars: number): string;
+/**
+ * Polls until the priority Salesforce content host is ready and body text has settled.
+ * Returns without throwing on timeout; the caller captures whatever rendered and judges quality.
+ */
+export declare function pollSalesforceContentReady(page: CdpPage, selectors: string[], minChars: number, timeoutMs: number): Promise<void>;

package/dist/sites/salesforce-dom-probe.js

@@ -0,0 +1,114 @@
+/** Shared in-page helpers for traversing Salesforce LWR open shadow roots. */
+export const SALESFORCE_SHADOW_DOM_HELPERS = `
+const deepElements = (root) => {
+  const out = [];
+  const visit = (node) => {
+    if (node.shadowRoot) visit(node.shadowRoot);
+    for (const child of Array.from(node.children || [])) {
+      out.push(child);
+      visit(child);
+    }
+  };
+  visit(root);
+  return out;
+};
+
+const deepText = (root) => {
+  let txt = '';
+  const visit = (node) => {
+    if (node.nodeType === 3) {
+      txt += node.textContent || '';
+    } else if (node.nodeType === 1 || node.nodeType === 9 || node.nodeType === 11) {
+      const tag = node.tagName ? node.tagName.toLowerCase() : '';
+      if (tag !== 'style' && tag !== 'script') {
+        if (node.shadowRoot) visit(node.shadowRoot);
+        for (const child of Array.from(node.childNodes || [])) visit(child);
+      }
+    }
+  };
+  visit(root);
+  return txt;
+};
+
+const textLenFor = (el) =>
+  Math.max((el.innerText || '').trim().length, deepText(el).trim().length);
+`;
+/**
+ * In-page block: pick the highest-priority content container that exists on the page.
+ * Does not fall through to lower-priority selectors when a higher-priority host is present
+ * (e.g. doc-content-layout banner vs doc-amf-reference table).
+ */
+export function buildPriorityContainerPickBlock(selectors, minChars, containerVar = 'container') {
+    return `
+    let ${containerVar} = null;
+    const SELECTORS = ${JSON.stringify(selectors)};
+    const MIN_CHARS = ${minChars};
+    for (const sel of SELECTORS) {
+      let best = null, bestLen = 0, anyMatch = false;
+      for (const el of deepElements(document)) {
+        let matched;
+        try { matched = el.matches && el.matches(sel); } catch (e) { matched = false; }
+        if (!matched) continue;
+        anyMatch = true;
+        const len = textLenFor(el);
+        if (len > bestLen) { best = el; bestLen = len; }
+      }
+      if (anyMatch) {
+        if (best && bestLen >= MIN_CHARS) ${containerVar} = best;
+        break;
+      }
+    }
+    ${containerVar} = ${containerVar} || document.body || document.documentElement;
+  `;
+}
+/** CDP expression polled until the priority content host reaches minChars and the body settles. */
+export function buildContentReadyProbeExpression(selectors, minChars) {
+    return `(() => {
+    ${SALESFORCE_SHADOW_DOM_HELPERS}
+    const SELS = ${JSON.stringify(selectors)};
+    const MIN = ${minChars};
+    let has = false;
+    for (const sel of SELS) {
+      let bestLen = 0, anyMatch = false;
+      for (const el of deepElements(document)) {
+        let matched;
+        try { matched = el.matches && el.matches(sel); } catch (e) { matched = false; }
+        if (!matched) continue;
+        anyMatch = true;
+        const len = textLenFor(el);
+        if (len > bestLen) bestLen = len;
+      }
+      if (anyMatch) {
+        has = bestLen >= MIN;
+        break;
+      }
+    }
+    const len = document.body ? ((document.body.innerText) || '').length : 0;
+    return { has, len };
+  })()`;
+}
+/**
+ * Polls until the priority Salesforce content host is ready and body text has settled.
+ * Returns without throwing on timeout; the caller captures whatever rendered and judges quality.
+ */
+export async function pollSalesforceContentReady(page, selectors, minChars, timeoutMs) {
+    const expression = buildContentReadyProbeExpression(selectors, minChars);
+    const start = Date.now();
+    let previousLength = -1;
+    let stablePolls = 0;
+    while (Date.now() - start < timeoutMs) {
+        const rawResult = await page.client
+            .send('Runtime.evaluate', { expression, returnByValue: true }, page.sessionId)
+            .catch(() => null);
+        const value = rawResult?.result?.value;
+        if (value?.has) {
+            const grew = Math.abs((value.len || 0) - previousLength) > Math.max(40, previousLength * 0.02);
+            if (!grew && ++stablePolls >= 2)
+                return;
+            if (grew)
+                stablePolls = 0;
+            previousLength = value.len || 0;
+        }
+        await new Promise((r) => setTimeout(r, 500));
+    }
+}

package/dist/sites/types.d.ts

@@ -10,11 +10,6 @@ export interface SiteFetchResult {
     };
     extraction: ExtractionResult;
 }
-export interface SiteSearchResult {
-    url: string;
-    title: string;
-    snippet?: string;
-}
 export interface SiteModule {
     id: string;
     name: string;
@@ -23,5 +18,4 @@ export interface SiteModule {
         rendered?: boolean;
     };
     fetchPage?: (url: string) => Promise<SiteFetchResult>;
-    search?: (query: string) => Promise<SiteSearchResult[]>;
 }