@taurgis/bonsai 1.0.0

package/dist/sites/salesforce/token.js

@@ -0,0 +1,154 @@
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+// Salesforce Help routes anonymous doc search through a public Coveo org. These are
+// the published defaults for that org; they are not user credentials.
+export const COVEO_DEFAULTS = {
+    organizationId: 'org62salesforce',
+    searchHub: 'HTCommunity',
+    endpointBase: 'https://help.salesforce.com/services/apexrest/coveo',
+    clientUri: 'https://platform.cloud.coveo.com',
+};
+const DEFAULT_TOKEN_TTL_MS = 24 * 60 * 60 * 1000;
+const TOKEN_EXPIRY_SKEW_MS = 2 * 60 * 1000;
+export function isTokenValid(expiresAtMs) {
+    if (!Number.isFinite(expiresAtMs))
+        return false;
+    return Date.now() + TOKEN_EXPIRY_SKEW_MS < expiresAtMs;
+}
+export function decodeJwtPayload(token) {
+    const parts = token.split('.');
+    if (parts.length < 2 || !parts[1])
+        return null;
+    try {
+        return JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
+    }
+    catch {
+        return null;
+    }
+}
+export function getTokenExpiryMs(accessToken) {
+    const payload = decodeJwtPayload(accessToken);
+    const rawExp = payload?.exp ?? payload?.expires_at ?? payload?.expiresAt;
+    const expNumber = Number(rawExp);
+    if (rawExp === undefined || rawExp === null || !Number.isFinite(expNumber))
+        return null;
+    // JWT `exp` is seconds; values past year ~33658 are already milliseconds.
+    return expNumber > 1_000_000_000_000 ? expNumber : expNumber * 1000;
+}
+export function applyTokenExpiry(tokenInfo) {
+    if (!tokenInfo?.accessToken)
+        return tokenInfo;
+    const expiresAtMs = tokenInfo.expiresAtMs ||
+        getTokenExpiryMs(tokenInfo.accessToken) ||
+        Date.now() + DEFAULT_TOKEN_TTL_MS;
+    return { ...tokenInfo, expiresAtMs };
+}
+function stripAuraResponsePrefix(text) {
+    return text
+        .trim()
+        .replace(/^for\(;;\);/, '')
+        .replace(/^\)\]\}',?/, '')
+        .trim();
+}
+/**
+ * Parses the access token out of a Salesforce Aura `getToken` response body.
+ * Returns null when the payload is missing, errored, or lacks a token.
+ */
+export function parseAuraToken(rawResponseBody) {
+    let payload;
+    try {
+        payload = JSON.parse(stripAuraResponsePrefix(rawResponseBody));
+    }
+    catch {
+        return null;
+    }
+    const action = Array.isArray(payload?.actions) ? payload.actions[0] : null;
+    if (!action || (action.state && action.state !== 'SUCCESS'))
+        return null;
+    let returnValue = action.returnValue ?? null;
+    if (typeof returnValue?.returnValue === 'string')
+        returnValue = returnValue.returnValue;
+    const fields = readTokenFields(returnValue);
+    const accessToken = fields.accessToken;
+    if (!accessToken)
+        return null;
+    let filterer = fields.filterer;
+    const jwtFilterer = decodeJwtPayload(accessToken)?.filterer;
+    if (!filterer && typeof jwtFilterer === 'string')
+        filterer = jwtFilterer;
+    return applyTokenExpiry({
+        accessToken,
+        organizationId: fields.organizationId || COVEO_DEFAULTS.organizationId,
+        searchHub: fields.searchHub || COVEO_DEFAULTS.searchHub,
+        endpointBase: fields.endpointBase || COVEO_DEFAULTS.endpointBase,
+        clientUri: fields.clientUri || COVEO_DEFAULTS.clientUri,
+        filterer,
+        expiresAtMs: 0,
+    });
+}
+function readTokenFields(returnValue) {
+    if (typeof returnValue === 'string') {
+        const trimmed = returnValue.trim();
+        if (/^[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+$/.test(trimmed)) {
+            return { accessToken: trimmed, filterer: null };
+        }
+        if (trimmed.startsWith('{')) {
+            try {
+                return readTokenFields(JSON.parse(trimmed));
+            }
+            catch {
+                return { accessToken: null, filterer: null };
+            }
+        }
+        return { accessToken: null, filterer: null };
+    }
+    if (returnValue && typeof returnValue === 'object') {
+        const v = returnValue;
+        return {
+            accessToken: v.accessToken || v.token || v.authToken || v.access_token || null,
+            organizationId: v.organizationId || v.organization || v.sfOrganizationId,
+            searchHub: v.searchHub,
+            endpointBase: v.platformUri || v.clientUri,
+            clientUri: v.clientUri,
+            filterer: v.filterer || v.filter || null,
+        };
+    }
+    return { accessToken: null, filterer: null };
+}
+function tokenCachePath() {
+    const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache');
+    return join(base, 'bonsai', 'salesforce-coveo-token.json');
+}
+export function loadToken() {
+    try {
+        const parsed = JSON.parse(readFileSync(tokenCachePath(), 'utf8'));
+        if (!parsed.accessToken)
+            return null;
+        return applyTokenExpiry({
+            ...COVEO_DEFAULTS,
+            filterer: null,
+            expiresAtMs: 0,
+            ...parsed,
+        });
+    }
+    catch {
+        return null;
+    }
+}
+// ponytail: token persisted as plaintext on disk. It is a public anonymous Coveo
+// search token (no user data, ~24h TTL), so the OS keychain isn't warranted. Move to
+// keytar here if these modules ever hold a privileged token.
+export function storeToken(tokenInfo) {
+    try {
+        const path = tokenCachePath();
+        mkdirSync(dirname(path), { recursive: true });
+        writeFileSync(path, JSON.stringify(applyTokenExpiry(tokenInfo)), {
+            encoding: 'utf8',
+            mode: 0o600,
+        });
+    }
+    catch {
+        // A non-writable cache dir just means we re-mint via the browser next time.
+    }
+}

package/dist/sites/salesforce-developer/fetch-page.d.ts

@@ -0,0 +1,3 @@
+import type { SiteFetchResult } from '../types.js';
+/** Fetches a developer.salesforce.com doc page (guides + API reference) via the shared LWR fetcher. */
+export declare function fetchDeveloperPage(url: string): Promise<SiteFetchResult>;

package/dist/sites/salesforce-developer/fetch-page.js

@@ -0,0 +1,18 @@
+import { fetchSalesforceDoc } from '../salesforce-doc-fetch.js';
+// Containers a developer doc renders into. doc-content-layout / doc-amf-reference are the
+// web-component hosts for guide pages and API-reference (AMF) pages respectively.
+const CONTENT_SELECTORS = [
+    'doc-content-layout',
+    'doc-amf-reference',
+    '.markdown-content',
+    'main article',
+    'article',
+    'main',
+];
+/** Fetches a developer.salesforce.com doc page (guides + API reference) via the shared LWR fetcher. */
+export async function fetchDeveloperPage(url) {
+    return fetchSalesforceDoc(url, {
+        allowedHost: 'developer.salesforce.com',
+        contentSelectors: CONTENT_SELECTORS,
+    });
+}

package/dist/sites/salesforce-developer/index.d.ts

@@ -0,0 +1,2 @@
+import type { SiteModule } from '../types.js';
+export declare const salesforceDeveloper: SiteModule;

package/dist/sites/salesforce-developer/index.js

@@ -0,0 +1,12 @@
+import { fetchDeveloperPage } from './fetch-page.js';
+// developer.salesforce.com docs (guides + API reference) render client-side inside web
+// components, so content lives in shadow DOM — a fundamentally different extraction strategy
+// than help.salesforce.com (see ../salesforce). No `search`: developer docs use a separate
+// search backend, not Help's Coveo HTCommunity hub.
+export const salesforceDeveloper = {
+    id: 'salesforce-developer',
+    name: 'Salesforce Developer',
+    domains: ['developer.salesforce.com'],
+    defaults: { rendered: true },
+    fetchPage: fetchDeveloperPage,
+};

package/dist/sites/salesforce-doc-fetch.d.ts

@@ -0,0 +1,14 @@
+import type { SiteFetchResult } from './types.js';
+/** True when the extracted text is an error/not-found/loading shell rather than real content. */
+export declare function looksLikeSalesforceError(text: string): boolean;
+export declare function stripBoilerplate(markdown: string): string;
+export interface SalesforceDocOptions {
+    allowedHost: string;
+    contentSelectors: string[];
+    removeSelectors?: string[];
+}
+/**
+ * Renders a Salesforce LWR doc page, serializes its shadow-DOM content container, and converts
+ * it to Markdown. Validates the host before any network access.
+ */
+export declare function fetchSalesforceDoc(url: string, { allowedHost, contentSelectors, removeSelectors }: SalesforceDocOptions): Promise<SiteFetchResult>;

package/dist/sites/salesforce-doc-fetch.js

@@ -0,0 +1,321 @@
+import { openCdpPage, waitForLoad, waitForContentReady } from '../lib/research/browser.js';
+import { checkDnsSafety } from '../lib/research/fetcher.js';
+import { htmlToMarkdown } from '../lib/research/markdown.js';
+// Both help.salesforce.com and developer.salesforce.com are LWR sites that render doc content
+// client-side inside web components (shadow DOM). document.documentElement.outerHTML captures
+// only the "Loading…" shell, and Readability prunes the structured result to a single block —
+// so this shared fetcher serializes the shadow DOM and converts the content container directly.
+const TIMEOUT_MS = 45_000;
+const SETTLE_MS = 1_500;
+const MIN_CONTENT_CHARS = 100;
+// Below this a capture is treated as a shell and retried (a fresh reload of the heavy page).
+const SUBSTANTIAL_CHARS = 300;
+const MAX_ATTEMPTS = 2;
+const BODY_LIMIT_BYTES = 6 * 1024 * 1024;
+const ACCEPT_CONSENT_EXPRESSION = "(() => { const b = document.querySelector('#onetrust-accept-btn-handler'); if (b) b.click(); })()";
+const ERROR_PATTERNS = [
+    /page\s*(not\s*found|doesn'?t\s*exist|can'?t\s*be\s*found)/i,
+    /we\s*couldn'?t\s*find/i,
+    /we looked high and low/i,
+    /sorry to interrupt/i,
+];
+/** True when the extracted text is an error/not-found/loading shell rather than real content. */
+export function looksLikeSalesforceError(text) {
+    return ERROR_PATTERNS.some((re) => re.test(text));
+}
+// Chrome present on every LWR doc page; stripped before serialization. Per-site noise (e.g.
+// Help's article-feedback widget) is appended via SalesforceDocOptions.removeSelectors.
+const BASE_REMOVE_SELECTORS = [
+    'nav',
+    'header',
+    'footer',
+    'aside',
+    'script',
+    'style',
+    'link',
+    'noscript',
+    'hgf-c360nav',
+    'hgf-c360contextnav',
+    'dx-scroll-manager',
+    'dx-skip-nav-link',
+    'dw-instrumentation',
+    '[role="navigation"]',
+    '[aria-label*="Table of Contents" i]',
+    '#onetrust-consent-sdk',
+    '#onetrust-banner-sdk',
+];
+function confidenceFor(length) {
+    if (length < 500)
+        return 'low';
+    if (length < 2000)
+        return 'medium';
+    return 'high';
+}
+// The "Did this article solve your issue?" feedback widget renders on every Salesforce doc
+// page. Help wraps it in a removable component (see fetch-page.ts), but Developer renders it
+// as bare divs, so we also strip its lines from the converted Markdown for both sites.
+const BOILERPLATE_LINE = /(did this article solve your issue|let us know so we can improve|share your feedback)/i;
+// A link with empty anchor text — e.g. an icon-only "home" logo link. It carries no readable
+// content and renders as a bare `[](url)`. (Images `![](url)` are intentionally not matched.)
+const EMPTY_LINK_LINE = /^\[\]\([^)]*\)$/;
+export function stripBoilerplate(markdown) {
+    return markdown
+        .split('\n')
+        .filter((line) => !BOILERPLATE_LINE.test(line) && !EMPTY_LINK_LINE.test(line.trim()))
+        .join('\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/**
+ * In-page capture. Expands collapsed "Show" sections, inlines dx-code-block source (its code
+ * lives in a `code-block` attribute, not text), strips chrome, then picks the richest content
+ * container and serializes it INCLUDING open shadow roots. The expand/code steps are no-ops on
+ * Help pages (which lack those components) and load-bearing on Developer API-reference pages.
+ */
+function buildCaptureScript(contentSelectors, extraRemove) {
+    return `(async () => {
+    // Collects every descendant element across open shadow roots AND light DOM, including the
+    // root's own shadow root (so it works both document-wide and scoped to a single component).
+    const deepElements = (root) => {
+      const out = [];
+      const visit = (node) => {
+        if (node.shadowRoot) visit(node.shadowRoot);
+        for (const child of Array.from(node.children || [])) {
+          out.push(child);
+          visit(child);
+        }
+      };
+      visit(root);
+      return out;
+    };
+
+    // Collapse controls vary by page: <anypoint-button class="toggle-button">, <dx-button>, and
+    // older <* class="complex-toggle">. They all render as a button-like element whose entire
+    // label is just "Show"/"Hide", so match on that shape rather than a fixed tag list.
+    const isToggleControl = (el) =>
+      el instanceof HTMLElement &&
+      (/button$/.test(el.tagName.toLowerCase()) || el.classList.contains('complex-toggle'));
+    const toggleLabel = (el) => (el.textContent || '').replace(/\\s+/g, ' ').trim();
+
+    try {
+      const toggles = deepElements(document).filter(
+        (el) => isToggleControl(el) && /^show$/i.test(toggleLabel(el))
+      );
+      for (const t of toggles) { t.click(); await new Promise((r) => setTimeout(r, 80)); }
+      // Once expanded the control flips its label to "Hide"; remove every Show/Hide toggle so the
+      // bare label doesn't leak into the captured Markdown. Generic across all article types.
+      for (const el of deepElements(document)) {
+        if (isToggleControl(el) && /^(show|hide)$/i.test(toggleLabel(el))) {
+          try { el.remove(); } catch (e) {}
+        }
+      }
+    } catch (e) {}
+
+    try {
+      for (const node of deepElements(document)) {
+        if (node.tagName && node.tagName.toLowerCase() === 'dx-code-block') {
+          const code = (node.getAttribute('code-block') || node.textContent || '')
+            .replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n').trim();
+          if (!code) continue;
+          const pre = document.createElement('pre');
+          const codeEl = document.createElement('code');
+          const lang = node.getAttribute('language');
+          if (lang) codeEl.className = 'language-' + lang;
+          codeEl.textContent = code;
+          pre.appendChild(codeEl);
+          node.replaceWith(pre);
+        }
+      }
+    } catch (e) {}
+
+    // AMF API-reference pages render each object's fields as tall stacked blocks
+    // (property-shape-document: name / type / constraints / description). Collapse each
+    // api-type-document's fields into one compact Markdown table instead. Generic across all
+    // API-ref pages; a no-op where these components are absent (e.g. Help articles).
+    const firstIn = (host, sel) => deepElements(host).find((e) => { try { return e.matches && e.matches(sel); } catch (x) { return false; } });
+    const fieldText = (e) => e ? (e.textContent || '').replace(/\\s+/g, ' ').trim() : '';
+    const nearestTypeDoc = (el) => {
+      let n = el.parentElement || (el.getRootNode && el.getRootNode().host) || null;
+      while (n) {
+        if (n.tagName && n.tagName.toLowerCase() === 'api-type-document') return n;
+        n = n.parentElement || (n.getRootNode && n.getRootNode().host) || null;
+      }
+      return null;
+    };
+    try {
+      const groups = new Map();
+      for (const shape of deepElements(document)) {
+        if (!shape.tagName || shape.tagName.toLowerCase() !== 'property-shape-document') continue;
+        const doc = nearestTypeDoc(shape);
+        if (!doc) continue;
+        if (!groups.has(doc)) groups.set(doc, []);
+        groups.get(doc).push(shape);
+      }
+      const buildRow = (cells, tag) => {
+        const tr = document.createElement('tr');
+        for (const value of cells) { const cell = document.createElement(tag); cell.textContent = value; tr.appendChild(cell); }
+        return tr;
+      };
+      for (const shapes of groups.values()) {
+        const rows = [];
+        for (const shape of shapes) {
+          const name = fieldText(firstIn(shape, '.property-title'));
+          if (!name) continue;
+          const type = fieldText(firstIn(shape, '.data-type'));
+          // The field description renders in an <arc-marked> markdown component (api-annotation-document
+          // is empty). The first one in document order is this field's own; constraints (Min/Max/Enum)
+          // come from .property-attribute. Combine them into the Description column.
+          const description = fieldText(firstIn(shape, 'arc-marked'));
+          const attrs = deepElements(shape)
+            .filter((e) => { try { return e.matches && e.matches('.property-attribute'); } catch (x) { return false; } })
+            .map(fieldText);
+          const desc = [description, attrs.join(', ')].filter(Boolean).join(' — ');
+          rows.push([name, type, desc]);
+        }
+        if (!rows.length) continue;
+        const table = document.createElement('table');
+        const thead = document.createElement('thead');
+        thead.appendChild(buildRow(['Field', 'Type', 'Description'], 'th'));
+        const tbody = document.createElement('tbody');
+        for (const row of rows) tbody.appendChild(buildRow(row, 'td'));
+        table.appendChild(thead);
+        table.appendChild(tbody);
+        const first = shapes[0];
+        if (first.parentNode) first.parentNode.insertBefore(table, first);
+        for (const shape of shapes) { try { shape.remove(); } catch (e) {} }
+      }
+    } catch (e) {}
+
+    const REMOVE = ${JSON.stringify([...BASE_REMOVE_SELECTORS, ...extraRemove])};
+    for (const el of deepElements(document)) {
+      try { if (el.matches && REMOVE.some((s) => el.matches(s))) el.remove(); } catch (e) {}
+    }
+
+    // Collapse api-console code-snippet language selectors to just the rendered example: the
+    // dropdown (cURL, HTTP, JavaScript-Fetch, …) is UI chrome, not content. Scoped to snippet
+    // widgets so unrelated toolbars elsewhere are untouched. Generic across all API-ref pages.
+    const inSnippetWidget = (el) => {
+      let n = el;
+      while (n) {
+        const t = n.tagName && n.tagName.toLowerCase();
+        if (t && /(code-snippet|http-snippet|example-render|example-document)/.test(t)) return true;
+        n = n.parentElement || (n.getRootNode && n.getRootNode().host) || null;
+      }
+      return false;
+    };
+    for (const el of deepElements(document)) {
+      const cls = (el.className || '').toString();
+      if (/(^|\\s)(toolbar|dropdown-item|dropdown-menu)(\\s|$)/.test(cls) && inSnippetWidget(el)) {
+        try { el.remove(); } catch (e) {}
+      }
+    }
+
+    const append = (src, target, depth) => {
+      if (depth > 30) return;
+      if (src.nodeType === 3) { target.appendChild(document.createTextNode(src.textContent || '')); return; }
+      if (src.nodeType !== 1) return;
+      const tag = src.tagName.toLowerCase();
+      if (tag === 'style' || tag === 'script' || tag === 'link' || tag === 'noscript' || tag === 'slot' || tag === 'title') return;
+      const clone = src.cloneNode(false);
+      target.appendChild(clone);
+      // Always descend into BOTH the shadow root and light children. API-reference components
+      // (property-shape-document, api-type-document, …) hold details in shadow DOM while also
+      // having light children; skipping either would cut content. <slot> is skipped above so
+      // slotted light content is not duplicated.
+      if (src.shadowRoot) {
+        for (const c of Array.from(src.shadowRoot.childNodes)) append(c, clone, depth + 1);
+      }
+      for (const c of Array.from(src.childNodes)) append(c, clone, depth + 1);
+    };
+    const serialize = (el) => { const root = document.createElement('div'); append(el, root, 0); return root.innerHTML; };
+
+    // Pick the content container by selector PRIORITY, not by max text: the selectors are ordered
+    // specific→general, so the first one with substantial content is the tight article/doc wrapper.
+    // Picking the largest match instead pulls in broader page chrome (nav tree, breadcrumb, chat
+    // widget) that surrounds the article. innerText is used because it pierces shadow roots
+    // (textContent does not), so a real container is measured rather than scoring 0.
+    const SELECTORS = ${JSON.stringify(contentSelectors)};
+    const MIN_PICK_CHARS = 200;
+    let container = null;
+    for (const sel of SELECTORS) {
+      let best = null, bestLen = 0;
+      for (const el of deepElements(document)) {
+        let matched; try { matched = el.matches && el.matches(sel); } catch (e) { matched = false; }
+        if (!matched) continue;
+        const len = (el.innerText || '').trim().length;
+        if (len > bestLen) { best = el; bestLen = len; }
+      }
+      if (best && bestLen >= MIN_PICK_CHARS) { container = best; break; }
+    }
+    container = container || document.body || document.documentElement;
+    const title = (document.querySelector('h1')?.textContent || document.title || '').trim();
+    return { html: serialize(container), title };
+  })()`;
+}
+/**
+ * Renders a Salesforce LWR doc page, serializes its shadow-DOM content container, and converts
+ * it to Markdown. Validates the host before any network access.
+ */
+export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, removeSelectors = [] }) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new Error(`Invalid Salesforce doc URL: ${url}`);
+    }
+    if (parsed.hostname.toLowerCase() !== allowedHost) {
+        throw new Error(`Refusing to fetch host "${parsed.hostname}" (expected ${allowedHost}).`);
+    }
+    await checkDnsSafety(parsed.hostname);
+    const captureScript = buildCaptureScript(contentSelectors, removeSelectors);
+    const page = await openCdpPage();
+    try {
+        let html = '';
+        let title = '';
+        let detailedMarkdown = '';
+        // The heavy api-console renders intermittently slowly; reload and retry if a capture comes
+        // back shell-thin rather than caching an empty page.
+        for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+            await page.client.send('Page.navigate', { url }, page.sessionId);
+            await waitForLoad(page.client, page.sessionId, TIMEOUT_MS, 0).catch(() => { });
+            await page.client
+                .send('Runtime.evaluate', { expression: ACCEPT_CONSENT_EXPRESSION }, page.sessionId)
+                .catch(() => { });
+            await waitForContentReady(page, contentSelectors, MIN_CONTENT_CHARS, TIMEOUT_MS);
+            await new Promise((r) => setTimeout(r, SETTLE_MS));
+            const result = await page.client.send('Runtime.evaluate', { expression: captureScript, awaitPromise: true, returnByValue: true }, page.sessionId);
+            const captured = result?.result?.value;
+            html = typeof captured?.html === 'string' ? captured.html : '';
+            title = captured?.title || '';
+            if (Buffer.byteLength(html) > BODY_LIMIT_BYTES) {
+                throw new Error(`Page exceeded body size limit (${Buffer.byteLength(html)} bytes).`);
+            }
+            detailedMarkdown = stripBoilerplate(htmlToMarkdown(html));
+            if (detailedMarkdown.length >= SUBSTANTIAL_CHARS)
+                break;
+        }
+        if (detailedMarkdown.length < MIN_CONTENT_CHARS || looksLikeSalesforceError(detailedMarkdown)) {
+            throw new Error(`Salesforce returned no readable content for ${url} (page may be missing, gated, or still loading).`);
+        }
+        return {
+            fetchResult: {
+                contentType: 'text/html',
+                etag: null,
+                lastModified: null,
+                finalUrl: url,
+                responseSize: Buffer.byteLength(html),
+                content: html,
+            },
+            extraction: {
+                title: title || url,
+                detailedMarkdown,
+                confidence: confidenceFor(detailedMarkdown.length),
+                qualityNotes: ['extracted from rendered shadow-DOM content container'],
+            },
+        };
+    }
+    finally {
+        await page.close();
+    }
+}

package/dist/sites/types.d.ts

@@ -0,0 +1,27 @@
+import type { ExtractionResult } from '../lib/research/extract.js';
+export interface SiteFetchResult {
+    fetchResult: {
+        contentType: string | null;
+        etag: string | null;
+        lastModified: string | null;
+        finalUrl: string;
+        responseSize: number;
+        content: string;
+    };
+    extraction: ExtractionResult;
+}
+export interface SiteSearchResult {
+    url: string;
+    title: string;
+    snippet?: string;
+}
+export interface SiteModule {
+    id: string;
+    name: string;
+    domains: string[];
+    defaults?: {
+        rendered?: boolean;
+    };
+    fetchPage?: (url: string) => Promise<SiteFetchResult>;
+    search?: (query: string) => Promise<SiteSearchResult[]>;
+}

package/dist/sites/types.js

@@ -0,0 +1 @@
+export {};