@dpopsuev/web-spider 0.10.4

package/dist/search.js

@@ -0,0 +1,112 @@
+import MiniSearch from "minisearch";
+// ---------------------------------------------------------------------------
+// Snippet builder — kept from v1, MiniSearch doesn't generate snippets.
+// ---------------------------------------------------------------------------
+/**
+ * Build a short snippet around the best match position.
+ * Falls back to the start of the text when no match is found.
+ */
+function buildSnippet(text, fullQuery, queryTokens, radius) {
+    const lower = text.toLowerCase();
+    let pos = lower.indexOf(fullQuery);
+    if (pos === -1) {
+        for (const qt of queryTokens) {
+            const p = lower.indexOf(qt);
+            if (p !== -1) {
+                pos = p;
+                break;
+            }
+        }
+    }
+    if (pos === -1)
+        pos = 0;
+    const start = Math.max(0, pos - radius);
+    const end = Math.min(text.length, pos + Math.max(fullQuery.length, queryTokens[0]?.length ?? 1) + radius);
+    const raw = text.slice(start, end).replace(/\s+/g, " ").trim();
+    return (start > 0 ? "…" : "") + raw + (end < text.length ? "…" : "");
+}
+/** Tokenise and lower-case a string — used only for snippet generation. */
+function tokenise(s) {
+    return s
+        .toLowerCase()
+        .split(/[\s\-_.,;:!?()[\]{}"'`/\\]+/)
+        .filter((t) => t.length > 1);
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Full-text search across a set of already-spidered pages using MiniSearch
+ * (BM25F ranking, fuzzy edit-distance, prefix search, heading field boost ×2).
+ *
+ * Searches both body chunks and page metadata (title, description, headings).
+ * Returns results ranked by score descending, normalised to 0–1.
+ *
+ * Designed for agent use: call after fetching pages to locate a specific
+ * fact, term, or section without dumping all content into context.
+ *
+ * @example
+ * const hits = searchPages(pages, "cost optimization selectors", { topN: 5 })
+ * // hits[0].snippet → "…LLM extraction vs Selectors…"
+ */
+export function searchPages(pages, query, opts = {}) {
+    const { topN = 10, snippetRadius = 100 } = opts;
+    if (!query.trim())
+        return [];
+    // Build a flat document list — one entry per chunk, one per metadata field.
+    const docs = [];
+    for (const page of pages) {
+        // Metadata documents
+        const metaDocs = [
+            { id: `${page.url}#meta-title`, heading: "title", text: page.title },
+            ...(page.description
+                ? [{ id: `${page.url}#meta-description`, heading: "description", text: page.description }]
+                : []),
+            ...page.headings.map((h, i) => ({
+                id: `${page.url}#meta-h${i}`,
+                heading: `h${h.level}`,
+                text: h.text,
+            })),
+        ];
+        for (const m of metaDocs) {
+            docs.push({ id: m.id, url: page.url, heading: m.heading, text: m.text, chunkId: "" });
+        }
+        // Chunk documents
+        for (const c of page.chunks) {
+            docs.push({ id: c.id, url: page.url, heading: c.heading, text: c.text, chunkId: c.id });
+        }
+    }
+    if (docs.length === 0)
+        return [];
+    const ms = new MiniSearch({
+        fields: ["text", "heading"],
+        storeFields: ["url", "heading", "chunkId", "text"],
+        searchOptions: {
+            // BM25F: headings are 2× more important than body text.
+            boost: { heading: 2 },
+            // Edit-distance fuzzy — 0.2 × term length, rounded (e.g. ≤1 for 5-char terms).
+            fuzzy: 0.2,
+            // Prefix match: "automat" finds "automation", "automated".
+            prefix: true,
+        },
+    });
+    ms.addAll(docs);
+    const results = ms.search(query);
+    if (results.length === 0)
+        return [];
+    // Normalise raw BM25 scores to 0–1 by dividing by the top score.
+    // This preserves relative ranking while keeping values agent-friendly.
+    const maxRaw = results[0].score;
+    const fullQuery = query.trim().toLowerCase();
+    const queryTokens = tokenise(query);
+    return results.slice(0, topN).map((r) => ({
+        url: String(r["url"]),
+        chunkId: String(r["chunkId"]),
+        heading: String(r["heading"]),
+        score: Math.round(Math.min(r.score / maxRaw, 1) * 100) / 100,
+        snippet: buildSnippet(String(r["text"]), fullQuery, queryTokens, snippetRadius),
+    }));
+}
+/** @deprecated Use {@link searchPages} — renamed in v0.4.0 to reflect BM25F ranking. */
+export const fuzzySearch = searchPages;
+//# sourceMappingURL=search.js.map

package/dist/sitemap.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Sitemap fetcher and parser.
+ *
+ * Attempts /sitemap.xml and /sitemap_index.xml. Extracts <loc> URLs.
+ * Fails open — any error returns an empty array so callers fall back
+ * to normal BFS without noise.
+ */
+import type { IHttpClient } from "./ports.js";
+/**
+ * Fetch and parse sitemap URLs for the given origin.
+ * Supports both standard sitemaps and sitemap index files.
+ * Returns deduplicated absolute URLs, empty array on any failure.
+ */
+export declare function fetchSitemapUrls(origin: string, httpClient: IHttpClient): Promise<string[]>;
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../src/sitemap.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,WAAW,GACrB,OAAO,CAAC,MAAM,EAAE,CAAC,CA4BnB"}

package/dist/sitemap.js

@@ -0,0 +1,65 @@
+/**
+ * Sitemap fetcher and parser.
+ *
+ * Attempts /sitemap.xml and /sitemap_index.xml. Extracts <loc> URLs.
+ * Fails open — any error returns an empty array so callers fall back
+ * to normal BFS without noise.
+ */
+/**
+ * Fetch and parse sitemap URLs for the given origin.
+ * Supports both standard sitemaps and sitemap index files.
+ * Returns deduplicated absolute URLs, empty array on any failure.
+ */
+export async function fetchSitemapUrls(origin, httpClient) {
+    const candidates = [`${origin}/sitemap.xml`, `${origin}/sitemap_index.xml`];
+    const urls = new Set();
+    for (const sitemapUrl of candidates) {
+        try {
+            const res = await httpClient.fetch({
+                url: sitemapUrl,
+                headers: { Accept: "application/xml, text/xml, */*" },
+            });
+            if (!res.ok)
+                continue;
+            const xml = await res.text();
+            for (const loc of extractLocs(xml)) {
+                // Sitemap index entries point to other sitemaps — fetch those too
+                if (loc.endsWith(".xml")) {
+                    const nested = await fetchSitemapXml(loc, httpClient);
+                    for (const u of nested)
+                        urls.add(u);
+                }
+                else {
+                    urls.add(loc);
+                }
+            }
+            if (urls.size > 0)
+                break; // found a working sitemap
+        }
+        catch {
+            continue;
+        }
+    }
+    return [...urls];
+}
+async function fetchSitemapXml(url, httpClient) {
+    try {
+        const res = await httpClient.fetch({ url });
+        if (!res.ok)
+            return [];
+        return extractLocs(await res.text());
+    }
+    catch {
+        return [];
+    }
+}
+function extractLocs(xml) {
+    const urls = [];
+    const re = /<loc>\s*(https?:\/\/[^<\s]+)\s*<\/loc>/gi;
+    let match;
+    while ((match = re.exec(xml)) !== null) {
+        urls.push(match[1].trim());
+    }
+    return urls;
+}
+//# sourceMappingURL=sitemap.js.map

package/dist/spider.d.ts

@@ -0,0 +1,74 @@
+import type { IHttpClient, IRobotsChecker, IThrottle } from "./ports.js";
+import type { DOMNode, LeanPage, SpideredPage } from "./types.js";
+export interface SpiderOptions {
+    /**
+     * ms before aborting the fetch (default 10 000).
+     */
+    timeoutMs?: number;
+    /**
+     * Value sent as User-Agent.
+     * Default identifies the tool; override for sites that block generic crawlers.
+     */
+    userAgent?: string;
+    /**
+     * CSS selector that scopes content extraction to a specific element.
+     * Everything outside the matched element is discarded before Readability runs.
+     * Example: "article", ".main-content", "#post-body"
+     */
+    rootSelector?: string;
+    /**
+     * Comma-separated CSS selectors whose matched elements are removed before
+     * extraction. Applied before Readability, so excluded content never reaches
+     * the chunks or markdown.
+     * Example: "nav, footer, .sidebar, #ads"
+     */
+    excludeSelectors?: string;
+    /**
+     * Approximate maximum token budget for the returned content.
+     * Markdown is truncated to fit. Rough estimate: 1 token ≈ 4 characters.
+     * Does not affect lean view (headings/links are always small).
+     * Default: unlimited.
+     */
+    tokenBudget?: number;
+    /**
+     * Per-domain throttle — shared across spider() calls to enforce rate limits
+     * and exponential backoff on 429/503 responses.
+     */
+    throttle?: IThrottle;
+    /**
+     * robots.txt checker — when provided, spider() checks robots.txt before
+     * fetching and respects Crawl-delay directives.
+     */
+    robotsCache?: IRobotsChecker;
+    /**
+     * HTTP client — defaults to a global fetch() adapter.
+     * Inject a stub for testing without real network access.
+     */
+    httpClient?: IHttpClient;
+    /**
+     * When true, fetch <img> src URLs found in the article content and attach
+     * them as base64-encoded ImageRef objects to SpideredPage.images.
+     * Default: false — preserves current behaviour exactly.
+     */
+    captureImages?: boolean;
+    /**
+     * Maximum number of images to fetch per page.
+     * Default: 10.
+     */
+    maxImages?: number;
+}
+/** A page with its full DOM tree attached. */
+export interface TreePage extends SpideredPage {
+    readonly view: "tree";
+    tree: DOMNode;
+}
+export declare function spider(url: string, opts: SpiderOptions & {
+    view: "lean";
+}): Promise<LeanPage>;
+export declare function spider(url: string, opts: SpiderOptions & {
+    view: "tree";
+}): Promise<TreePage>;
+export declare function spider(url: string, opts?: SpiderOptions & {
+    view?: "full";
+}): Promise<SpideredPage>;
+//# sourceMappingURL=spider.d.ts.map

package/dist/spider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spider.d.ts","sourceRoot":"","sources":["../src/spider.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEzE,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAoClE,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB;;;OAGG;IACH,WAAW,CAAC,EAAE,cAAc,CAAC;IAC7B;;;OAGG;IACH,UAAU,CAAC,EAAE,WAAW,CAAC;IACzB;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AA6GD,8CAA8C;AAC9C,MAAM,WAAW,QAAS,SAAQ,YAAY;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,OAAO,CAAC;CACd;AAED,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACrG,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;AACrG,wBAAsB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC"}

package/dist/spider.js

@@ -0,0 +1,349 @@
+import { Readability } from "@mozilla/readability";
+import { chunk, toMarkdown } from "./convert.js";
+import { extractCanonicalUrl, extractHeadings, extractLinks, extractTags, parseDom } from "./parse.js";
+import { buildTree } from "./tree.js";
+import { toLean } from "./views.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const WORDS_PER_MINUTE = 200;
+// ---------------------------------------------------------------------------
+// Default HTTP client adapter
+// ---------------------------------------------------------------------------
+const defaultHttpClient = {
+    async fetch(req) {
+        const res = await globalThis.fetch(req.url, {
+            signal: req.signal,
+            headers: req.headers,
+        });
+        return {
+            ok: res.ok,
+            status: res.status,
+            statusText: res.statusText,
+            headers: { get: (name) => res.headers.get(name) },
+            text: () => res.text(),
+            arrayBuffer: () => res.arrayBuffer(),
+        };
+    },
+};
+/**
+ * Spider a single URL and return a fully structured SpideredPage.
+ *
+ * Pass `view: "lean"` to skip chunking and markdown conversion — returns a
+ * LeanPage with only identity, metadata, and the heading/link outline.
+ * Significantly faster (~3×) and uses far fewer tokens in agent context.
+ *
+ * Errors are returned as thrown exceptions with a descriptive message rather
+ * than crashing silently. Common cases:
+ * - Non-HTTP URLs throw immediately with a clear message.
+ * - HTTP errors include the status code.
+ * - JS-rendered pages (wordCount === 0) include a hint.
+ * - Timeouts include the configured limit.
+ *
+ * @example
+ * // Full page — chunks, markdown, all metadata
+ * const page = await spider("https://example.com")
+ *
+ * @example
+ * // Lean overview — no body text, ideal for navigation decisions
+ * const lean = await spider("https://example.com", { view: "lean" })
+ */
+// ---------------------------------------------------------------------------
+// Image fetching
+// ---------------------------------------------------------------------------
+/** Detect MIME type from a URL path extension, defaulting to image/jpeg. */
+function mimeFromUrl(src) {
+    const ext = src.split("?")[0].split(".").pop()?.toLowerCase();
+    const map = {
+        jpg: "image/jpeg",
+        jpeg: "image/jpeg",
+        png: "image/png",
+        webp: "image/webp",
+        gif: "image/gif",
+        svg: "image/svg+xml",
+        avif: "image/avif",
+    };
+    return map[ext ?? ""] ?? "image/jpeg";
+}
+/**
+ * Extract <img> elements from article HTML, resolve src URLs, and fetch
+ * each as a base64-encoded ImageRef. data: URLs are included without fetching.
+ * Failed fetches are silently skipped.
+ */
+async function fetchImages(articleHtml, pageUrl, httpClient, maxImages, throttle) {
+    // Parse the article HTML to extract img elements.
+    const { parseDom } = await import("./parse.js");
+    const doc = parseDom(articleHtml, pageUrl);
+    const imgEls = [...doc.querySelectorAll("img")].slice(0, maxImages);
+    const results = [];
+    for (const el of imgEls) {
+        const rawSrc = el.getAttribute("src") ?? "";
+        if (!rawSrc)
+            continue;
+        const alt = el.getAttribute("alt") ?? "";
+        // data: URLs — include without fetching.
+        if (rawSrc.startsWith("data:")) {
+            const match = /^data:([^;]+);base64,(.+)$/.exec(rawSrc);
+            if (match) {
+                results.push({ src: rawSrc, mimeType: match[1], alt, base64: match[2] });
+            }
+            continue;
+        }
+        // Resolve relative URLs.
+        let absoluteSrc;
+        try {
+            absoluteSrc = new URL(rawSrc, pageUrl).toString();
+        }
+        catch {
+            continue;
+        }
+        try {
+            if (throttle)
+                await throttle.wait(absoluteSrc);
+            const res = await httpClient.fetch({
+                url: absoluteSrc,
+                headers: { "User-Agent": "web-spider/0.1", Accept: "image/*" },
+            });
+            if (!res.ok)
+                continue;
+            throttle?.success(absoluteSrc);
+            const buf = await res.arrayBuffer();
+            const base64 = Buffer.from(buf).toString("base64");
+            const contentType = res.headers.get("content-type");
+            const mimeType = contentType?.split(";")[0].trim() || mimeFromUrl(absoluteSrc);
+            results.push({ src: absoluteSrc, mimeType, alt, base64 });
+        }
+        catch {
+            // Skip failed image fetches silently — a missing image should never
+            // cause the whole page scrape to fail.
+        }
+    }
+    return results;
+}
+export async function spider(url, opts) {
+    const { timeoutMs = 30_000, userAgent = "web-spider/0.1 (AI agent research tool; +https://github.com/dpopsuev)", view = "full", rootSelector, excludeSelectors, tokenBudget, throttle, robotsCache, httpClient = defaultHttpClient, captureImages = false, maxImages = 10, } = opts ?? {};
+    // Poka-yoke: reject non-HTTP URLs immediately with a clear message.
+    let parsedUrl;
+    try {
+        parsedUrl = new URL(url);
+    }
+    catch {
+        throw new Error(`Invalid URL: "${url}" — must be a fully-qualified http/https URL`);
+    }
+    if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+        throw new Error(`Unsupported protocol "${parsedUrl.protocol}" — only http and https are supported`);
+    }
+    // Check robots.txt before fetching.
+    if (robotsCache) {
+        const { allowed, crawlDelayMs } = await robotsCache.check(url);
+        if (!allowed)
+            throw new Error(`Blocked by robots.txt: ${url}`);
+        if (crawlDelayMs && throttle) {
+            throttle.setDomainDelay(parsedUrl.hostname, crawlDelayMs);
+        }
+    }
+    // Fetch with optional throttle + retry on 429/503.
+    const maxRetries = throttle?.maxRetries ?? 0;
+    let html = "";
+    let fetchError = null;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        if (throttle)
+            await throttle.wait(url);
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        let res;
+        try {
+            res = await httpClient.fetch({
+                url,
+                signal: controller.signal,
+                headers: { "User-Agent": userAgent, Accept: "text/html" },
+            });
+        }
+        catch (err) {
+            clearTimeout(timer);
+            if (err instanceof Error && err.name === "AbortError") {
+                throw new Error(`Timeout after ${timeoutMs}ms — ${url}`);
+            }
+            throw err;
+        }
+        clearTimeout(timer);
+        if (res.status === 429 || res.status === 503) {
+            if (throttle && attempt < maxRetries) {
+                throttle.rateLimit(url, res.headers.get("Retry-After"));
+                fetchError = new Error(`HTTP ${res.status} — retrying (attempt ${attempt + 1}/${maxRetries})`);
+                continue;
+            }
+            throw new Error(`HTTP ${res.status} ${res.statusText} — ${url}`);
+        }
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status} ${res.statusText} — ${url}`);
+        throttle?.success(url);
+        html = await res.text();
+        fetchError = null;
+        break;
+    }
+    if (fetchError)
+        throw fetchError;
+    // Parse DOM via parse.ts — keeps the JSDOM dependency in one module.
+    const doc = parseDom(html, url);
+    // Apply excludeSelectors before Readability strips the DOM.
+    if (excludeSelectors) {
+        for (const sel of excludeSelectors
+            .split(",")
+            .map((s) => s.trim())
+            .filter(Boolean)) {
+            for (const el of [...doc.querySelectorAll(sel)])
+                el.remove();
+        }
+    }
+    // Scope to rootSelector: replace body content with the matched element.
+    if (rootSelector) {
+        const root = doc.querySelector(rootSelector);
+        if (root) {
+            doc.body.innerHTML = root.outerHTML;
+        }
+    }
+    const links = extractLinks(doc, url);
+    const canonicalUrl = extractCanonicalUrl(doc, url);
+    // Readability content extraction (Firefox Reader View engine).
+    const readabilityResult = new Readability(doc).parse();
+    const jsRendered = !readabilityResult;
+    // Graceful degradation: if Readability finds nothing, return a partial page
+    // with jsRendered:true rather than throwing. The agent can decide what to do.
+    const article = readabilityResult ?? {
+        title: (doc.querySelector("title")?.textContent ?? "").trim(),
+        content: "",
+        textContent: "",
+        length: 0,
+        excerpt: "",
+        byline: "",
+        dir: "",
+        site_name: "",
+        lang: "",
+        publishedTime: null,
+        readingTimeMinutes: 0,
+    };
+    const domain = new URL(url).hostname.replace(/^www\./, "");
+    const fetchedAt = new Date().toISOString();
+    const meta = (name) => {
+        const el = doc.querySelector(`meta[name="${name}"]`) ??
+            doc.querySelector(`meta[property="og:${name}"]`) ??
+            doc.querySelector(`meta[property="${name}"]`);
+        return (el?.getAttribute("content") ?? "").trim();
+    };
+    // headings must come before tags so the heading fallback is available.
+    const headings = extractHeadings(article.content ?? "");
+    const tags = extractTags(doc);
+    // ---------------------------------------------------------------------------
+    // Lean fast-path — skip turndown + chunking entirely
+    // ---------------------------------------------------------------------------
+    if (view === "lean") {
+        const textContent = (article.textContent ?? "").trim();
+        const wordCount = textContent.split(/\s+/).filter(Boolean).length;
+        const chunkCount = Math.max(0, Math.floor(wordCount / 150));
+        const full = {
+            url,
+            domain,
+            fetchedAt,
+            ...(canonicalUrl !== undefined ? { canonicalUrl } : {}),
+            title: article.title ?? meta("title"),
+            description: meta("description"),
+            author: article.byline ?? meta("author"),
+            publishedAt: meta("article:published_time") ?? meta("date"),
+            lang: doc.documentElement.lang ?? "en",
+            tags,
+            wordCount,
+            readingTimeMinutes: Math.ceil(wordCount / WORDS_PER_MINUTE),
+            chunks: [], // placeholder — toLean reads chunks.length
+            headings,
+            links,
+            markdown: "",
+        };
+        const lean = toLean(full);
+        return { ...lean, chunkCount, ...(jsRendered ? { jsRendered: true } : {}) };
+    }
+    // ---------------------------------------------------------------------------
+    // Tree path — build semantic DOM tree, then also produce full markdown
+    // ---------------------------------------------------------------------------
+    if (view === "tree") {
+        const tree = buildTree(article.content ?? "", url);
+        const markdown = toMarkdown(article.content ?? "", { keepImages: captureImages });
+        const wordCount = markdown.split(/\s+/).filter(Boolean).length;
+        const chunks = chunk(markdown, url);
+        const images = captureImages
+            ? await fetchImages(article.content ?? "", url, httpClient, maxImages, throttle)
+            : undefined;
+        return {
+            view: "tree",
+            url,
+            domain,
+            fetchedAt,
+            ...(canonicalUrl !== undefined ? { canonicalUrl } : {}),
+            title: article.title ?? meta("title"),
+            description: meta("description"),
+            author: article.byline ?? meta("author"),
+            publishedAt: meta("article:published_time") ?? meta("date"),
+            lang: doc.documentElement.lang ?? "en",
+            tags,
+            wordCount,
+            readingTimeMinutes: Math.ceil(wordCount / WORDS_PER_MINUTE),
+            headings,
+            chunks,
+            links,
+            markdown,
+            tree,
+            ...(images ? { images } : {}),
+        };
+    }
+    // ---------------------------------------------------------------------------
+    // Full path — turndown + chunk
+    // ---------------------------------------------------------------------------
+    const markdown = toMarkdown(article.content ?? "", { keepImages: captureImages });
+    const wordCount = markdown.split(/\s+/).filter(Boolean).length;
+    // Chunk-aware tokenBudget: select whole chunks up to the budget rather
+    // than slicing markdown mid-sentence. Preserves chunk boundaries and
+    // returns the richest complete content that fits.
+    let allChunks = chunk(markdown, url);
+    if (tokenBudget !== undefined) {
+        const charBudget = tokenBudget * 4;
+        let remaining = charBudget;
+        let first = true;
+        allChunks = allChunks.filter((c) => {
+            // Always include at least the first chunk — agents need something
+            // even if it exceeds the budget.
+            if (!first && remaining <= 0)
+                return false;
+            first = false;
+            remaining -= c.text.length;
+            return true;
+        });
+    }
+    // Reconstruct markdown from selected chunks for full-page consumers.
+    const finalMarkdown = tokenBudget !== undefined
+        ? allChunks.map((c) => c.text).join("\n\n")
+        : markdown;
+    const images = captureImages
+        ? await fetchImages(article.content ?? "", url, httpClient, maxImages, throttle)
+        : undefined;
+    return {
+        url,
+        domain,
+        fetchedAt,
+        ...(canonicalUrl !== undefined ? { canonicalUrl } : {}),
+        title: article.title ?? meta("title"),
+        description: meta("description"),
+        author: article.byline ?? meta("author"),
+        publishedAt: meta("article:published_time") ?? meta("date"),
+        lang: doc.documentElement.lang ?? "en",
+        tags,
+        wordCount,
+        readingTimeMinutes: Math.ceil(wordCount / WORDS_PER_MINUTE),
+        headings,
+        chunks: allChunks,
+        links,
+        markdown: finalMarkdown,
+        ...(images ? { images } : {}),
+        ...(jsRendered ? { jsRendered: true } : {}),
+    };
+}
+//# sourceMappingURL=spider.js.map

package/dist/throttle.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Per-domain request throttle with exponential backoff and jitter.
+ *
+ * Enforces a minimum gap between requests to the same hostname.
+ * On 429/503, backs off exponentially and respects Retry-After headers.
+ * Shared instances should be passed into spider() and crawl() so that
+ * all requests to a domain coordinate through one rate limiter.
+ */
+import type { IThrottle } from "./ports.js";
+export interface ThrottleOptions {
+    /** Minimum gap between requests to the same domain (ms). Default 500. */
+    minDelayMs?: number;
+    /** Base for exponential backoff (ms). Default 1000. */
+    backoffBaseMs?: number;
+    /** Maximum backoff delay (ms). Default 30 000. */
+    backoffCapMs?: number;
+    /** Maximum retry attempts on 429/503 before giving up. Default 3. */
+    maxRetries?: number;
+}
+export declare class DomainThrottle implements IThrottle {
+    private readonly states;
+    readonly minDelayMs: number;
+    readonly backoffBaseMs: number;
+    readonly backoffCapMs: number;
+    readonly maxRetries: number;
+    constructor(opts?: ThrottleOptions);
+    private state;
+    /** Wait until the domain's rate limit and backoff have cleared. */
+    wait(url: string): Promise<void>;
+    /** Record a successful request — resets backoff for the domain. */
+    success(url: string): void;
+    /**
+     * Record a rate-limit hit. Applies exponential backoff with jitter,
+     * using Retry-After header when present. Returns the wait duration in ms.
+     */
+    rateLimit(url: string, retryAfterHeader: string | null): number;
+    /**
+     * Override the minimum delay for a specific domain.
+     * Used to honour robots.txt Crawl-delay directives.
+     */
+    setDomainDelay(host: string, ms: number): void;
+}
+/**
+ * Factory — avoids jiti/Bun CJS re-export interop where class constructors
+ * accessed through a re-export chain can appear undefined at call site.
+ * Use this in extension code instead of `new DomainThrottle()`.
+ */
+export declare function createThrottle(opts?: ThrottleOptions): DomainThrottle;
+//# sourceMappingURL=throttle.d.ts.map

package/dist/throttle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"throttle.d.ts","sourceRoot":"","sources":["../src/throttle.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,WAAW,eAAe;IAC/B,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAuBD,qBAAa,cAAe,YAAW,SAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IACzD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;gBAEhB,IAAI,GAAE,eAAoB;IAOtC,OAAO,CAAC,KAAK;IASb,mEAAmE;IAC7D,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYtC,mEAAmE;IACnE,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAM1B;;;OAGG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM;IAW/D;;;OAGG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI;CAG9C;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,CAAC,EAAE,eAAe,GAAG,cAAc,CAErE"}