@dpopsuev/web-spider 0.10.4

package/dist/disk-cache.js

@@ -0,0 +1,185 @@
+/**
+ * Disk-backed cache implementing ICache<string, SpideredPage>.
+ *
+ * Persists to a JSON file so the cache survives extension reloads and
+ * pi restarts. Call flush() to write — set() auto-flushes by default.
+ *
+ * The images directory is derived automatically from `dirname(path)/images`.
+ * Callers do not need to create it — DiskCache creates it on first large-image
+ * flush. Pre-creating it at startup (e.g. in the extension boot path) is
+ * harmless and avoids a first-write delay.
+ *
+ * Internal storage uses a plain object (Object.create(null)) rather than a
+ * Map. Plain objects carry no realm-specific internal slots, making them safe
+ * across V8 context (realm) boundaries — e.g. when DiskCache is constructed
+ * in an ESM module realm but called from a jiti VM-sandbox realm (Bun binary
+ * mode). The Map-backed version threw "Map operation called on non-Map object"
+ * in that scenario.
+ *
+ * A schema version field in the persisted JSON guards against stale cache
+ * files from previous major versions being silently loaded with wrong shapes.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, extname, join } from "node:path";
+/** Bump when the on-disk entry shape changes incompatibly. */
+const SCHEMA_VERSION = 2;
+export class DiskCache {
+    constructor(path, opts = {}) {
+        this.store = Object.create(null);
+        this.path = path;
+        this.ttlMs = opts.ttlMs ?? 30 * 60 * 1000;
+        this.maxSize = opts.maxSize ?? 500;
+        this.autoFlush = opts.autoFlush ?? true;
+        this.inlineImageThreshold = opts.inlineImageThreshold ?? 32 * 1024;
+        this.imagesDir = join(dirname(path), "images");
+        this.load();
+    }
+    key(url) {
+        try {
+            const u = new URL(url);
+            u.hash = "";
+            return u.toString().replace(/\/$/, "");
+        }
+        catch {
+            return url;
+        }
+    }
+    set(url, page) {
+        const k = this.key(url);
+        if (Object.keys(this.store).length >= this.maxSize && !(k in this.store)) {
+            const oldest = Object.keys(this.store)[0];
+            if (oldest !== undefined)
+                delete this.store[oldest];
+        }
+        this.store[k] = { page, expiresAt: Date.now() + this.ttlMs };
+        if (this.autoFlush)
+            this.flush();
+    }
+    has(url) {
+        return this.get(url) !== undefined;
+    }
+    delete(url) {
+        delete this.store[this.key(url)];
+        if (this.autoFlush)
+            this.flush();
+    }
+    // ---------------------------------------------------------------------------
+    // Image helpers
+    // ---------------------------------------------------------------------------
+    /** Derive a stable filename for an image binary from its src URL. */
+    imageFilename(src) {
+        const hash = createHash("sha1").update(src).digest("hex");
+        const ext = extname(src.split("?")[0]) || ".bin";
+        return `${hash}${ext}`;
+    }
+    /**
+     * Prepare images for serialisation:
+     * - Images whose base64 length ≤ threshold are kept inline.
+     * - Larger images are written to imagesDir as binary files; base64 is
+     *   replaced by filePath in the serialised entry.
+     */
+    spill(images) {
+        if (!existsSync(this.imagesDir)) {
+            mkdirSync(this.imagesDir, { recursive: true });
+        }
+        return images.map((img) => {
+            if (!img.base64 || img.base64.length <= this.inlineImageThreshold) {
+                return img;
+            }
+            const filename = this.imageFilename(img.src);
+            const filePath = join(this.imagesDir, filename);
+            writeFileSync(filePath, Buffer.from(img.base64, "base64"));
+            const { base64: _omit, ...rest } = img;
+            return { ...rest, filePath };
+        });
+    }
+    /**
+     * Hydrate images on read: if an image has filePath but no base64,
+     * load the binary from disk and re-encode.
+     */
+    hydrate(images) {
+        return images.map((img) => {
+            if (img.base64 || !img.filePath)
+                return img;
+            if (!existsSync(img.filePath))
+                return img;
+            try {
+                const base64 = readFileSync(img.filePath).toString("base64");
+                return { ...img, base64 };
+            }
+            catch {
+                return img;
+            }
+        });
+    }
+    // ---------------------------------------------------------------------------
+    // Persistence
+    // ---------------------------------------------------------------------------
+    /** Write current contents to disk. Large images are spilled to imagesDir. */
+    flush() {
+        const now = Date.now();
+        const entries = {};
+        for (const [k, v] of Object.entries(this.store)) {
+            if (!v || v.expiresAt <= now)
+                continue;
+            const page = v.page;
+            const serialised = page.images
+                ? { ...page, images: this.spill(page.images) }
+                : page;
+            entries[k] = { page: serialised, expiresAt: v.expiresAt };
+        }
+        const payload = { v: SCHEMA_VERSION, entries };
+        writeFileSync(this.path, JSON.stringify(payload), "utf8");
+    }
+    load() {
+        if (!existsSync(this.path))
+            return;
+        try {
+            const raw = JSON.parse(readFileSync(this.path, "utf8"));
+            // Reject files from incompatible schema versions (including old
+            // unversioned files that lack the "v" field entirely).
+            if (typeof raw !== "object" ||
+                raw === null ||
+                raw.v !== SCHEMA_VERSION) {
+                return; // stale schema — start fresh, do not throw
+            }
+            const payload = raw;
+            const now = Date.now();
+            for (const [k, v] of Object.entries(payload.entries)) {
+                if (v.expiresAt > now)
+                    this.store[k] = v;
+            }
+        }
+        catch {
+            // Corrupt or unreadable file — start fresh.
+        }
+    }
+    /** All currently valid (non-expired) pages, sorted newest-first. */
+    values() {
+        const now = Date.now();
+        return Object.values(this.store)
+            .filter((e) => e !== undefined && e.expiresAt > now)
+            .sort((a, b) => b.expiresAt - a.expiresAt)
+            .map((e) => {
+            const page = e.page;
+            return page.images ? { ...page, images: this.hydrate(page.images) } : page;
+        });
+    }
+    /** Retrieve a page, hydrating any file-backed images from disk. */
+    get(url) {
+        const k = this.key(url);
+        const entry = this.store[k];
+        if (!entry)
+            return undefined;
+        if (Date.now() > entry.expiresAt) {
+            delete this.store[k];
+            return undefined;
+        }
+        const page = entry.page;
+        if (page.images)
+            return { ...page, images: this.hydrate(page.images) };
+        return page;
+    }
+}
+//# sourceMappingURL=disk-cache.js.map

package/dist/graph.d.ts

@@ -0,0 +1,76 @@
+import type { SpideredPage } from "./types.js";
+/** A node in the knowledge graph — lightweight reference, not the full page. */
+export interface PageNode {
+    url: string;
+    domain: string;
+    title: string;
+    description: string;
+    wordCount: number;
+    fetchedAt: string;
+    chunkCount: number;
+}
+/** A directed edge between two pages. */
+export interface PageEdge {
+    from: string;
+    to: string;
+    /** Anchor text of the link */
+    text: string;
+    isExternal: boolean;
+}
+/** Serialisable snapshot for storage or embedding. */
+export interface PageGraphSnapshot {
+    nodes: PageNode[];
+    edges: PageEdge[];
+}
+/**
+ * Directed knowledge graph of spidered pages.
+ *
+ * Nodes are pages. Edges are outbound links.
+ * Maintains a reverse index (inbound links) for O(1) lookup.
+ *
+ * All graph queries return plain data — no PageNode references —
+ * so the graph is trivially serialisable.
+ *
+ * Internal storage uses plain objects (Object.create(null)) rather than
+ * Maps. Plain objects carry no realm-specific internal slots, making them
+ * safe across V8 context (realm) boundaries — e.g. when the graph is
+ * constructed in an ESM module realm but called from a jiti VM-sandbox.
+ */
+export declare class PageGraph {
+    private readonly nodes;
+    /** url → outbound edges */
+    private readonly out;
+    /** url → inbound source urls */
+    private readonly in_;
+    /** Add or update a node from a spidered page. */
+    addPage(page: SpideredPage): void;
+    /** Add a directed edge without requiring the target to be spidered yet. */
+    addEdge(from: string, to: string, text: string, isExternal: boolean): void;
+    node(url: string): PageNode | undefined;
+    /** Outbound edges from a node. */
+    outbound(url: string): PageEdge[];
+    /** URLs that link TO this page. */
+    inbound(url: string): string[];
+    /** Pages with no inbound links — entry points to the graph. */
+    roots(): PageNode[];
+    /** Pages with no outbound links to other spidered nodes. */
+    sinks(): PageNode[];
+    /** BFS shortest path between two page URLs. Returns null if unreachable. */
+    findPath(from: string, to: string): string[] | null;
+    /**
+     * All pages reachable from `startUrl` via spidered links.
+     * BFS, bounded by the nodes present in the graph.
+     */
+    reachableFrom(startUrl: string): PageNode[];
+    /** Nodes ranked by inbound link count (highest first). */
+    byPageRank(): Array<{
+        node: PageNode;
+        inboundCount: number;
+    }>;
+    get nodeCount(): number;
+    get edgeCount(): number;
+    /** Plain snapshot — safe to JSON.stringify or embed. */
+    toJSON(): PageGraphSnapshot;
+    static fromJSON(snap: PageGraphSnapshot): PageGraph;
+}
+//# sourceMappingURL=graph.d.ts.map

package/dist/graph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph.d.ts","sourceRoot":"","sources":["../src/graph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,OAAO,CAAC;CACpB;AAED,sDAAsD;AACtD,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,KAAK,EAAE,QAAQ,EAAE,CAAC;CAClB;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,SAAS;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6D;IACnF,2BAA2B;IAC3B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA+D;IACnF,gCAAgC;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6D;IAEjF,iDAAiD;IACjD,OAAO,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI;IAiBjC,2EAA2E;IAC3E,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,GAAG,IAAI;IAY1E,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAIvC,kCAAkC;IAClC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,EAAE;IAIjC,mCAAmC;IACnC,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IAI9B,+DAA+D;IAC/D,KAAK,IAAI,QAAQ,EAAE;IAKnB,4DAA4D;IAC5D,KAAK,IAAI,QAAQ,EAAE;IASnB,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI;IAmBnD;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,EAAE;IAgB3C,0DAA0D;IAC1D,UAAU,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,QAAQ,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAO7D,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,SAAS,IAAI,MAAM,CAMtB;IAED,wDAAwD;IACxD,MAAM,IAAI,iBAAiB;IAW3B,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,GAAG,SAAS;CAMnD"}

package/dist/graph.js

@@ -0,0 +1,156 @@
+/**
+ * Directed knowledge graph of spidered pages.
+ *
+ * Nodes are pages. Edges are outbound links.
+ * Maintains a reverse index (inbound links) for O(1) lookup.
+ *
+ * All graph queries return plain data — no PageNode references —
+ * so the graph is trivially serialisable.
+ *
+ * Internal storage uses plain objects (Object.create(null)) rather than
+ * Maps. Plain objects carry no realm-specific internal slots, making them
+ * safe across V8 context (realm) boundaries — e.g. when the graph is
+ * constructed in an ESM module realm but called from a jiti VM-sandbox.
+ */
+export class PageGraph {
+    constructor() {
+        this.nodes = Object.create(null);
+        /** url → outbound edges */
+        this.out = Object.create(null);
+        /** url → inbound source urls */
+        this.in_ = Object.create(null);
+    }
+    /** Add or update a node from a spidered page. */
+    addPage(page) {
+        this.nodes[page.url] = {
+            url: page.url,
+            domain: page.domain,
+            title: page.title,
+            description: page.description,
+            wordCount: page.wordCount,
+            fetchedAt: page.fetchedAt,
+            chunkCount: page.chunks.length,
+        };
+        for (const link of page.links) {
+            if (!link.href)
+                continue;
+            this.addEdge(page.url, link.href, link.text, link.isExternal);
+        }
+    }
+    /** Add a directed edge without requiring the target to be spidered yet. */
+    addEdge(from, to, text, isExternal) {
+        const edge = { from, to, text, isExternal };
+        const existing = this.out[from] ?? [];
+        if (!existing.some((e) => e.to === to)) {
+            this.out[from] = [...existing, edge];
+        }
+        const inbound = this.in_[to] ?? [];
+        if (!inbound.includes(from)) {
+            this.in_[to] = [...inbound, from];
+        }
+    }
+    node(url) {
+        return this.nodes[url];
+    }
+    /** Outbound edges from a node. */
+    outbound(url) {
+        return this.out[url] ?? [];
+    }
+    /** URLs that link TO this page. */
+    inbound(url) {
+        return this.in_[url] ?? [];
+    }
+    /** Pages with no inbound links — entry points to the graph. */
+    roots() {
+        return Object.values(this.nodes)
+            .filter((n) => n !== undefined && (this.in_[n.url] ?? []).length === 0);
+    }
+    /** Pages with no outbound links to other spidered nodes. */
+    sinks() {
+        return Object.values(this.nodes)
+            .filter((n) => {
+            if (!n)
+                return false;
+            const edges = this.out[n.url] ?? [];
+            return !edges.some((e) => e.to in this.nodes);
+        });
+    }
+    /** BFS shortest path between two page URLs. Returns null if unreachable. */
+    findPath(from, to) {
+        if (from === to)
+            return [from];
+        const visited = new Set([from]);
+        const queue = [[from]];
+        while (queue.length > 0) {
+            const path = queue.shift();
+            const current = path[path.length - 1];
+            for (const edge of this.out[current] ?? []) {
+                if (edge.to === to)
+                    return [...path, to];
+                if (!visited.has(edge.to) && edge.to in this.nodes) {
+                    visited.add(edge.to);
+                    queue.push([...path, edge.to]);
+                }
+            }
+        }
+        return null;
+    }
+    /**
+     * All pages reachable from `startUrl` via spidered links.
+     * BFS, bounded by the nodes present in the graph.
+     */
+    reachableFrom(startUrl) {
+        const visited = new Set([startUrl]);
+        const queue = [startUrl];
+        while (queue.length > 0) {
+            const url = queue.shift();
+            for (const edge of this.out[url] ?? []) {
+                if (!visited.has(edge.to) && edge.to in this.nodes) {
+                    visited.add(edge.to);
+                    queue.push(edge.to);
+                }
+            }
+        }
+        visited.delete(startUrl);
+        return [...visited].map((u) => this.nodes[u]).filter((n) => n !== undefined);
+    }
+    /** Nodes ranked by inbound link count (highest first). */
+    byPageRank() {
+        return Object.values(this.nodes)
+            .filter((n) => n !== undefined)
+            .map((n) => ({ node: n, inboundCount: (this.in_[n.url] ?? []).length }))
+            .sort((a, b) => b.inboundCount - a.inboundCount);
+    }
+    get nodeCount() {
+        return Object.keys(this.nodes).length;
+    }
+    get edgeCount() {
+        let total = 0;
+        for (const edges of Object.values(this.out)) {
+            if (edges)
+                total += edges.length;
+        }
+        return total;
+    }
+    /** Plain snapshot — safe to JSON.stringify or embed. */
+    toJSON() {
+        const edges = [];
+        for (const edgeList of Object.values(this.out)) {
+            if (edgeList)
+                edges.push(...edgeList);
+        }
+        return {
+            nodes: Object.values(this.nodes).filter((n) => n !== undefined),
+            edges,
+        };
+    }
+    static fromJSON(snap) {
+        const g = new PageGraph();
+        for (const n of snap.nodes)
+            g.nodes[n.url] = n;
+        for (const e of snap.edges)
+            g.addEdge(e.from, e.to, e.text, e.isExternal);
+        return g;
+    }
+}
+//# sourceMappingURL=graph.js.map

package/dist/index.d.ts

@@ -0,0 +1,45 @@
+export type { SpiderCacheOptions } from "./cache.js";
+export { SpiderCache } from "./cache.js";
+export type { CrawlOptions, CrawlResult } from "./crawl.js";
+export { crawl } from "./crawl.js";
+export type { PageEdge, PageGraphSnapshot, PageNode } from "./graph.js";
+export { PageGraph } from "./graph.js";
+export type { FuzzySearchOptions, SearchHit } from "./search.js";
+export { searchPages } from "./search.js";
+/** @deprecated Use {@link searchPages} — renamed in v0.4.0 to reflect BM25F ranking (not fuzzy-only). */
+export { searchPages as fuzzySearch } from "./search.js";
+export type { SpiderOptions, TreePage } from "./spider.js";
+export { spider } from "./spider.js";
+export type { QueryTreeOptions } from "./tree.js";
+export { buildTree, navigateTree, queryTree } from "./tree.js";
+export type { Chunk, ChunkType, DOMNode, ImageRef, LeanLink, LeanPage, Link, PageView, SpideredPage, TreeHit } from "./types.js";
+export { toLean } from "./views.js";
+export type { BraveSearchOptions, DdgSearchOptions, ExaSearchOptions, FallbackSearchEngineOptions, SearchEngine, TavilySearchOptions, WebSearchResult } from "./web-search.js";
+export { braveSearch, ddgSearch, exaSearch, registerSearchEngine, resolveSearchEngine, tavilySearch, webSearch } from "./web-search.js";
+import type { ICache } from "./ports.js";
+import type { Chunk, SpideredPage } from "./types.js";
+/**
+ * Retrieve a single chunk from a cached page by URL and chunk index.
+ *
+ * Avoids loading the full page markdown when an agent only needs one
+ * specific chunk — e.g. to re-read a section after a highlights hit.
+ *
+ * Returns undefined when the URL is not cached, the index is out of range,
+ * or the index is negative.
+ *
+ * @example
+ * const chunk = getChunk(cache, "https://example.com/article", 3)
+ * if (chunk) console.log(chunk.text)
+ */
+export declare function getChunk(cache: ICache<string, SpideredPage>, url: string, index: number): Chunk | undefined;
+export type { HttpRequest, HttpResponse, ICache, IHttpClient, IRobotsChecker, ISearchEngine, IThrottle, RobotsResult, SearchQuery } from "./ports.js";
+export type { DiskCacheOptions } from "./disk-cache.js";
+export { DiskCache } from "./disk-cache.js";
+export type { PlaywrightClientOptions } from "./playwright.js";
+export { PlaywrightHttpClient, createPlaywrightClient } from "./playwright.js";
+export { RobotsCache, createRobotsCache } from "./robots.js";
+export { fetchSitemapUrls } from "./sitemap.js";
+export type { ThrottleOptions } from "./throttle.js";
+export { DomainThrottle, createThrottle } from "./throttle.js";
+export { BraveSearchEngine, DdgSearchEngine, ExaSearchEngine, FallbackSearchEngine, TavilySearchEngine, defaultSearchEngine } from "./web-search.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,YAAY,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,YAAY,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,YAAY,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,yGAAyG;AACzG,OAAO,EAAE,WAAW,IAAI,WAAW,EAAE,MAAM,aAAa,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC/D,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACjI,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACpC,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,2BAA2B,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAC/K,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAMxI,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEtD;;;;;;;;;;;;GAYG;AACH,wBAAgB,QAAQ,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACX,KAAK,GAAG,SAAS,CAGnB;AAOD,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtJ,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,YAAY,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,eAAe,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/index.js

@@ -0,0 +1,44 @@
+// ---------------------------------------------------------------------------
+// Public API — what most consumers need
+// ---------------------------------------------------------------------------
+export { SpiderCache } from "./cache.js";
+export { crawl } from "./crawl.js";
+export { PageGraph } from "./graph.js";
+export { searchPages } from "./search.js";
+/** @deprecated Use {@link searchPages} — renamed in v0.4.0 to reflect BM25F ranking (not fuzzy-only). */
+export { searchPages as fuzzySearch } from "./search.js";
+export { spider } from "./spider.js";
+export { buildTree, navigateTree, queryTree } from "./tree.js";
+export { toLean } from "./views.js";
+export { braveSearch, ddgSearch, exaSearch, registerSearchEngine, resolveSearchEngine, tavilySearch, webSearch } from "./web-search.js";
+/**
+ * Retrieve a single chunk from a cached page by URL and chunk index.
+ *
+ * Avoids loading the full page markdown when an agent only needs one
+ * specific chunk — e.g. to re-read a section after a highlights hit.
+ *
+ * Returns undefined when the URL is not cached, the index is out of range,
+ * or the index is negative.
+ *
+ * @example
+ * const chunk = getChunk(cache, "https://example.com/article", 3)
+ * if (chunk) console.log(chunk.text)
+ */
+export function getChunk(cache, url, index) {
+    if (index < 0)
+        return undefined;
+    return cache.get(url)?.chunks[index];
+}
+export { DiskCache } from "./disk-cache.js";
+export { PlaywrightHttpClient, createPlaywrightClient } from "./playwright.js";
+export { RobotsCache, createRobotsCache } from "./robots.js";
+export { fetchSitemapUrls } from "./sitemap.js";
+export { DomainThrottle, createThrottle } from "./throttle.js";
+export { BraveSearchEngine, DdgSearchEngine, ExaSearchEngine, FallbackSearchEngine, TavilySearchEngine, defaultSearchEngine } from "./web-search.js";
+// ---------------------------------------------------------------------------
+// parse.ts, convert.ts, views.ts are internal implementation modules.
+// They are NOT exported here — they are consumed only by spider.ts.
+// If you need lower-level DOM or markdown utilities, import from the
+// sub-modules directly (not covered by semver stability guarantees).
+// ---------------------------------------------------------------------------
+//# sourceMappingURL=index.js.map

package/dist/parse.d.ts

@@ -0,0 +1,27 @@
+/**
+ * DOM parsing helpers.
+ *
+ * Owns the DOM parsing dependency. spider.ts calls these after fetching HTML;
+ * it never touches the DOM library directly.
+ */
+import type { Link, SpideredPage } from "./types.js";
+/**
+ * Parse raw HTML into a DOM Document.
+ * Uses linkedom — a lightweight server-side DOM that has no CSS engine,
+ * no module-level Maps, and a flat CJS dependency tree. Safe to load
+ * through jiti's transform pipeline without nativeModules workarounds.
+ */
+export declare function parseDom(html: string, url: string): Document;
+/** True if el or any ancestor up to 5 levels looks like navigation chrome. */
+export declare function isNavElement(el: Element): boolean;
+/** Extract visible text from an anchor, skipping SVG subtrees. */
+export declare function anchorText(a: Element): string;
+/** Extract outbound links from the DOM, classified as body or nav. */
+export declare function extractLinks(doc: Document, baseUrl: string): Link[];
+/** Extract h1/h2/h3 headings from Readability article HTML. */
+export declare function extractHeadings(html: string): SpideredPage["headings"];
+/** Extract topic tags from meta keywords and article:tag. */
+export declare function extractTags(doc: Document): string[];
+/** Extract canonical URL from link[rel=canonical] or og:url. */
+export declare function extractCanonicalUrl(doc: Document, fetchedUrl: string): string | undefined;
+//# sourceMappingURL=parse.d.ts.map

package/dist/parse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse.d.ts","sourceRoot":"","sources":["../src/parse.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAMrD;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,QAAQ,CAE5D;AASD,8EAA8E;AAC9E,wBAAgB,YAAY,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAkBjD;AAMD,kEAAkE;AAClE,wBAAgB,UAAU,CAAC,CAAC,EAAE,OAAO,GAAG,MAAM,CAO7C;AAMD,sEAAsE;AACtE,wBAAgB,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,CAuBnE;AAMD,+DAA+D;AAC/D,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAC,UAAU,CAAC,CAStE;AAMD,6DAA6D;AAC7D,wBAAgB,WAAW,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,CAsBnD;AAMD,gEAAgE;AAChE,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAOzF"}