npm - @dpopsuev/web-spider - Versions diffs - 0.10.4 - Mend

@dpopsuev/web-spider 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (105) hide show

package/dist/batch.d.ts +24 -0
package/dist/batch.d.ts.map +1 -0
package/dist/batch.js +68 -0
package/dist/cache.d.ts +40 -0
package/dist/cache.d.ts.map +1 -0
package/dist/cache.js +78 -0
package/dist/convert.d.ts +29 -0
package/dist/convert.d.ts.map +1 -0
package/dist/convert.js +131 -0
package/dist/crawl.d.ts +56 -0
package/dist/crawl.d.ts.map +1 -0
package/dist/crawl.js +126 -0
package/dist/disk-cache.d.ts +75 -0
package/dist/disk-cache.d.ts.map +1 -0
package/dist/disk-cache.js +185 -0
package/dist/graph.d.ts +76 -0
package/dist/graph.d.ts.map +1 -0
package/dist/graph.js +156 -0
package/dist/index.d.ts +45 -0
package/dist/index.d.ts.map +1 -0
package/dist/index.js +44 -0
package/dist/parse.d.ts +27 -0
package/dist/parse.d.ts.map +1 -0
package/dist/parse.js +131 -0
package/dist/playwright.d.ts +75 -0
package/dist/playwright.d.ts.map +1 -0
package/dist/playwright.js +141 -0
package/dist/ports.d.ts +104 -0
package/dist/ports.d.ts.map +1 -0
package/dist/ports.js +10 -0
package/dist/robots.d.ts +24 -0
package/dist/robots.d.ts.map +1 -0
package/dist/robots.js +104 -0
package/dist/search.d.ts +47 -0
package/dist/search.d.ts.map +1 -0
package/dist/search.js +112 -0
package/dist/sitemap.d.ts +15 -0
package/dist/sitemap.d.ts.map +1 -0
package/dist/sitemap.js +65 -0
package/dist/spider.d.ts +74 -0
package/dist/spider.d.ts.map +1 -0
package/dist/spider.js +349 -0
package/dist/throttle.d.ts +49 -0
package/dist/throttle.d.ts.map +1 -0
package/dist/throttle.js +85 -0
package/dist/tree.d.ts +34 -0
package/dist/tree.d.ts.map +1 -0
package/dist/tree.js +354 -0
package/dist/types.d.ts +189 -0
package/dist/types.d.ts.map +1 -0
package/dist/types.js +2 -0
package/dist/views.d.ts +17 -0
package/dist/views.d.ts.map +1 -0
package/dist/views.js +39 -0
package/dist/web-search.d.ts +184 -0
package/dist/web-search.d.ts.map +1 -0
package/dist/web-search.js +399 -0
package/fixtures/article-with-images.html +94 -0
package/fixtures/gh-shell.html +32 -0
package/fixtures/guide-ai-agents-web-scraping.json +552 -0
package/fixtures/images/large.jpg +0 -0
package/fixtures/images/small.jpg +0 -0
package/fixtures/images/tiny.png +0 -0
package/fixtures/quotes-index.json +40 -0
package/package.json +47 -0
package/scripts/fetch-guide.mjs +25 -0
package/src/cache.ts +99 -0
package/src/convert.ts +161 -0
package/src/crawl.ts +186 -0
package/src/disk-cache.ts +228 -0
package/src/graph.ts +189 -0
package/src/index.ts +74 -0
package/src/parse.ts +154 -0
package/src/playwright.ts +193 -0
package/src/ports.ts +131 -0
package/src/robots.ts +121 -0
package/src/search.ts +173 -0
package/src/sitemap.ts +67 -0
package/src/spider.ts +475 -0
package/src/throttle.ts +118 -0
package/src/tree.ts +379 -0
package/src/types.ts +225 -0
package/src/views.ts +42 -0
package/src/web-search.ts +548 -0
package/test/convert-images.test.ts +69 -0
package/test/disk-cache-images.test.ts +193 -0
package/test/engine-registry.test.ts +114 -0
package/test/exports.test.ts +124 -0
package/test/get-chunk.test.ts +115 -0
package/test/images-integration.test.ts +359 -0
package/test/improvements.test.ts +279 -0
package/test/inbound-count.test.ts +111 -0
package/test/lean.test.ts +105 -0
package/test/playwright.test.ts +128 -0
package/test/ports.test.ts +161 -0
package/test/search.test.ts +219 -0
package/test/spider-images.test.ts +180 -0
package/test/spider-unit.test.ts +610 -0
package/test/tree.test.ts +272 -0
package/test/types.test.ts +169 -0
package/test/web-search-integration.test.ts +180 -0
package/test/web-search.test.ts +305 -0
package/tsconfig.json +9 -0
package/tsconfig.test.json +7 -0
package/vitest.config.ts +8 -0

package/dist/parse.js ADDED Viewed

@@ -0,0 +1,131 @@
+/**
+ * DOM parsing helpers.
+ *
+ * Owns the DOM parsing dependency. spider.ts calls these after fetching HTML;
+ * it never touches the DOM library directly.
+ */
+import { parseHTML } from "linkedom";
+// ---------------------------------------------------------------------------
+// DOM creation
+// ---------------------------------------------------------------------------
+/**
+ * Parse raw HTML into a DOM Document.
+ * Uses linkedom — a lightweight server-side DOM that has no CSS engine,
+ * no module-level Maps, and a flat CJS dependency tree. Safe to load
+ * through jiti's transform pipeline without nativeModules workarounds.
+ */
+export function parseDom(html, url) {
+    return parseHTML(html, { url }).document;
+}
+// ---------------------------------------------------------------------------
+// Nav classification
+// ---------------------------------------------------------------------------
+const NAV_CLASS_RE = /^(nav|navbar|navigation|menu|menubar|header|footer|sidebar|breadcrumb|topbar|toolbar|site-nav|main-nav|primary-nav|global-nav)$/i;
+/** True if el or any ancestor up to 5 levels looks like navigation chrome. */
+export function isNavElement(el) {
+    if (el.closest("nav, header, footer, aside"))
+        return true;
+    if (el.closest("[role='navigation'],[role='banner'],[role='contentinfo'],[role='complementary']"))
+        return true;
+    let node = el;
+    for (let i = 0; i < 5; i++) {
+        if (!node)
+            break;
+        for (const cls of node.classList) {
+            if (NAV_CLASS_RE.test(cls))
+                return true;
+        }
+        node = node.parentElement;
+    }
+    return false;
+}
+// ---------------------------------------------------------------------------
+// Link text extraction
+// ---------------------------------------------------------------------------
+/** Extract visible text from an anchor, skipping SVG subtrees. */
+export function anchorText(a) {
+    if (!a.querySelector("svg")) {
+        return (a.textContent ?? "").replace(/\s+/g, " ").trim();
+    }
+    const clone = a.cloneNode(true);
+    for (const svg of [...clone.querySelectorAll("svg")])
+        svg.remove();
+    return (clone.textContent ?? "").replace(/\s+/g, " ").trim();
+}
+// ---------------------------------------------------------------------------
+// Link extraction
+// ---------------------------------------------------------------------------
+/** Extract outbound links from the DOM, classified as body or nav. */
+export function extractLinks(doc, baseUrl) {
+    const origin = new URL(baseUrl).origin;
+    return Array.from(doc.querySelectorAll("a[href]"))
+        .map((a) => {
+        const href = a.href;
+        const text = anchorText(a)
+            .replace(/\b(open_in_new|navigate_next|navigate_before|arrow_drop_down|arrow_drop_up|chevron_right|chevron_left|expand_more|expand_less)\b/g, "")
+            .replace(/\s+/g, " ")
+            .trim();
+        if (!href || !text || href.startsWith("javascript:"))
+            return null;
+        return {
+            href,
+            text,
+            isExternal: !href.startsWith(origin),
+            rel: isNavElement(a) ? "nav" : "body",
+        };
+    })
+        .filter((l) => l !== null)
+        .slice(0, 200);
+}
+// ---------------------------------------------------------------------------
+// Heading extraction
+// ---------------------------------------------------------------------------
+/** Extract h1/h2/h3 headings from Readability article HTML. */
+export function extractHeadings(html) {
+    const { document } = parseHTML(`<html><body>${html}</body></html>`);
+    const headings = [];
+    document.querySelectorAll("h1, h2, h3").forEach((el) => {
+        const level = parseInt(el.tagName[1], 10);
+        const text = (el.textContent ?? "").trim();
+        if (text)
+            headings.push({ level, text });
+    });
+    return headings;
+}
+// ---------------------------------------------------------------------------
+// Tag extraction
+// ---------------------------------------------------------------------------
+/** Extract topic tags from meta keywords and article:tag. */
+export function extractTags(doc) {
+    const tags = new Set();
+    const keywords = doc.querySelector('meta[name="keywords"]')?.getAttribute("content") ?? "";
+    for (const k of keywords
+        .split(/[,;]/)
+        .map((k) => k.trim().toLowerCase())
+        .filter(Boolean)) {
+        tags.add(k);
+    }
+    doc.querySelectorAll('meta[property="article:tag"], meta[name="article:tag"]').forEach((el) => {
+        const t = el.getAttribute("content")?.trim().toLowerCase();
+        if (t)
+            tags.add(t);
+    });
+    const section = doc.querySelector('meta[property="article:section"]')?.getAttribute("content") ??
+        doc.querySelector('meta[property="og:article:section"]')?.getAttribute("content");
+    if (section)
+        tags.add(section.trim().toLowerCase());
+    return [...tags].slice(0, 20);
+}
+// ---------------------------------------------------------------------------
+// Canonical URL extraction
+// ---------------------------------------------------------------------------
+/** Extract canonical URL from link[rel=canonical] or og:url. */
+export function extractCanonicalUrl(doc, fetchedUrl) {
+    const canonical = doc.querySelector('link[rel="canonical"]')?.getAttribute("href") ??
+        doc.querySelector('meta[property="og:url"]')?.getAttribute("content");
+    if (!canonical)
+        return undefined;
+    const norm = (u) => u.replace(/\/$/, "");
+    return norm(canonical) !== norm(fetchedUrl) ? canonical : undefined;
+}
+//# sourceMappingURL=parse.js.map

package/dist/playwright.d.ts ADDED Viewed

@@ -0,0 +1,75 @@
+/**
+ * Playwright adapter — implements IHttpClient using a headless browser.
+ *
+ * Uses playwright-extra with the stealth plugin, which patches ~15 headless
+ * fingerprint signals (navigator.webdriver, User-Agent, plugins, WebGL, etc.)
+ * so the browser is indistinguishable from a real Chrome session.
+ *
+ * Requires system-installed Chrome (channel:"chrome") — no browser binary
+ * is downloaded. Falls back gracefully to plain playwright-core if
+ * playwright-extra or the stealth plugin are not installed.
+ *
+ * Browser lifecycle:
+ *   - Launched lazily on the first fetch() call.
+ *   - Reused across all subsequent requests (one browser, one tab per request).
+ *   - Call close() when done to release the browser process.
+ *
+ * Usage:
+ *   const client = new PlaywrightHttpClient()
+ *   const page   = await spider(url, { httpClient: client })
+ *   await client.close()
+ */
+import type { HttpRequest, HttpResponse, IHttpClient } from "./ports.js";
+export interface PlaywrightClientOptions {
+    /**
+     * Browser channel — finds a system-installed browser automatically.
+     * "chrome"   — Google Chrome (default)
+     * "msedge"   — Microsoft Edge
+     * "chromium" — Playwright's own Chromium (must be installed separately)
+     */
+    channel?: "chrome" | "msedge" | "chromium";
+    /**
+     * Explicit path to a browser executable.
+     * Overrides `channel`. Use when Chrome is not in the standard location.
+     */
+    executablePath?: string;
+    /**
+     * Navigation timeout in ms. Default: 30 000.
+     */
+    timeoutMs?: number;
+    /**
+     * When to consider navigation complete.
+     * "networkidle"      — no network activity for 500ms (best for SPAs, default).
+     * "domcontentloaded" — HTML parsed; faster but may miss lazy-loaded content.
+     * "load"             — window load event fired.
+     */
+    waitUntil?: "load" | "domcontentloaded" | "networkidle" | "commit";
+    /**
+     * When true, image and media resource types are allowed through instead of
+     * being aborted. Required when spider() is called with captureImages: true
+     * so that individual image fetches via this client succeed.
+     * Fonts are always blocked regardless of this flag.
+     * Default: false.
+     */
+    captureImages?: boolean;
+}
+export declare class PlaywrightHttpClient implements IHttpClient {
+    private browser;
+    private readonly channel;
+    private readonly executablePath;
+    private readonly timeoutMs;
+    private readonly waitUntil;
+    private readonly captureImages;
+    constructor(opts?: PlaywrightClientOptions);
+    private getChromium;
+    private getBrowser;
+    fetch(req: HttpRequest): Promise<HttpResponse>;
+    /** Close the shared browser process. Call when the client is no longer needed. */
+    close(): Promise<void>;
+}
+/**
+ * Create a PlaywrightHttpClient, returning null if playwright-core is not
+ * installed. Useful for graceful degradation in environments without a browser.
+ */
+export declare function createPlaywrightClient(opts?: PlaywrightClientOptions): PlaywrightHttpClient | null;
+//# sourceMappingURL=playwright.d.ts.map

package/dist/playwright.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"playwright.d.ts","sourceRoot":"","sources":["../src/playwright.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzE,MAAM,WAAW,uBAAuB;IACvC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,UAAU,CAAC;IAC3C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,GAAG,QAAQ,CAAC;IACnE;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACxB;AAMD,qBAAa,oBAAqB,YAAW,WAAW;IAEvD,OAAO,CAAC,OAAO,CAAoB;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;gBAE5B,IAAI,GAAE,uBAA4B;YAQhC,WAAW;YAiBX,UAAU;IAUlB,KAAK,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAiEpD,kFAAkF;IAC5E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM5B;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACrC,IAAI,CAAC,EAAE,uBAAuB,GAC5B,oBAAoB,GAAG,IAAI,CAM7B"}

package/dist/playwright.js ADDED Viewed

@@ -0,0 +1,141 @@
+/**
+ * Playwright adapter — implements IHttpClient using a headless browser.
+ *
+ * Uses playwright-extra with the stealth plugin, which patches ~15 headless
+ * fingerprint signals (navigator.webdriver, User-Agent, plugins, WebGL, etc.)
+ * so the browser is indistinguishable from a real Chrome session.
+ *
+ * Requires system-installed Chrome (channel:"chrome") — no browser binary
+ * is downloaded. Falls back gracefully to plain playwright-core if
+ * playwright-extra or the stealth plugin are not installed.
+ *
+ * Browser lifecycle:
+ *   - Launched lazily on the first fetch() call.
+ *   - Reused across all subsequent requests (one browser, one tab per request).
+ *   - Call close() when done to release the browser process.
+ *
+ * Usage:
+ *   const client = new PlaywrightHttpClient()
+ *   const page   = await spider(url, { httpClient: client })
+ *   await client.close()
+ */
+// Module-level flag: stealth is wired to the playwright-extra chromium
+// singleton once and stays active for the lifetime of the process.
+let stealthApplied = false;
+export class PlaywrightHttpClient {
+    constructor(opts = {}) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.browser = null;
+        this.channel = opts.channel ?? "chrome";
+        this.executablePath = opts.executablePath ?? "";
+        this.timeoutMs = opts.timeoutMs ?? 30_000;
+        this.waitUntil = opts.waitUntil ?? "networkidle";
+        this.captureImages = opts.captureImages ?? false;
+    }
+    async getChromium() {
+        // Prefer playwright-extra + stealth — patches headless fingerprints.
+        // Falls back to plain playwright-core if playwright-extra isn't installed.
+        try {
+            const { chromium } = await import("playwright-extra");
+            if (!stealthApplied) {
+                const { default: StealthPlugin } = await import("puppeteer-extra-plugin-stealth");
+                chromium.use(StealthPlugin());
+                stealthApplied = true;
+            }
+            return chromium;
+        }
+        catch {
+            const { chromium } = await import("playwright-core");
+            return chromium;
+        }
+    }
+    async getBrowser() {
+        if (this.browser?.isConnected())
+            return this.browser;
+        const chromium = await this.getChromium();
+        const launchOpts = this.executablePath
+            ? { executablePath: this.executablePath, headless: true }
+            : { channel: this.channel, headless: true };
+        this.browser = await chromium.launch(launchOpts);
+        return this.browser;
+    }
+    async fetch(req) {
+        const browser = await this.getBrowser();
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const page = await browser.newPage();
+        // Suppress browser-side console output and JS errors — they are not
+        // useful to the caller and would leak into Pi's TUI stream.
+        page.on("console", () => { });
+        page.on("pageerror", () => { });
+        try {
+            // Block fonts always (never needed for HTML extraction).
+            // Block images and media during page navigation for speed — unless
+            // this is a direct image fetch (Accept: image/*), in which case
+            // captureImages:true lets it through so fetchImages() can retrieve
+            // the binary via arrayBuffer().
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            await page.route("**/*", (route) => {
+                const type = route.request().resourceType();
+                const accept = route.request().headers()["accept"] ?? "";
+                const isImageFetch = accept.startsWith("image/");
+                if (type === "font") {
+                    route.abort();
+                }
+                else if (["image", "media"].includes(type) && !(this.captureImages && isImageFetch)) {
+                    route.abort();
+                }
+                else {
+                    route.continue();
+                }
+            });
+            const response = await page.goto(req.url, {
+                timeout: this.timeoutMs,
+                waitUntil: this.waitUntil,
+            });
+            if (!response) {
+                throw new Error(`Navigation failed — no response for ${req.url}`);
+            }
+            const status = response.status();
+            if (status >= 400) {
+                throw new Error(`HTTP ${status} ${response.statusText()} — ${req.url}`);
+            }
+            // page.content() returns the full serialised DOM after JS execution.
+            const html = await page.content();
+            const headers = await response.allHeaders();
+            return {
+                ok: true,
+                status,
+                statusText: response.statusText(),
+                headers: { get: (name) => headers[name.toLowerCase()] ?? null },
+                text: async () => html,
+                arrayBuffer: async () => {
+                    const buf = await response.body();
+                    return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+                },
+            };
+        }
+        finally {
+            await page.close();
+        }
+    }
+    /** Close the shared browser process. Call when the client is no longer needed. */
+    async close() {
+        if (this.browser) {
+            await this.browser.close();
+            this.browser = null;
+        }
+    }
+}
+/**
+ * Create a PlaywrightHttpClient, returning null if playwright-core is not
+ * installed. Useful for graceful degradation in environments without a browser.
+ */
+export function createPlaywrightClient(opts) {
+    try {
+        return new PlaywrightHttpClient(opts);
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=playwright.js.map

package/dist/ports.d.ts ADDED Viewed

@@ -0,0 +1,104 @@
+/**
+ * Port interfaces — the contracts the core depends on.
+ *
+ * No concrete imports. Adapters implement these; the core orchestrates them.
+ * All ports are optional in SpiderOptions — concrete defaults are wired in
+ * spider.ts and crawl.ts so callers need not supply them unless they want
+ * to substitute (e.g. inject a mock HTTP client for testing).
+ */
+export interface HttpRequest {
+    url: string;
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+}
+export interface HttpResponse {
+    ok: boolean;
+    status: number;
+    statusText: string;
+    headers: {
+        get(name: string): string | null;
+    };
+    text(): Promise<string>;
+    arrayBuffer(): Promise<ArrayBuffer>;
+}
+/**
+ * Minimal HTTP client port.
+ * Default adapter wraps global fetch().
+ * Swap for tests: return fixed HTML without touching the network.
+ */
+export interface IHttpClient {
+    fetch(req: HttpRequest): Promise<HttpResponse>;
+}
+/**
+ * Generic cache port.
+ * Default adapter: SpiderCache (LRU, TTL).
+ * Swap for tests or production: in-memory Map, Redis, SQLite, etc.
+ */
+export interface ICache<K, V> {
+    get(key: K): V | undefined;
+    set(key: K, value: V): void;
+    has(key: K): boolean;
+    delete(key: K): void;
+    /** All currently valid (non-expired) values. */
+    values(): V[];
+}
+/**
+ * Per-domain request throttle port.
+ * Default adapter: DomainThrottle (token bucket + exponential backoff).
+ * Swap for tests: no-op implementation that always resolves immediately.
+ */
+export interface IThrottle {
+    wait(url: string): Promise<void>;
+    success(url: string): void;
+    rateLimit(url: string, retryAfterHeader: string | null): number;
+    setDomainDelay(host: string, ms: number): void;
+    readonly maxRetries: number;
+}
+export interface RobotsResult {
+    allowed: boolean;
+    crawlDelayMs?: number;
+}
+/**
+ * robots.txt compliance port.
+ * Default adapter: RobotsCache (fetches + parses per origin, 1h TTL).
+ * Swap for tests: permissive stub that always returns { allowed: true }.
+ */
+export interface IRobotsChecker {
+    check(url: string): Promise<RobotsResult>;
+}
+export interface SearchQuery {
+    query: string;
+    numResults?: number;
+    /**
+     * Restrict results to content published within this window.
+     * Supported by Tavily ("day"|"week"|"month"|"year") and Brave ("pd"|"pw"|"pm"|"py").
+     * Adapters map this to their engine-specific parameter name.
+     */
+    timeRange?: "day" | "week" | "month" | "year";
+    /**
+     * Search topic mode. "news" prioritises freshly indexed news articles.
+     * Supported by Tavily. Ignored by engines that don't support it.
+     */
+    topic?: "news" | "general";
+}
+/**
+ * A single result from a web search engine.
+ * Defined here so port interfaces have no dependency on adapter modules.
+ */
+export interface WebSearchResult {
+    url: string;
+    title: string;
+    /** Short description or snippet from the search engine. */
+    snippet: string;
+    /** ISO-8601 or human-readable date, if the engine returned one. */
+    publishedAt?: string;
+}
+/**
+ * Web search engine port.
+ * Adapters: BraveSearchEngine, TavilySearchEngine (in web-search.ts).
+ * Swap for tests: stub returning fixed results.
+ */
+export interface ISearchEngine {
+    search(req: SearchQuery): Promise<WebSearchResult[]>;
+}
+//# sourceMappingURL=ports.d.ts.map

package/dist/ports.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ports.d.ts","sourceRoot":"","sources":["../src/ports.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,MAAM,WAAW,WAAW;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC5B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE;QAAE,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IAC9C,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IACxB,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;CACpC;AAED;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC3B,KAAK,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC/C;AAMD;;;;GAIG;AACH,MAAM,WAAW,MAAM,CAAC,CAAC,EAAE,CAAC;IAC3B,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;IAC3B,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC;IACrB,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;IACrB,gDAAgD;IAChD,MAAM,IAAI,CAAC,EAAE,CAAC;CACd;AAMD;;;;GAIG;AACH,MAAM,WAAW,SAAS;IACzB,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAAC;IAChE,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC5B;AAMD,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC9B,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC1C;AAMD,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC7B,MAAM,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACrD"}

package/dist/ports.js ADDED Viewed

@@ -0,0 +1,10 @@
+/**
+ * Port interfaces — the contracts the core depends on.
+ *
+ * No concrete imports. Adapters implement these; the core orchestrates them.
+ * All ports are optional in SpiderOptions — concrete defaults are wired in
+ * spider.ts and crawl.ts so callers need not supply them unless they want
+ * to substitute (e.g. inject a mock HTTP client for testing).
+ */
+export {};
+//# sourceMappingURL=ports.js.map

package/dist/robots.d.ts ADDED Viewed

@@ -0,0 +1,24 @@
+/**
+ * Minimal robots.txt fetcher and per-domain cache.
+ * Respects User-agent: * directives (Allow, Disallow, Crawl-delay).
+ * Fails open — any fetch/parse error allows all URLs.
+ */
+import type { IRobotsChecker, RobotsResult } from "./ports.js";
+export declare class RobotsCache implements IRobotsChecker {
+    private readonly cache;
+    private readonly userAgent;
+    constructor(userAgent?: string);
+    /**
+     * Returns whether the URL is allowed and the crawl-delay if specified.
+     * Caches per origin for 1 hour. Fails open on any error.
+     */
+    check(url: string): Promise<RobotsResult>;
+    private fetchRobots;
+}
+/**
+ * Factory — avoids jiti/Bun CJS re-export interop where class constructors
+ * accessed through a re-export chain can appear undefined at call site.
+ * Use this in extension code instead of `new RobotsCache()`.
+ */
+export declare function createRobotsCache(userAgent?: string): RobotsCache;
+//# sourceMappingURL=robots.d.ts.map

package/dist/robots.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAwDH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/D,qBAAa,WAAY,YAAW,cAAc;IACjD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkE;IACxF,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,SAAS,SAAmB;IAIxC;;;OAGG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAgBjC,WAAW;CAmBzB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,WAAW,CAEjE"}

package/dist/robots.js ADDED Viewed

@@ -0,0 +1,104 @@
+/**
+ * Minimal robots.txt fetcher and per-domain cache.
+ * Respects User-agent: * directives (Allow, Disallow, Crawl-delay).
+ * Fails open — any fetch/parse error allows all URLs.
+ */
+function parse(text) {
+    const lines = text.split(/\r?\n/);
+    const directives = [];
+    let crawlDelayMs;
+    let inBlock = false;
+    for (const raw of lines) {
+        const line = raw.split("#")[0].trim();
+        if (!line)
+            continue;
+        const colon = line.indexOf(":");
+        if (colon === -1)
+            continue;
+        const key = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (key === "user-agent") {
+            inBlock = value === "*";
+        }
+        else if (inBlock) {
+            if (key === "disallow" && value) {
+                directives.push({ allow: false, path: value });
+            }
+            else if (key === "allow" && value) {
+                directives.push({ allow: true, path: value });
+            }
+            else if (key === "crawl-delay") {
+                const s = parseFloat(value);
+                if (!isNaN(s) && s > 0)
+                    crawlDelayMs = Math.min(s * 1_000, 60_000);
+            }
+        }
+    }
+    return { directives, crawlDelayMs };
+}
+function isAllowed(robots, path) {
+    // Longest matching path prefix wins.
+    let best;
+    for (const d of robots.directives) {
+        if (path.startsWith(d.path)) {
+            if (!best || d.path.length > best.path.length)
+                best = d;
+        }
+    }
+    return best?.allow ?? true; // default: allow
+}
+const TTL_MS = 60 * 60 * 1_000; // 1 hour
+export class RobotsCache {
+    constructor(userAgent = "web-spider/0.1") {
+        this.cache = new Map();
+        this.userAgent = userAgent;
+    }
+    /**
+     * Returns whether the URL is allowed and the crawl-delay if specified.
+     * Caches per origin for 1 hour. Fails open on any error.
+     */
+    async check(url) {
+        const { origin, pathname } = new URL(url);
+        let entry = this.cache.get(origin);
+        if (!entry || Date.now() > entry.expiresAt) {
+            const robots = await this.fetchRobots(`${origin}/robots.txt`);
+            entry = { robots, expiresAt: Date.now() + TTL_MS };
+            this.cache.set(origin, entry);
+        }
+        return {
+            allowed: isAllowed(entry.robots, pathname),
+            crawlDelayMs: entry.robots.crawlDelayMs,
+        };
+    }
+    async fetchRobots(robotsUrl) {
+        try {
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), 5_000);
+            let res;
+            try {
+                res = await globalThis.fetch(robotsUrl, {
+                    signal: controller.signal,
+                    headers: { "User-Agent": this.userAgent },
+                });
+            }
+            finally {
+                clearTimeout(timer);
+            }
+            if (!res.ok)
+                return { directives: [] }; // 404 → allow all
+            return parse(await res.text());
+        }
+        catch {
+            return { directives: [] }; // network error → fail open
+        }
+    }
+}
+/**
+ * Factory — avoids jiti/Bun CJS re-export interop where class constructors
+ * accessed through a re-export chain can appear undefined at call site.
+ * Use this in extension code instead of `new RobotsCache()`.
+ */
+export function createRobotsCache(userAgent) {
+    return new RobotsCache(userAgent);
+}
+//# sourceMappingURL=robots.js.map

package/dist/search.d.ts ADDED Viewed

@@ -0,0 +1,47 @@
+import type { SpideredPage } from "./types.js";
+/** A single ranked match from fuzzySearch. */
+export interface SearchHit {
+    /** URL of the page the match came from. */
+    url: string;
+    /**
+     * Stable chunk ID ("url#chunk-N") when the match is in body text.
+     * Empty string when the match is in page metadata (title, description,
+     * headings).
+     */
+    chunkId: string;
+    /** Nearest heading for the matched chunk, or the matched field name for
+     *  metadata hits (e.g. "title", "description"). */
+    heading: string;
+    /** Normalised score 0–1. Higher is a better match. */
+    score: number;
+    /** Short context window around the best match, ≤ 2×snippetRadius chars.
+     *  Prefixed/suffixed with "…" when truncated. */
+    snippet: string;
+}
+export interface FuzzySearchOptions {
+    /** Maximum hits to return (default 10). */
+    topN?: number;
+    /**
+     * Characters of context on each side of the match in the snippet
+     * (default 100). Keep low to save tokens; raise when you need more context.
+     */
+    snippetRadius?: number;
+}
+/**
+ * Full-text search across a set of already-spidered pages using MiniSearch
+ * (BM25F ranking, fuzzy edit-distance, prefix search, heading field boost ×2).
+ *
+ * Searches both body chunks and page metadata (title, description, headings).
+ * Returns results ranked by score descending, normalised to 0–1.
+ *
+ * Designed for agent use: call after fetching pages to locate a specific
+ * fact, term, or section without dumping all content into context.
+ *
+ * @example
+ * const hits = searchPages(pages, "cost optimization selectors", { topN: 5 })
+ * // hits[0].snippet → "…LLM extraction vs Selectors…"
+ */
+export declare function searchPages(pages: SpideredPage[], query: string, opts?: FuzzySearchOptions): SearchHit[];
+/** @deprecated Use {@link searchPages} — renamed in v0.4.0 to reflect BM25F ranking. */
+export declare const fuzzySearch: typeof searchPages;
+//# sourceMappingURL=search.d.ts.map

package/dist/search.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,8CAA8C;AAC9C,MAAM,WAAW,SAAS;IACzB,2CAA2C;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;uDACmD;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;IACd;qDACiD;IACjD,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IAClC,2CAA2C;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB;AA2DD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,kBAAuB,GAAG,SAAS,EAAE,CAiE5G;AAED,wFAAwF;AACxF,eAAO,MAAM,WAAW,oBAAc,CAAA"}