@dpopsuev/web-spider 0.10.4

package/src/disk-cache.ts

@@ -0,0 +1,228 @@
+/**
+ * Disk-backed cache implementing ICache<string, SpideredPage>.
+ *
+ * Persists to a JSON file so the cache survives extension reloads and
+ * pi restarts. Call flush() to write — set() auto-flushes by default.
+ *
+ * The images directory is derived automatically from `dirname(path)/images`.
+ * Callers do not need to create it — DiskCache creates it on first large-image
+ * flush. Pre-creating it at startup (e.g. in the extension boot path) is
+ * harmless and avoids a first-write delay.
+ *
+ * Internal storage uses a plain object (Object.create(null)) rather than a
+ * Map. Plain objects carry no realm-specific internal slots, making them safe
+ * across V8 context (realm) boundaries — e.g. when DiskCache is constructed
+ * in an ESM module realm but called from a jiti VM-sandbox realm (Bun binary
+ * mode). The Map-backed version threw "Map operation called on non-Map object"
+ * in that scenario.
+ *
+ * A schema version field in the persisted JSON guards against stale cache
+ * files from previous major versions being silently loaded with wrong shapes.
+ */
+
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, extname, join } from "node:path";
+import type { ICache } from "./ports.js";
+import type { ImageRef, SpideredPage } from "./types.js";
+
+/** Bump when the on-disk entry shape changes incompatibly. */
+const SCHEMA_VERSION = 2;
+
+export interface DiskCacheOptions {
+	/** Time-to-live in ms. Default 30 min. */
+	ttlMs?: number;
+	/** Max entries. Default 500. */
+	maxSize?: number;
+	/** Auto-flush to disk on every set(). Default true. */
+	autoFlush?: boolean;
+	/**
+	 * Base64 byte threshold for inline vs. file storage of images.
+	 * Images whose base64 string length exceeds this are written as binary
+	 * files to <cache-dir>/images/ instead of being stored inline in the JSON.
+	 * Default: 32 * 1024 (32 KB of base64 ≈ 24 KB binary).
+	 */
+	inlineImageThreshold?: number;
+}
+
+interface Entry {
+	page: SpideredPage;
+	expiresAt: number;
+}
+
+/** Versioned wrapper written to disk. */
+interface DiskPayload {
+	v: number;
+	entries: Record<string, Entry>;
+}
+
+export class DiskCache implements ICache<string, SpideredPage> {
+	private readonly store: Record<string, Entry | undefined> = Object.create(null);
+	private readonly path: string;
+	private readonly ttlMs: number;
+	private readonly maxSize: number;
+	private readonly autoFlush: boolean;
+	private readonly inlineImageThreshold: number;
+	/** Directory where large image binaries are stored. */
+	private readonly imagesDir: string;
+
+	constructor(path: string, opts: DiskCacheOptions = {}) {
+		this.path = path;
+		this.ttlMs = opts.ttlMs ?? 30 * 60 * 1000;
+		this.maxSize = opts.maxSize ?? 500;
+		this.autoFlush = opts.autoFlush ?? true;
+		this.inlineImageThreshold = opts.inlineImageThreshold ?? 32 * 1024;
+		this.imagesDir = join(dirname(path), "images");
+		this.load();
+	}
+
+	private key(url: string): string {
+		try {
+			const u = new URL(url);
+			u.hash = "";
+			return u.toString().replace(/\/$/, "");
+		} catch {
+			return url;
+		}
+	}
+
+	set(url: string, page: SpideredPage): void {
+		const k = this.key(url);
+		if (Object.keys(this.store).length >= this.maxSize && !(k in this.store)) {
+			const oldest = Object.keys(this.store)[0];
+			if (oldest !== undefined) delete this.store[oldest];
+		}
+		this.store[k] = { page, expiresAt: Date.now() + this.ttlMs };
+		if (this.autoFlush) this.flush();
+	}
+
+	has(url: string): boolean {
+		return this.get(url) !== undefined;
+	}
+
+	delete(url: string): void {
+		delete this.store[this.key(url)];
+		if (this.autoFlush) this.flush();
+	}
+
+	// ---------------------------------------------------------------------------
+	// Image helpers
+	// ---------------------------------------------------------------------------
+
+	/** Derive a stable filename for an image binary from its src URL. */
+	private imageFilename(src: string): string {
+		const hash = createHash("sha1").update(src).digest("hex");
+		const ext = extname(src.split("?")[0]) || ".bin";
+		return `${hash}${ext}`;
+	}
+
+	/**
+	 * Prepare images for serialisation:
+	 * - Images whose base64 length ≤ threshold are kept inline.
+	 * - Larger images are written to imagesDir as binary files; base64 is
+	 *   replaced by filePath in the serialised entry.
+	 */
+	private spill(images: ImageRef[]): ImageRef[] {
+		if (!existsSync(this.imagesDir)) {
+			mkdirSync(this.imagesDir, { recursive: true });
+		}
+		return images.map((img) => {
+			if (!img.base64 || img.base64.length <= this.inlineImageThreshold) {
+				return img;
+			}
+			const filename = this.imageFilename(img.src);
+			const filePath = join(this.imagesDir, filename);
+			writeFileSync(filePath, Buffer.from(img.base64, "base64"));
+			const { base64: _omit, ...rest } = img;
+			return { ...rest, filePath };
+		});
+	}
+
+	/**
+	 * Hydrate images on read: if an image has filePath but no base64,
+	 * load the binary from disk and re-encode.
+	 */
+	private hydrate(images: ImageRef[]): ImageRef[] {
+		return images.map((img) => {
+			if (img.base64 || !img.filePath) return img;
+			if (!existsSync(img.filePath)) return img;
+			try {
+				const base64 = readFileSync(img.filePath).toString("base64");
+				return { ...img, base64 };
+			} catch {
+				return img;
+			}
+		});
+	}
+
+	// ---------------------------------------------------------------------------
+	// Persistence
+	// ---------------------------------------------------------------------------
+
+	/** Write current contents to disk. Large images are spilled to imagesDir. */
+	flush(): void {
+		const now = Date.now();
+		const entries: Record<string, Entry> = {};
+		for (const [k, v] of Object.entries(this.store)) {
+			if (!v || v.expiresAt <= now) continue;
+			const page = v.page;
+			const serialised: SpideredPage = page.images
+				? { ...page, images: this.spill(page.images) }
+				: page;
+			entries[k] = { page: serialised, expiresAt: v.expiresAt };
+		}
+		const payload: DiskPayload = { v: SCHEMA_VERSION, entries };
+		writeFileSync(this.path, JSON.stringify(payload), "utf8");
+	}
+
+	private load(): void {
+		if (!existsSync(this.path)) return;
+		try {
+			const raw = JSON.parse(readFileSync(this.path, "utf8")) as unknown;
+
+			// Reject files from incompatible schema versions (including old
+			// unversioned files that lack the "v" field entirely).
+			if (
+				typeof raw !== "object" ||
+				raw === null ||
+				(raw as { v?: unknown }).v !== SCHEMA_VERSION
+			) {
+				return; // stale schema — start fresh, do not throw
+			}
+
+			const payload = raw as DiskPayload;
+			const now = Date.now();
+			for (const [k, v] of Object.entries(payload.entries)) {
+				if (v.expiresAt > now) this.store[k] = v;
+			}
+		} catch {
+			// Corrupt or unreadable file — start fresh.
+		}
+	}
+
+	/** All currently valid (non-expired) pages, sorted newest-first. */
+	values(): SpideredPage[] {
+		const now = Date.now();
+		return Object.values(this.store)
+			.filter((e): e is Entry => e !== undefined && e.expiresAt > now)
+			.sort((a, b) => b.expiresAt - a.expiresAt)
+			.map((e) => {
+				const page = e.page;
+				return page.images ? { ...page, images: this.hydrate(page.images) } : page;
+			});
+	}
+
+	/** Retrieve a page, hydrating any file-backed images from disk. */
+	get(url: string): SpideredPage | undefined {
+		const k = this.key(url);
+		const entry = this.store[k];
+		if (!entry) return undefined;
+		if (Date.now() > entry.expiresAt) {
+			delete this.store[k];
+			return undefined;
+		}
+		const page = entry.page;
+		if (page.images) return { ...page, images: this.hydrate(page.images) };
+		return page;
+	}
+}

package/src/graph.ts

@@ -0,0 +1,189 @@
+import type { SpideredPage } from "./types.js";
+
+/** A node in the knowledge graph — lightweight reference, not the full page. */
+export interface PageNode {
+	url: string;
+	domain: string;
+	title: string;
+	description: string;
+	wordCount: number;
+	fetchedAt: string;
+	chunkCount: number;
+}
+
+/** A directed edge between two pages. */
+export interface PageEdge {
+	from: string;
+	to: string;
+	/** Anchor text of the link */
+	text: string;
+	isExternal: boolean;
+}
+
+/** Serialisable snapshot for storage or embedding. */
+export interface PageGraphSnapshot {
+	nodes: PageNode[];
+	edges: PageEdge[];
+}
+
+/**
+ * Directed knowledge graph of spidered pages.
+ *
+ * Nodes are pages. Edges are outbound links.
+ * Maintains a reverse index (inbound links) for O(1) lookup.
+ *
+ * All graph queries return plain data — no PageNode references —
+ * so the graph is trivially serialisable.
+ *
+ * Internal storage uses plain objects (Object.create(null)) rather than
+ * Maps. Plain objects carry no realm-specific internal slots, making them
+ * safe across V8 context (realm) boundaries — e.g. when the graph is
+ * constructed in an ESM module realm but called from a jiti VM-sandbox.
+ */
+export class PageGraph {
+	private readonly nodes: Record<string, PageNode | undefined> = Object.create(null);
+	/** url → outbound edges */
+	private readonly out: Record<string, PageEdge[] | undefined> = Object.create(null);
+	/** url → inbound source urls */
+	private readonly in_: Record<string, string[] | undefined> = Object.create(null);
+
+	/** Add or update a node from a spidered page. */
+	addPage(page: SpideredPage): void {
+		this.nodes[page.url] = {
+			url: page.url,
+			domain: page.domain,
+			title: page.title,
+			description: page.description,
+			wordCount: page.wordCount,
+			fetchedAt: page.fetchedAt,
+			chunkCount: page.chunks.length,
+		};
+
+		for (const link of page.links) {
+			if (!link.href) continue;
+			this.addEdge(page.url, link.href, link.text, link.isExternal);
+		}
+	}
+
+	/** Add a directed edge without requiring the target to be spidered yet. */
+	addEdge(from: string, to: string, text: string, isExternal: boolean): void {
+		const edge: PageEdge = { from, to, text, isExternal };
+		const existing = this.out[from] ?? [];
+		if (!existing.some((e) => e.to === to)) {
+			this.out[from] = [...existing, edge];
+		}
+		const inbound = this.in_[to] ?? [];
+		if (!inbound.includes(from)) {
+			this.in_[to] = [...inbound, from];
+		}
+	}
+
+	node(url: string): PageNode | undefined {
+		return this.nodes[url];
+	}
+
+	/** Outbound edges from a node. */
+	outbound(url: string): PageEdge[] {
+		return this.out[url] ?? [];
+	}
+
+	/** URLs that link TO this page. */
+	inbound(url: string): string[] {
+		return this.in_[url] ?? [];
+	}
+
+	/** Pages with no inbound links — entry points to the graph. */
+	roots(): PageNode[] {
+		return Object.values(this.nodes)
+			.filter((n): n is PageNode => n !== undefined && (this.in_[n.url] ?? []).length === 0);
+	}
+
+	/** Pages with no outbound links to other spidered nodes. */
+	sinks(): PageNode[] {
+		return Object.values(this.nodes)
+			.filter((n): n is PageNode => {
+				if (!n) return false;
+				const edges = this.out[n.url] ?? [];
+				return !edges.some((e) => e.to in this.nodes);
+			});
+	}
+
+	/** BFS shortest path between two page URLs. Returns null if unreachable. */
+	findPath(from: string, to: string): string[] | null {
+		if (from === to) return [from];
+		const visited = new Set<string>([from]);
+		const queue: Array<string[]> = [[from]];
+
+		while (queue.length > 0) {
+			const path = queue.shift()!;
+			const current = path[path.length - 1];
+			for (const edge of this.out[current] ?? []) {
+				if (edge.to === to) return [...path, to];
+				if (!visited.has(edge.to) && edge.to in this.nodes) {
+					visited.add(edge.to);
+					queue.push([...path, edge.to]);
+				}
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * All pages reachable from `startUrl` via spidered links.
+	 * BFS, bounded by the nodes present in the graph.
+	 */
+	reachableFrom(startUrl: string): PageNode[] {
+		const visited = new Set<string>([startUrl]);
+		const queue = [startUrl];
+		while (queue.length > 0) {
+			const url = queue.shift()!;
+			for (const edge of this.out[url] ?? []) {
+				if (!visited.has(edge.to) && edge.to in this.nodes) {
+					visited.add(edge.to);
+					queue.push(edge.to);
+				}
+			}
+		}
+		visited.delete(startUrl);
+		return [...visited].map((u) => this.nodes[u]).filter((n): n is PageNode => n !== undefined);
+	}
+
+	/** Nodes ranked by inbound link count (highest first). */
+	byPageRank(): Array<{ node: PageNode; inboundCount: number }> {
+		return Object.values(this.nodes)
+			.filter((n): n is PageNode => n !== undefined)
+			.map((n) => ({ node: n, inboundCount: (this.in_[n.url] ?? []).length }))
+			.sort((a, b) => b.inboundCount - a.inboundCount);
+	}
+
+	get nodeCount(): number {
+		return Object.keys(this.nodes).length;
+	}
+
+	get edgeCount(): number {
+		let total = 0;
+		for (const edges of Object.values(this.out)) {
+			if (edges) total += edges.length;
+		}
+		return total;
+	}
+
+	/** Plain snapshot — safe to JSON.stringify or embed. */
+	toJSON(): PageGraphSnapshot {
+		const edges: PageEdge[] = [];
+		for (const edgeList of Object.values(this.out)) {
+			if (edgeList) edges.push(...edgeList);
+		}
+		return {
+			nodes: Object.values(this.nodes).filter((n): n is PageNode => n !== undefined),
+			edges,
+		};
+	}
+
+	static fromJSON(snap: PageGraphSnapshot): PageGraph {
+		const g = new PageGraph();
+		for (const n of snap.nodes) g.nodes[n.url] = n;
+		for (const e of snap.edges) g.addEdge(e.from, e.to, e.text, e.isExternal);
+		return g;
+	}
+}

package/src/index.ts

@@ -0,0 +1,74 @@
+// ---------------------------------------------------------------------------
+// Public API — what most consumers need
+// ---------------------------------------------------------------------------
+
+export type { SpiderCacheOptions } from "./cache.js";
+export { SpiderCache } from "./cache.js";
+export type { CrawlOptions, CrawlResult } from "./crawl.js";
+export { crawl } from "./crawl.js";
+export type { PageEdge, PageGraphSnapshot, PageNode } from "./graph.js";
+export { PageGraph } from "./graph.js";
+export type { FuzzySearchOptions, SearchHit } from "./search.js";
+export { searchPages } from "./search.js";
+/** @deprecated Use {@link searchPages} — renamed in v0.4.0 to reflect BM25F ranking (not fuzzy-only). */
+export { searchPages as fuzzySearch } from "./search.js";
+export type { SpiderOptions, TreePage } from "./spider.js";
+export { spider } from "./spider.js";
+export type { QueryTreeOptions } from "./tree.js";
+export { buildTree, navigateTree, queryTree } from "./tree.js";
+export type { Chunk, ChunkType, DOMNode, ImageRef, LeanLink, LeanPage, Link, PageView, SpideredPage, TreeHit } from "./types.js";
+export { toLean } from "./views.js";
+export type { BraveSearchOptions, DdgSearchOptions, ExaSearchOptions, FallbackSearchEngineOptions, SearchEngine, TavilySearchOptions, WebSearchResult } from "./web-search.js";
+export { braveSearch, ddgSearch, exaSearch, registerSearchEngine, resolveSearchEngine, tavilySearch, webSearch } from "./web-search.js";
+
+// ---------------------------------------------------------------------------
+// Utilities
+// ---------------------------------------------------------------------------
+
+import type { ICache } from "./ports.js";
+import type { Chunk, SpideredPage } from "./types.js";
+
+/**
+ * Retrieve a single chunk from a cached page by URL and chunk index.
+ *
+ * Avoids loading the full page markdown when an agent only needs one
+ * specific chunk — e.g. to re-read a section after a highlights hit.
+ *
+ * Returns undefined when the URL is not cached, the index is out of range,
+ * or the index is negative.
+ *
+ * @example
+ * const chunk = getChunk(cache, "https://example.com/article", 3)
+ * if (chunk) console.log(chunk.text)
+ */
+export function getChunk(
+	cache: ICache<string, SpideredPage>,
+	url: string,
+	index: number,
+): Chunk | undefined {
+	if (index < 0) return undefined;
+	return cache.get(url)?.chunks[index];
+}
+
+// ---------------------------------------------------------------------------
+// Extension / DI — port interfaces and their concrete adapters.
+// Import these when you need to inject custom implementations.
+// ---------------------------------------------------------------------------
+
+export type { HttpRequest, HttpResponse, ICache, IHttpClient, IRobotsChecker, ISearchEngine, IThrottle, RobotsResult, SearchQuery } from "./ports.js";
+export type { DiskCacheOptions } from "./disk-cache.js";
+export { DiskCache } from "./disk-cache.js";
+export type { PlaywrightClientOptions } from "./playwright.js";
+export { PlaywrightHttpClient, createPlaywrightClient } from "./playwright.js";
+export { RobotsCache, createRobotsCache } from "./robots.js";
+export { fetchSitemapUrls } from "./sitemap.js";
+export type { ThrottleOptions } from "./throttle.js";
+export { DomainThrottle, createThrottle } from "./throttle.js";
+export { BraveSearchEngine, DdgSearchEngine, ExaSearchEngine, FallbackSearchEngine, TavilySearchEngine, defaultSearchEngine } from "./web-search.js";
+
+// ---------------------------------------------------------------------------
+// parse.ts, convert.ts, views.ts are internal implementation modules.
+// They are NOT exported here — they are consumed only by spider.ts.
+// If you need lower-level DOM or markdown utilities, import from the
+// sub-modules directly (not covered by semver stability guarantees).
+// ---------------------------------------------------------------------------

package/src/parse.ts

@@ -0,0 +1,154 @@
+/**
+ * DOM parsing helpers.
+ *
+ * Owns the DOM parsing dependency. spider.ts calls these after fetching HTML;
+ * it never touches the DOM library directly.
+ */
+
+import { parseHTML } from "linkedom";
+import type { Link, SpideredPage } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// DOM creation
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse raw HTML into a DOM Document.
+ * Uses linkedom — a lightweight server-side DOM that has no CSS engine,
+ * no module-level Maps, and a flat CJS dependency tree. Safe to load
+ * through jiti's transform pipeline without nativeModules workarounds.
+ */
+export function parseDom(html: string, url: string): Document {
+	return parseHTML(html, { url }).document as unknown as Document;
+}
+
+// ---------------------------------------------------------------------------
+// Nav classification
+// ---------------------------------------------------------------------------
+
+const NAV_CLASS_RE =
+	/^(nav|navbar|navigation|menu|menubar|header|footer|sidebar|breadcrumb|topbar|toolbar|site-nav|main-nav|primary-nav|global-nav)$/i;
+
+/** True if el or any ancestor up to 5 levels looks like navigation chrome. */
+export function isNavElement(el: Element): boolean {
+	if (el.closest("nav, header, footer, aside")) return true;
+	if (
+		el.closest(
+			"[role='navigation'],[role='banner'],[role='contentinfo'],[role='complementary']",
+		)
+	)
+		return true;
+
+	let node: Element | null = el;
+	for (let i = 0; i < 5; i++) {
+		if (!node) break;
+		for (const cls of node.classList) {
+			if (NAV_CLASS_RE.test(cls)) return true;
+		}
+		node = node.parentElement;
+	}
+	return false;
+}
+
+// ---------------------------------------------------------------------------
+// Link text extraction
+// ---------------------------------------------------------------------------
+
+/** Extract visible text from an anchor, skipping SVG subtrees. */
+export function anchorText(a: Element): string {
+	if (!a.querySelector("svg")) {
+		return (a.textContent ?? "").replace(/\s+/g, " ").trim();
+	}
+	const clone = a.cloneNode(true) as Element;
+	for (const svg of [...clone.querySelectorAll("svg")]) svg.remove();
+	return (clone.textContent ?? "").replace(/\s+/g, " ").trim();
+}
+
+// ---------------------------------------------------------------------------
+// Link extraction
+// ---------------------------------------------------------------------------
+
+/** Extract outbound links from the DOM, classified as body or nav. */
+export function extractLinks(doc: Document, baseUrl: string): Link[] {
+	const origin = new URL(baseUrl).origin;
+	return Array.from(doc.querySelectorAll("a[href]"))
+		.map((a) => {
+			const href = (a as HTMLAnchorElement).href;
+			const text = anchorText(a)
+				.replace(
+					/\b(open_in_new|navigate_next|navigate_before|arrow_drop_down|arrow_drop_up|chevron_right|chevron_left|expand_more|expand_less)\b/g,
+					"",
+				)
+				.replace(/\s+/g, " ")
+				.trim();
+			if (!href || !text || href.startsWith("javascript:")) return null;
+
+			return {
+				href,
+				text,
+				isExternal: !href.startsWith(origin),
+				rel: isNavElement(a) ? ("nav" as const) : ("body" as const),
+			} satisfies Link;
+		})
+		.filter((l): l is Link => l !== null)
+		.slice(0, 200);
+}
+
+// ---------------------------------------------------------------------------
+// Heading extraction
+// ---------------------------------------------------------------------------
+
+/** Extract h1/h2/h3 headings from Readability article HTML. */
+export function extractHeadings(html: string): SpideredPage["headings"] {
+	const { document } = parseHTML(`<html><body>${html}</body></html>`);
+	const headings: SpideredPage["headings"] = [];
+	document.querySelectorAll("h1, h2, h3").forEach((el) => {
+		const level = parseInt(el.tagName[1], 10) as 1 | 2 | 3;
+		const text = (el.textContent ?? "").trim();
+		if (text) headings.push({ level, text });
+	});
+	return headings;
+}
+
+// ---------------------------------------------------------------------------
+// Tag extraction
+// ---------------------------------------------------------------------------
+
+/** Extract topic tags from meta keywords and article:tag. */
+export function extractTags(doc: Document): string[] {
+	const tags = new Set<string>();
+
+	const keywords = doc.querySelector('meta[name="keywords"]')?.getAttribute("content") ?? "";
+	for (const k of keywords
+		.split(/[,;]/)
+		.map((k) => k.trim().toLowerCase())
+		.filter(Boolean)) {
+		tags.add(k);
+	}
+
+	doc.querySelectorAll('meta[property="article:tag"], meta[name="article:tag"]').forEach((el) => {
+		const t = el.getAttribute("content")?.trim().toLowerCase();
+		if (t) tags.add(t);
+	});
+
+	const section =
+		doc.querySelector('meta[property="article:section"]')?.getAttribute("content") ??
+		doc.querySelector('meta[property="og:article:section"]')?.getAttribute("content");
+	if (section) tags.add(section.trim().toLowerCase());
+
+	return [...tags].slice(0, 20);
+}
+
+// ---------------------------------------------------------------------------
+// Canonical URL extraction
+// ---------------------------------------------------------------------------
+
+/** Extract canonical URL from link[rel=canonical] or og:url. */
+export function extractCanonicalUrl(doc: Document, fetchedUrl: string): string | undefined {
+	const canonical =
+		doc.querySelector('link[rel="canonical"]')?.getAttribute("href") ??
+		doc.querySelector('meta[property="og:url"]')?.getAttribute("content");
+	if (!canonical) return undefined;
+	const norm = (u: string) => u.replace(/\/$/, "");
+	return norm(canonical) !== norm(fetchedUrl) ? canonical : undefined;
+}