@dpopsuev/web-spider 0.10.4 → 0.10.5

package/src/playwright.ts

@@ -1,193 +0,0 @@
-/**
- * Playwright adapter — implements IHttpClient using a headless browser.
- *
- * Uses playwright-extra with the stealth plugin, which patches ~15 headless
- * fingerprint signals (navigator.webdriver, User-Agent, plugins, WebGL, etc.)
- * so the browser is indistinguishable from a real Chrome session.
- *
- * Requires system-installed Chrome (channel:"chrome") — no browser binary
- * is downloaded. Falls back gracefully to plain playwright-core if
- * playwright-extra or the stealth plugin are not installed.
- *
- * Browser lifecycle:
- *   - Launched lazily on the first fetch() call.
- *   - Reused across all subsequent requests (one browser, one tab per request).
- *   - Call close() when done to release the browser process.
- *
- * Usage:
- *   const client = new PlaywrightHttpClient()
- *   const page   = await spider(url, { httpClient: client })
- *   await client.close()
- */
-
-import type { HttpRequest, HttpResponse, IHttpClient } from "./ports.js";
-
-export interface PlaywrightClientOptions {
-	/**
-	 * Browser channel — finds a system-installed browser automatically.
-	 * "chrome"   — Google Chrome (default)
-	 * "msedge"   — Microsoft Edge
-	 * "chromium" — Playwright's own Chromium (must be installed separately)
-	 */
-	channel?: "chrome" | "msedge" | "chromium";
-	/**
-	 * Explicit path to a browser executable.
-	 * Overrides `channel`. Use when Chrome is not in the standard location.
-	 */
-	executablePath?: string;
-	/**
-	 * Navigation timeout in ms. Default: 30 000.
-	 */
-	timeoutMs?: number;
-	/**
-	 * When to consider navigation complete.
-	 * "networkidle"      — no network activity for 500ms (best for SPAs, default).
-	 * "domcontentloaded" — HTML parsed; faster but may miss lazy-loaded content.
-	 * "load"             — window load event fired.
-	 */
-	waitUntil?: "load" | "domcontentloaded" | "networkidle" | "commit";
-	/**
-	 * When true, image and media resource types are allowed through instead of
-	 * being aborted. Required when spider() is called with captureImages: true
-	 * so that individual image fetches via this client succeed.
-	 * Fonts are always blocked regardless of this flag.
-	 * Default: false.
-	 */
-	captureImages?: boolean;
-}
-
-// Module-level flag: stealth is wired to the playwright-extra chromium
-// singleton once and stays active for the lifetime of the process.
-let stealthApplied = false;
-
-export class PlaywrightHttpClient implements IHttpClient {
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any
-	private browser: any | null = null;
-	private readonly channel: string;
-	private readonly executablePath: string;
-	private readonly timeoutMs: number;
-	private readonly waitUntil: string;
-	private readonly captureImages: boolean;
-
-	constructor(opts: PlaywrightClientOptions = {}) {
-		this.channel = opts.channel ?? "chrome";
-		this.executablePath = opts.executablePath ?? "";
-		this.timeoutMs = opts.timeoutMs ?? 30_000;
-		this.waitUntil = opts.waitUntil ?? "networkidle";
-		this.captureImages = opts.captureImages ?? false;
-	}
-
-	private async getChromium() {
-		// Prefer playwright-extra + stealth — patches headless fingerprints.
-		// Falls back to plain playwright-core if playwright-extra isn't installed.
-		try {
-			const { chromium } = await import("playwright-extra");
-			if (!stealthApplied) {
-				const { default: StealthPlugin } = await import("puppeteer-extra-plugin-stealth");
-				chromium.use(StealthPlugin());
-				stealthApplied = true;
-			}
-			return chromium;
-		} catch {
-			const { chromium } = await import("playwright-core");
-			return chromium;
-		}
-	}
-
-	private async getBrowser() {
-		if (this.browser?.isConnected()) return this.browser;
-		const chromium = await this.getChromium();
-		const launchOpts = this.executablePath
-			? { executablePath: this.executablePath, headless: true }
-			: { channel: this.channel, headless: true };
-		this.browser = await chromium.launch(launchOpts);
-		return this.browser;
-	}
-
-	async fetch(req: HttpRequest): Promise<HttpResponse> {
-		const browser = await this.getBrowser();
-		// eslint-disable-next-line @typescript-eslint/no-explicit-any
-		const page: any = await browser.newPage();
-
-		// Suppress browser-side console output and JS errors — they are not
-		// useful to the caller and would leak into Pi's TUI stream.
-		page.on("console", () => {});
-		page.on("pageerror", () => {});
-
-		try {
-			// Block fonts always (never needed for HTML extraction).
-			// Block images and media during page navigation for speed — unless
-			// this is a direct image fetch (Accept: image/*), in which case
-			// captureImages:true lets it through so fetchImages() can retrieve
-			// the binary via arrayBuffer().
-			// eslint-disable-next-line @typescript-eslint/no-explicit-any
-			await page.route("**/*", (route: any) => {
-				const type: string = route.request().resourceType();
-				const accept: string = route.request().headers()["accept"] ?? "";
-				const isImageFetch = accept.startsWith("image/");
-
-				if (type === "font") {
-					route.abort();
-				} else if (["image", "media"].includes(type) && !(this.captureImages && isImageFetch)) {
-					route.abort();
-				} else {
-					route.continue();
-				}
-			});
-
-			const response = await page.goto(req.url, {
-				timeout: this.timeoutMs,
-				waitUntil: this.waitUntil,
-			});
-
-			if (!response) {
-				throw new Error(`Navigation failed — no response for ${req.url}`);
-			}
-
-			const status: number = response.status();
-			if (status >= 400) {
-				throw new Error(`HTTP ${status} ${response.statusText()} — ${req.url}`);
-			}
-
-			// page.content() returns the full serialised DOM after JS execution.
-			const html: string = await page.content();
-			const headers: Record<string, string> = await response.allHeaders();
-
-			return {
-				ok: true,
-				status,
-				statusText: response.statusText(),
-				headers: { get: (name: string) => headers[name.toLowerCase()] ?? null },
-				text: async () => html,
-				arrayBuffer: async () => {
-					const buf: Buffer = await response.body();
-					return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer;
-				},
-			};
-		} finally {
-			await page.close();
-		}
-	}
-
-	/** Close the shared browser process. Call when the client is no longer needed. */
-	async close(): Promise<void> {
-		if (this.browser) {
-			await this.browser.close();
-			this.browser = null;
-		}
-	}
-}
-
-/**
- * Create a PlaywrightHttpClient, returning null if playwright-core is not
- * installed. Useful for graceful degradation in environments without a browser.
- */
-export function createPlaywrightClient(
-	opts?: PlaywrightClientOptions,
-): PlaywrightHttpClient | null {
-	try {
-		return new PlaywrightHttpClient(opts);
-	} catch {
-		return null;
-	}
-}

package/src/ports.ts

@@ -1,131 +0,0 @@
-/**
- * Port interfaces — the contracts the core depends on.
- *
- * No concrete imports. Adapters implement these; the core orchestrates them.
- * All ports are optional in SpiderOptions — concrete defaults are wired in
- * spider.ts and crawl.ts so callers need not supply them unless they want
- * to substitute (e.g. inject a mock HTTP client for testing).
- */
-
-// ---------------------------------------------------------------------------
-// IHttpClient
-// ---------------------------------------------------------------------------
-
-export interface HttpRequest {
-	url: string;
-	headers?: Record<string, string>;
-	signal?: AbortSignal;
-}
-
-export interface HttpResponse {
-	ok: boolean;
-	status: number;
-	statusText: string;
-	headers: { get(name: string): string | null };
-	text(): Promise<string>;
-	arrayBuffer(): Promise<ArrayBuffer>;
-}
-
-/**
- * Minimal HTTP client port.
- * Default adapter wraps global fetch().
- * Swap for tests: return fixed HTML without touching the network.
- */
-export interface IHttpClient {
-	fetch(req: HttpRequest): Promise<HttpResponse>;
-}
-
-// ---------------------------------------------------------------------------
-// ICache<K, V>
-// ---------------------------------------------------------------------------
-
-/**
- * Generic cache port.
- * Default adapter: SpiderCache (LRU, TTL).
- * Swap for tests or production: in-memory Map, Redis, SQLite, etc.
- */
-export interface ICache<K, V> {
-	get(key: K): V | undefined;
-	set(key: K, value: V): void;
-	has(key: K): boolean;
-	delete(key: K): void;
-	/** All currently valid (non-expired) values. */
-	values(): V[];
-}
-
-// ---------------------------------------------------------------------------
-// IThrottle
-// ---------------------------------------------------------------------------
-
-/**
- * Per-domain request throttle port.
- * Default adapter: DomainThrottle (token bucket + exponential backoff).
- * Swap for tests: no-op implementation that always resolves immediately.
- */
-export interface IThrottle {
-	wait(url: string): Promise<void>;
-	success(url: string): void;
-	rateLimit(url: string, retryAfterHeader: string | null): number;
-	setDomainDelay(host: string, ms: number): void;
-	readonly maxRetries: number;
-}
-
-// ---------------------------------------------------------------------------
-// IRobotsChecker
-// ---------------------------------------------------------------------------
-
-export interface RobotsResult {
-	allowed: boolean;
-	crawlDelayMs?: number;
-}
-
-/**
- * robots.txt compliance port.
- * Default adapter: RobotsCache (fetches + parses per origin, 1h TTL).
- * Swap for tests: permissive stub that always returns { allowed: true }.
- */
-export interface IRobotsChecker {
-	check(url: string): Promise<RobotsResult>;
-}
-
-// ---------------------------------------------------------------------------
-// ISearchEngine
-// ---------------------------------------------------------------------------
-
-export interface SearchQuery {
-	query: string;
-	numResults?: number;
-	/**
-	 * Restrict results to content published within this window.
-	 * Supported by Tavily ("day"|"week"|"month"|"year") and Brave ("pd"|"pw"|"pm"|"py").
-	 * Adapters map this to their engine-specific parameter name.
-	 */
-	timeRange?: "day" | "week" | "month" | "year";
-	/**
-	 * Search topic mode. "news" prioritises freshly indexed news articles.
-	 * Supported by Tavily. Ignored by engines that don't support it.
-	 */
-	topic?: "news" | "general";
-}
-
-/**
- * A single result from a web search engine.
- * Defined here so port interfaces have no dependency on adapter modules.
- */
-export interface WebSearchResult {
-	url: string;
-	title: string;
-	/** Short description or snippet from the search engine. */
-	snippet: string;
-	/** ISO-8601 or human-readable date, if the engine returned one. */
-	publishedAt?: string;
-}
-
-/**
- * Web search engine port.
- * Adapters: BraveSearchEngine, TavilySearchEngine (in web-search.ts).
- * Swap for tests: stub returning fixed results.
- */
-export interface ISearchEngine {
-	search(req: SearchQuery): Promise<WebSearchResult[]>;
-}

package/src/robots.ts

@@ -1,121 +0,0 @@
-/**
- * Minimal robots.txt fetcher and per-domain cache.
- * Respects User-agent: * directives (Allow, Disallow, Crawl-delay).
- * Fails open — any fetch/parse error allows all URLs.
- */
-
-interface RobotsDirective {
-	allow: boolean;
-	path: string;
-}
-
-interface ParsedRobots {
-	directives: RobotsDirective[];
-	/** Crawl-delay in ms, if the robots.txt specified one (capped at 60s). */
-	crawlDelayMs?: number;
-}
-
-function parse(text: string): ParsedRobots {
-	const lines = text.split(/\r?\n/);
-	const directives: RobotsDirective[] = [];
-	let crawlDelayMs: number | undefined;
-	let inBlock = false;
-
-	for (const raw of lines) {
-		const line = raw.split("#")[0].trim();
-		if (!line) continue;
-
-		const colon = line.indexOf(":");
-		if (colon === -1) continue;
-		const key = line.slice(0, colon).trim().toLowerCase();
-		const value = line.slice(colon + 1).trim();
-
-		if (key === "user-agent") {
-			inBlock = value === "*";
-		} else if (inBlock) {
-			if (key === "disallow" && value) {
-				directives.push({ allow: false, path: value });
-			} else if (key === "allow" && value) {
-				directives.push({ allow: true, path: value });
-			} else if (key === "crawl-delay") {
-				const s = parseFloat(value);
-				if (!isNaN(s) && s > 0) crawlDelayMs = Math.min(s * 1_000, 60_000);
-			}
-		}
-	}
-
-	return { directives, crawlDelayMs };
-}
-
-function isAllowed(robots: ParsedRobots, path: string): boolean {
-	// Longest matching path prefix wins.
-	let best: RobotsDirective | undefined;
-	for (const d of robots.directives) {
-		if (path.startsWith(d.path)) {
-			if (!best || d.path.length > best.path.length) best = d;
-		}
-	}
-	return best?.allow ?? true; // default: allow
-}
-
-import type { IRobotsChecker, RobotsResult } from "./ports.js";
-
-const TTL_MS = 60 * 60 * 1_000; // 1 hour
-
-export class RobotsCache implements IRobotsChecker {
-	private readonly cache = new Map<string, { robots: ParsedRobots; expiresAt: number }>();
-	private readonly userAgent: string;
-
-	constructor(userAgent = "web-spider/0.1") {
-		this.userAgent = userAgent;
-	}
-
-	/**
-	 * Returns whether the URL is allowed and the crawl-delay if specified.
-	 * Caches per origin for 1 hour. Fails open on any error.
-	 */
-	async check(url: string): Promise<RobotsResult> {
-		const { origin, pathname } = new URL(url);
-		let entry = this.cache.get(origin);
-
-		if (!entry || Date.now() > entry.expiresAt) {
-			const robots = await this.fetchRobots(`${origin}/robots.txt`);
-			entry = { robots, expiresAt: Date.now() + TTL_MS };
-			this.cache.set(origin, entry);
-		}
-
-		return {
-			allowed: isAllowed(entry.robots, pathname),
-			crawlDelayMs: entry.robots.crawlDelayMs,
-		};
-	}
-
-	private async fetchRobots(robotsUrl: string): Promise<ParsedRobots> {
-		try {
-			const controller = new AbortController();
-			const timer = setTimeout(() => controller.abort(), 5_000);
-			let res: Response;
-			try {
-				res = await globalThis.fetch(robotsUrl, {
-					signal: controller.signal,
-					headers: { "User-Agent": this.userAgent },
-				});
-			} finally {
-				clearTimeout(timer);
-			}
-			if (!res.ok) return { directives: [] }; // 404 → allow all
-			return parse(await res.text());
-		} catch {
-			return { directives: [] }; // network error → fail open
-		}
-	}
-}
-
-/**
- * Factory — avoids jiti/Bun CJS re-export interop where class constructors
- * accessed through a re-export chain can appear undefined at call site.
- * Use this in extension code instead of `new RobotsCache()`.
- */
-export function createRobotsCache(userAgent?: string): RobotsCache {
-	return new RobotsCache(userAgent);
-}

package/src/search.ts

@@ -1,173 +0,0 @@
-import MiniSearch from "minisearch";
-import type { SpideredPage } from "./types.js";
-
-/** A single ranked match from fuzzySearch. */
-export interface SearchHit {
-	/** URL of the page the match came from. */
-	url: string;
-	/**
-	 * Stable chunk ID ("url#chunk-N") when the match is in body text.
-	 * Empty string when the match is in page metadata (title, description,
-	 * headings).
-	 */
-	chunkId: string;
-	/** Nearest heading for the matched chunk, or the matched field name for
-	 *  metadata hits (e.g. "title", "description"). */
-	heading: string;
-	/** Normalised score 0–1. Higher is a better match. */
-	score: number;
-	/** Short context window around the best match, ≤ 2×snippetRadius chars.
-	 *  Prefixed/suffixed with "…" when truncated. */
-	snippet: string;
-}
-
-export interface FuzzySearchOptions {
-	/** Maximum hits to return (default 10). */
-	topN?: number;
-	/**
-	 * Characters of context on each side of the match in the snippet
-	 * (default 100). Keep low to save tokens; raise when you need more context.
-	 */
-	snippetRadius?: number;
-}
-
-// ---------------------------------------------------------------------------
-// Internal types
-// ---------------------------------------------------------------------------
-
-interface SearchDoc {
-	/** Unique stable ID used by MiniSearch — chunk id or synthetic meta id. */
-	id: string;
-	url: string;
-	/** Nearest heading or metadata field name ("title", "description", "h2", …). */
-	heading: string;
-	/** The text that was indexed and will be searched. */
-	text: string;
-	/** Same as id for chunks; empty string for metadata docs. */
-	chunkId: string;
-}
-
-// ---------------------------------------------------------------------------
-// Snippet builder — kept from v1, MiniSearch doesn't generate snippets.
-// ---------------------------------------------------------------------------
-
-/**
- * Build a short snippet around the best match position.
- * Falls back to the start of the text when no match is found.
- */
-function buildSnippet(text: string, fullQuery: string, queryTokens: string[], radius: number): string {
-	const lower = text.toLowerCase();
-
-	let pos = lower.indexOf(fullQuery);
-	if (pos === -1) {
-		for (const qt of queryTokens) {
-			const p = lower.indexOf(qt);
-			if (p !== -1) {
-				pos = p;
-				break;
-			}
-		}
-	}
-	if (pos === -1) pos = 0;
-
-	const start = Math.max(0, pos - radius);
-	const end = Math.min(text.length, pos + Math.max(fullQuery.length, queryTokens[0]?.length ?? 1) + radius);
-	const raw = text.slice(start, end).replace(/\s+/g, " ").trim();
-	return (start > 0 ? "…" : "") + raw + (end < text.length ? "…" : "");
-}
-
-/** Tokenise and lower-case a string — used only for snippet generation. */
-function tokenise(s: string): string[] {
-	return s
-		.toLowerCase()
-		.split(/[\s\-_.,;:!?()[\]{}"'`/\\]+/)
-		.filter((t) => t.length > 1);
-}
-
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-
-/**
- * Full-text search across a set of already-spidered pages using MiniSearch
- * (BM25F ranking, fuzzy edit-distance, prefix search, heading field boost ×2).
- *
- * Searches both body chunks and page metadata (title, description, headings).
- * Returns results ranked by score descending, normalised to 0–1.
- *
- * Designed for agent use: call after fetching pages to locate a specific
- * fact, term, or section without dumping all content into context.
- *
- * @example
- * const hits = searchPages(pages, "cost optimization selectors", { topN: 5 })
- * // hits[0].snippet → "…LLM extraction vs Selectors…"
- */
-export function searchPages(pages: SpideredPage[], query: string, opts: FuzzySearchOptions = {}): SearchHit[] {
-	const { topN = 10, snippetRadius = 100 } = opts;
-
-	if (!query.trim()) return [];
-
-	// Build a flat document list — one entry per chunk, one per metadata field.
-	const docs: SearchDoc[] = [];
-
-	for (const page of pages) {
-		// Metadata documents
-		const metaDocs: Array<{ id: string; heading: string; text: string }> = [
-			{ id: `${page.url}#meta-title`, heading: "title", text: page.title },
-			...(page.description
-				? [{ id: `${page.url}#meta-description`, heading: "description", text: page.description }]
-				: []),
-			...page.headings.map((h, i) => ({
-				id: `${page.url}#meta-h${i}`,
-				heading: `h${h.level}`,
-				text: h.text,
-			})),
-		];
-		for (const m of metaDocs) {
-			docs.push({ id: m.id, url: page.url, heading: m.heading, text: m.text, chunkId: "" });
-		}
-
-		// Chunk documents
-		for (const c of page.chunks) {
-			docs.push({ id: c.id, url: page.url, heading: c.heading, text: c.text, chunkId: c.id });
-		}
-	}
-
-	if (docs.length === 0) return [];
-
-	const ms = new MiniSearch<SearchDoc>({
-		fields: ["text", "heading"],
-		storeFields: ["url", "heading", "chunkId", "text"],
-		searchOptions: {
-			// BM25F: headings are 2× more important than body text.
-			boost: { heading: 2 },
-			// Edit-distance fuzzy — 0.2 × term length, rounded (e.g. ≤1 for 5-char terms).
-			fuzzy: 0.2,
-			// Prefix match: "automat" finds "automation", "automated".
-			prefix: true,
-		},
-	});
-
-	ms.addAll(docs);
-
-	const results = ms.search(query);
-	if (results.length === 0) return [];
-
-	// Normalise raw BM25 scores to 0–1 by dividing by the top score.
-	// This preserves relative ranking while keeping values agent-friendly.
-	const maxRaw = results[0].score;
-
-	const fullQuery = query.trim().toLowerCase();
-	const queryTokens = tokenise(query);
-
-	return results.slice(0, topN).map((r) => ({
-		url: String(r["url"]),
-		chunkId: String(r["chunkId"]),
-		heading: String(r["heading"]),
-		score: Math.round(Math.min(r.score / maxRaw, 1) * 100) / 100,
-		snippet: buildSnippet(String(r["text"]), fullQuery, queryTokens, snippetRadius),
-	}));
-}
-
-/** @deprecated Use {@link searchPages} — renamed in v0.4.0 to reflect BM25F ranking. */
-export const fuzzySearch = searchPages

package/src/sitemap.ts

@@ -1,67 +0,0 @@
-/**
- * Sitemap fetcher and parser.
- *
- * Attempts /sitemap.xml and /sitemap_index.xml. Extracts <loc> URLs.
- * Fails open — any error returns an empty array so callers fall back
- * to normal BFS without noise.
- */
-
-import type { IHttpClient } from "./ports.js";
-
-/**
- * Fetch and parse sitemap URLs for the given origin.
- * Supports both standard sitemaps and sitemap index files.
- * Returns deduplicated absolute URLs, empty array on any failure.
- */
-export async function fetchSitemapUrls(
-	origin: string,
-	httpClient: IHttpClient,
-): Promise<string[]> {
-	const candidates = [`${origin}/sitemap.xml`, `${origin}/sitemap_index.xml`];
-	const urls = new Set<string>();
-
-	for (const sitemapUrl of candidates) {
-		try {
-			const res = await httpClient.fetch({
-				url: sitemapUrl,
-				headers: { Accept: "application/xml, text/xml, */*" },
-			});
-			if (!res.ok) continue;
-			const xml = await res.text();
-			for (const loc of extractLocs(xml)) {
-				// Sitemap index entries point to other sitemaps — fetch those too
-				if (loc.endsWith(".xml")) {
-					const nested = await fetchSitemapXml(loc, httpClient);
-					for (const u of nested) urls.add(u);
-				} else {
-					urls.add(loc);
-				}
-			}
-			if (urls.size > 0) break; // found a working sitemap
-		} catch {
-			continue;
-		}
-	}
-
-	return [...urls];
-}
-
-async function fetchSitemapXml(url: string, httpClient: IHttpClient): Promise<string[]> {
-	try {
-		const res = await httpClient.fetch({ url });
-		if (!res.ok) return [];
-		return extractLocs(await res.text());
-	} catch {
-		return [];
-	}
-}
-
-function extractLocs(xml: string): string[] {
-	const urls: string[] = [];
-	const re = /<loc>\s*(https?:\/\/[^<\s]+)\s*<\/loc>/gi;
-	let match: RegExpExecArray | null;
-	while ((match = re.exec(xml)) !== null) {
-		urls.push(match[1].trim());
-	}
-	return urls;
-}