@dpopsuev/web-spider 0.10.4 → 0.10.5

package/src/spider.ts

@@ -1,475 +0,0 @@
-import { Readability } from "@mozilla/readability";
-import { chunk, toMarkdown } from "./convert.js";
-import type { ImageRef } from "./types.js";
-import { extractCanonicalUrl, extractHeadings, extractLinks, extractTags, parseDom } from "./parse.js";
-import type { IHttpClient, IRobotsChecker, IThrottle } from "./ports.js";
-import { buildTree } from "./tree.js";
-import type { DOMNode, LeanPage, SpideredPage } from "./types.js";
-import { toLean } from "./views.js";
-
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-
-const WORDS_PER_MINUTE = 200;
-
-// ---------------------------------------------------------------------------
-// Default HTTP client adapter
-// ---------------------------------------------------------------------------
-
-const defaultHttpClient: IHttpClient = {
-	async fetch(req) {
-		const res = await globalThis.fetch(req.url, {
-			signal: req.signal,
-			headers: req.headers,
-		});
-		return {
-			ok: res.ok,
-			status: res.status,
-			statusText: res.statusText,
-			headers: { get: (name: string) => res.headers.get(name) },
-			text: () => res.text(),
-			arrayBuffer: () => res.arrayBuffer(),
-		};
-	},
-};
-
-
-
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-
-export interface SpiderOptions {
-	/**
-	 * ms before aborting the fetch (default 10 000).
-	 */
-	timeoutMs?: number;
-	/**
-	 * Value sent as User-Agent.
-	 * Default identifies the tool; override for sites that block generic crawlers.
-	 */
-	userAgent?: string;
-	/**
-	 * CSS selector that scopes content extraction to a specific element.
-	 * Everything outside the matched element is discarded before Readability runs.
-	 * Example: "article", ".main-content", "#post-body"
-	 */
-	rootSelector?: string;
-	/**
-	 * Comma-separated CSS selectors whose matched elements are removed before
-	 * extraction. Applied before Readability, so excluded content never reaches
-	 * the chunks or markdown.
-	 * Example: "nav, footer, .sidebar, #ads"
-	 */
-	excludeSelectors?: string;
-	/**
-	 * Approximate maximum token budget for the returned content.
-	 * Markdown is truncated to fit. Rough estimate: 1 token ≈ 4 characters.
-	 * Does not affect lean view (headings/links are always small).
-	 * Default: unlimited.
-	 */
-	tokenBudget?: number;
-	/**
-	 * Per-domain throttle — shared across spider() calls to enforce rate limits
-	 * and exponential backoff on 429/503 responses.
-	 */
-	throttle?: IThrottle;
-	/**
-	 * robots.txt checker — when provided, spider() checks robots.txt before
-	 * fetching and respects Crawl-delay directives.
-	 */
-	robotsCache?: IRobotsChecker;
-	/**
-	 * HTTP client — defaults to a global fetch() adapter.
-	 * Inject a stub for testing without real network access.
-	 */
-	httpClient?: IHttpClient;
-	/**
-	 * When true, fetch <img> src URLs found in the article content and attach
-	 * them as base64-encoded ImageRef objects to SpideredPage.images.
-	 * Default: false — preserves current behaviour exactly.
-	 */
-	captureImages?: boolean;
-	/**
-	 * Maximum number of images to fetch per page.
-	 * Default: 10.
-	 */
-	maxImages?: number;
-}
-
-/**
- * Spider a single URL and return a fully structured SpideredPage.
- *
- * Pass `view: "lean"` to skip chunking and markdown conversion — returns a
- * LeanPage with only identity, metadata, and the heading/link outline.
- * Significantly faster (~3×) and uses far fewer tokens in agent context.
- *
- * Errors are returned as thrown exceptions with a descriptive message rather
- * than crashing silently. Common cases:
- * - Non-HTTP URLs throw immediately with a clear message.
- * - HTTP errors include the status code.
- * - JS-rendered pages (wordCount === 0) include a hint.
- * - Timeouts include the configured limit.
- *
- * @example
- * // Full page — chunks, markdown, all metadata
- * const page = await spider("https://example.com")
- *
- * @example
- * // Lean overview — no body text, ideal for navigation decisions
- * const lean = await spider("https://example.com", { view: "lean" })
- */
-// ---------------------------------------------------------------------------
-// Image fetching
-// ---------------------------------------------------------------------------
-
-/** Detect MIME type from a URL path extension, defaulting to image/jpeg. */
-function mimeFromUrl(src: string): string {
-	const ext = src.split("?")[0].split(".").pop()?.toLowerCase();
-	const map: Record<string, string> = {
-		jpg: "image/jpeg",
-		jpeg: "image/jpeg",
-		png: "image/png",
-		webp: "image/webp",
-		gif: "image/gif",
-		svg: "image/svg+xml",
-		avif: "image/avif",
-	};
-	return map[ext ?? ""] ?? "image/jpeg";
-}
-
-/**
- * Extract <img> elements from article HTML, resolve src URLs, and fetch
- * each as a base64-encoded ImageRef. data: URLs are included without fetching.
- * Failed fetches are silently skipped.
- */
-async function fetchImages(
-	articleHtml: string,
-	pageUrl: string,
-	httpClient: IHttpClient,
-	maxImages: number,
-	throttle?: IThrottle,
-): Promise<ImageRef[]> {
-	// Parse the article HTML to extract img elements.
-	const { parseDom } = await import("./parse.js");
-	const doc = parseDom(articleHtml, pageUrl);
-	const imgEls = [...doc.querySelectorAll("img")].slice(0, maxImages);
-
-	const results: ImageRef[] = [];
-
-	for (const el of imgEls) {
-		const rawSrc = el.getAttribute("src") ?? "";
-		if (!rawSrc) continue;
-
-		const alt = el.getAttribute("alt") ?? "";
-
-		// data: URLs — include without fetching.
-		if (rawSrc.startsWith("data:")) {
-			const match = /^data:([^;]+);base64,(.+)$/.exec(rawSrc);
-			if (match) {
-				results.push({ src: rawSrc, mimeType: match[1], alt, base64: match[2] });
-			}
-			continue;
-		}
-
-		// Resolve relative URLs.
-		let absoluteSrc: string;
-		try {
-			absoluteSrc = new URL(rawSrc, pageUrl).toString();
-		} catch {
-			continue;
-		}
-
-		try {
-			if (throttle) await throttle.wait(absoluteSrc);
-			const res = await httpClient.fetch({
-				url: absoluteSrc,
-				headers: { "User-Agent": "web-spider/0.1", Accept: "image/*" },
-			});
-			if (!res.ok) continue;
-			throttle?.success(absoluteSrc);
-
-			const buf = await res.arrayBuffer();
-			const base64 = Buffer.from(buf).toString("base64");
-			const contentType = res.headers.get("content-type");
-			const mimeType = contentType?.split(";")[0].trim() || mimeFromUrl(absoluteSrc);
-
-			results.push({ src: absoluteSrc, mimeType, alt, base64 });
-		} catch {
-			// Skip failed image fetches silently — a missing image should never
-			// cause the whole page scrape to fail.
-		}
-	}
-
-	return results;
-}
-
-/** A page with its full DOM tree attached. */
-export interface TreePage extends SpideredPage {
-	readonly view: "tree";
-	tree: DOMNode;
-}
-
-export async function spider(url: string, opts: SpiderOptions & { view: "lean" }): Promise<LeanPage>;
-export async function spider(url: string, opts: SpiderOptions & { view: "tree" }): Promise<TreePage>;
-export async function spider(url: string, opts?: SpiderOptions & { view?: "full" }): Promise<SpideredPage>;
-export async function spider(
-	url: string,
-	opts?: SpiderOptions & { view?: "lean" | "full" | "tree" },
-): Promise<SpideredPage | LeanPage | TreePage> {
-	const {
-		timeoutMs = 30_000,
-		userAgent = "web-spider/0.1 (AI agent research tool; +https://github.com/dpopsuev)",
-		view = "full",
-		rootSelector,
-		excludeSelectors,
-		tokenBudget,
-		throttle,
-		robotsCache,
-		httpClient = defaultHttpClient,
-		captureImages = false,
-		maxImages = 10,
-	} = opts ?? {};
-
-	// Poka-yoke: reject non-HTTP URLs immediately with a clear message.
-	let parsedUrl: URL;
-	try {
-		parsedUrl = new URL(url);
-	} catch {
-		throw new Error(`Invalid URL: "${url}" — must be a fully-qualified http/https URL`);
-	}
-	if (!["http:", "https:"].includes(parsedUrl.protocol)) {
-		throw new Error(`Unsupported protocol "${parsedUrl.protocol}" — only http and https are supported`);
-	}
-
-	// Check robots.txt before fetching.
-	if (robotsCache) {
-		const { allowed, crawlDelayMs } = await robotsCache.check(url);
-		if (!allowed) throw new Error(`Blocked by robots.txt: ${url}`);
-		if (crawlDelayMs && throttle) {
-			throttle.setDomainDelay(parsedUrl.hostname, crawlDelayMs);
-		}
-	}
-
-	// Fetch with optional throttle + retry on 429/503.
-	const maxRetries = throttle?.maxRetries ?? 0;
-	let html = "";
-	let fetchError: Error | null = null;
-
-	for (let attempt = 0; attempt <= maxRetries; attempt++) {
-		if (throttle) await throttle.wait(url);
-
-		const controller = new AbortController();
-		const timer = setTimeout(() => controller.abort(), timeoutMs);
-		let res: Awaited<ReturnType<IHttpClient["fetch"]>>;
-		try {
-			res = await httpClient.fetch({
-				url,
-				signal: controller.signal,
-				headers: { "User-Agent": userAgent, Accept: "text/html" },
-			});
-		} catch (err) {
-			clearTimeout(timer);
-			if (err instanceof Error && err.name === "AbortError") {
-				throw new Error(`Timeout after ${timeoutMs}ms — ${url}`);
-			}
-			throw err;
-		}
-		clearTimeout(timer);
-
-		if (res.status === 429 || res.status === 503) {
-			if (throttle && attempt < maxRetries) {
-				throttle.rateLimit(url, res.headers.get("Retry-After"));
-				fetchError = new Error(`HTTP ${res.status} — retrying (attempt ${attempt + 1}/${maxRetries})`);
-				continue;
-			}
-			throw new Error(`HTTP ${res.status} ${res.statusText} — ${url}`);
-		}
-
-		if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText} — ${url}`);
-
-		throttle?.success(url);
-		html = await res.text();
-		fetchError = null;
-		break;
-	}
-
-	if (fetchError) throw fetchError;
-
-	// Parse DOM via parse.ts — keeps the JSDOM dependency in one module.
-	const doc = parseDom(html, url);
-
-	// Apply excludeSelectors before Readability strips the DOM.
-	if (excludeSelectors) {
-		for (const sel of excludeSelectors
-			.split(",")
-			.map((s) => s.trim())
-			.filter(Boolean)) {
-			for (const el of [...doc.querySelectorAll(sel)]) el.remove();
-		}
-	}
-
-	// Scope to rootSelector: replace body content with the matched element.
-	if (rootSelector) {
-		const root = doc.querySelector(rootSelector);
-		if (root) {
-			doc.body.innerHTML = root.outerHTML;
-		}
-	}
-
-	const links = extractLinks(doc, url);
-	const canonicalUrl = extractCanonicalUrl(doc, url);
-
-	// Readability content extraction (Firefox Reader View engine).
-	const readabilityResult = new Readability(doc).parse();
-	const jsRendered = !readabilityResult;
-	// Graceful degradation: if Readability finds nothing, return a partial page
-	// with jsRendered:true rather than throwing. The agent can decide what to do.
-	const article = readabilityResult ?? {
-		title: (doc.querySelector("title")?.textContent ?? "").trim(),
-		content: "",
-		textContent: "",
-		length: 0,
-		excerpt: "",
-		byline: "",
-		dir: "",
-		site_name: "",
-		lang: "",
-		publishedTime: null,
-		readingTimeMinutes: 0,
-	};
-
-	const domain = new URL(url).hostname.replace(/^www\./, "");
-	const fetchedAt = new Date().toISOString();
-
-	const meta = (name: string): string => {
-		const el =
-			doc.querySelector(`meta[name="${name}"]`) ??
-			doc.querySelector(`meta[property="og:${name}"]`) ??
-			doc.querySelector(`meta[property="${name}"]`);
-		return (el?.getAttribute("content") ?? "").trim();
-	};
-
-	// headings must come before tags so the heading fallback is available.
-	const headings = extractHeadings(article.content ?? "");
-	const tags = extractTags(doc);
-
-	// ---------------------------------------------------------------------------
-	// Lean fast-path — skip turndown + chunking entirely
-	// ---------------------------------------------------------------------------
-	if (view === "lean") {
-		const textContent = (article.textContent ?? "").trim();
-		const wordCount = textContent.split(/\s+/).filter(Boolean).length;
-		const chunkCount = Math.max(0, Math.floor(wordCount / 150));
-
-		const full = {
-			url,
-			domain,
-			fetchedAt,
-			...(canonicalUrl !== undefined ? { canonicalUrl } : {}),
-			title: article.title ?? meta("title"),
-			description: meta("description"),
-			author: article.byline ?? meta("author"),
-			publishedAt: meta("article:published_time") ?? meta("date"),
-			lang: doc.documentElement.lang ?? "en",
-			tags,
-			wordCount,
-			readingTimeMinutes: Math.ceil(wordCount / WORDS_PER_MINUTE),
-			chunks: [], // placeholder — toLean reads chunks.length
-			headings,
-			links,
-			markdown: "",
-		} satisfies SpideredPage;
-		const lean = toLean(full);
-		return { ...lean, chunkCount, ...(jsRendered ? { jsRendered: true } : {}) };
-	}
-
-	// ---------------------------------------------------------------------------
-	// Tree path — build semantic DOM tree, then also produce full markdown
-	// ---------------------------------------------------------------------------
-	if (view === "tree") {
-		const tree = buildTree(article.content ?? "", url);
-		const markdown = toMarkdown(article.content ?? "", { keepImages: captureImages });
-		const wordCount = markdown.split(/\s+/).filter(Boolean).length;
-		const chunks = chunk(markdown, url);
-		const images = captureImages
-			? await fetchImages(article.content ?? "", url, httpClient, maxImages, throttle)
-			: undefined;
-		return {
-			view: "tree",
-			url,
-			domain,
-			fetchedAt,
-			...(canonicalUrl !== undefined ? { canonicalUrl } : {}),
-			title: article.title ?? meta("title"),
-			description: meta("description"),
-			author: article.byline ?? meta("author"),
-			publishedAt: meta("article:published_time") ?? meta("date"),
-			lang: doc.documentElement.lang ?? "en",
-			tags,
-			wordCount,
-			readingTimeMinutes: Math.ceil(wordCount / WORDS_PER_MINUTE),
-			headings,
-			chunks,
-			links,
-			markdown,
-			tree,
-			...(images ? { images } : {}),
-		};
-	}
-
-	// ---------------------------------------------------------------------------
-	// Full path — turndown + chunk
-	// ---------------------------------------------------------------------------
-	const markdown = toMarkdown(article.content ?? "", { keepImages: captureImages });
-	const wordCount = markdown.split(/\s+/).filter(Boolean).length;
-
-	// Chunk-aware tokenBudget: select whole chunks up to the budget rather
-	// than slicing markdown mid-sentence. Preserves chunk boundaries and
-	// returns the richest complete content that fits.
-	let allChunks = chunk(markdown, url);
-	if (tokenBudget !== undefined) {
-		const charBudget = tokenBudget * 4;
-		let remaining = charBudget;
-		let first = true;
-		allChunks = allChunks.filter((c) => {
-			// Always include at least the first chunk — agents need something
-			// even if it exceeds the budget.
-			if (!first && remaining <= 0) return false;
-			first = false;
-			remaining -= c.text.length;
-			return true;
-		});
-	}
-
-	// Reconstruct markdown from selected chunks for full-page consumers.
-	const finalMarkdown = tokenBudget !== undefined
-		? allChunks.map((c) => c.text).join("\n\n")
-		: markdown;
-
-	const images = captureImages
-		? await fetchImages(article.content ?? "", url, httpClient, maxImages, throttle)
-		: undefined;
-
-	return {
-		url,
-		domain,
-		fetchedAt,
-		...(canonicalUrl !== undefined ? { canonicalUrl } : {}),
-		title: article.title ?? meta("title"),
-		description: meta("description"),
-		author: article.byline ?? meta("author"),
-		publishedAt: meta("article:published_time") ?? meta("date"),
-		lang: doc.documentElement.lang ?? "en",
-		tags,
-		wordCount,
-		readingTimeMinutes: Math.ceil(wordCount / WORDS_PER_MINUTE),
-		headings,
-		chunks: allChunks,
-		links,
-		markdown: finalMarkdown,
-		...(images ? { images } : {}),
-		...(jsRendered ? { jsRendered: true } : {}),
-	};
-}

package/src/throttle.ts

@@ -1,118 +0,0 @@
-/**
- * Per-domain request throttle with exponential backoff and jitter.
- *
- * Enforces a minimum gap between requests to the same hostname.
- * On 429/503, backs off exponentially and respects Retry-After headers.
- * Shared instances should be passed into spider() and crawl() so that
- * all requests to a domain coordinate through one rate limiter.
- */
-
-import type { IThrottle } from "./ports.js";
-
-export interface ThrottleOptions {
-	/** Minimum gap between requests to the same domain (ms). Default 500. */
-	minDelayMs?: number;
-	/** Base for exponential backoff (ms). Default 1000. */
-	backoffBaseMs?: number;
-	/** Maximum backoff delay (ms). Default 30 000. */
-	backoffCapMs?: number;
-	/** Maximum retry attempts on 429/503 before giving up. Default 3. */
-	maxRetries?: number;
-}
-
-interface DomainState {
-	lastAt: number;
-	backoffUntil: number;
-	errors: number;
-	/** Per-domain minimum delay override (e.g. from robots.txt Crawl-delay). */
-	minDelayMs?: number;
-}
-
-function sleep(ms: number): Promise<void> {
-	return new Promise((r) => setTimeout(r, ms));
-}
-
-function parseRetryAfter(header: string | null): number {
-	if (!header) return 0;
-	const seconds = parseInt(header, 10);
-	if (!isNaN(seconds)) return seconds * 1_000;
-	const date = new Date(header).getTime();
-	if (!isNaN(date)) return Math.max(0, date - Date.now());
-	return 0;
-}
-
-export class DomainThrottle implements IThrottle {
-	private readonly states = new Map<string, DomainState>();
-	readonly minDelayMs: number;
-	readonly backoffBaseMs: number;
-	readonly backoffCapMs: number;
-	readonly maxRetries: number;
-
-	constructor(opts: ThrottleOptions = {}) {
-		this.minDelayMs = opts.minDelayMs ?? 500;
-		this.backoffBaseMs = opts.backoffBaseMs ?? 1_000;
-		this.backoffCapMs = opts.backoffCapMs ?? 30_000;
-		this.maxRetries = opts.maxRetries ?? 3;
-	}
-
-	private state(host: string): DomainState {
-		let s = this.states.get(host);
-		if (!s) {
-			s = { lastAt: 0, backoffUntil: 0, errors: 0 };
-			this.states.set(host, s);
-		}
-		return s;
-	}
-
-	/** Wait until the domain's rate limit and backoff have cleared. */
-	async wait(url: string): Promise<void> {
-		const s = this.state(new URL(url).hostname);
-		const minDelay = s.minDelayMs ?? this.minDelayMs;
-		const now = Date.now();
-		const delay = Math.max(
-			Math.max(0, s.backoffUntil - now),
-			Math.max(0, s.lastAt + minDelay - now),
-		);
-		if (delay > 0) await sleep(delay);
-		s.lastAt = Date.now();
-	}
-
-	/** Record a successful request — resets backoff for the domain. */
-	success(url: string): void {
-		const s = this.state(new URL(url).hostname);
-		s.errors = 0;
-		s.backoffUntil = 0;
-	}
-
-	/**
-	 * Record a rate-limit hit. Applies exponential backoff with jitter,
-	 * using Retry-After header when present. Returns the wait duration in ms.
-	 */
-	rateLimit(url: string, retryAfterHeader: string | null): number {
-		const s = this.state(new URL(url).hostname);
-		s.errors++;
-		const retryAfterMs = parseRetryAfter(retryAfterHeader);
-		const jitter = Math.random() * this.backoffBaseMs;
-		const backoffMs = Math.min(this.backoffCapMs, this.backoffBaseMs * 2 ** (s.errors - 1) + jitter);
-		const waitMs = Math.max(retryAfterMs, backoffMs);
-		s.backoffUntil = Date.now() + waitMs;
-		return waitMs;
-	}
-
-	/**
-	 * Override the minimum delay for a specific domain.
-	 * Used to honour robots.txt Crawl-delay directives.
-	 */
-	setDomainDelay(host: string, ms: number): void {
-		this.state(host).minDelayMs = ms;
-	}
-}
-
-/**
- * Factory — avoids jiti/Bun CJS re-export interop where class constructors
- * accessed through a re-export chain can appear undefined at call site.
- * Use this in extension code instead of `new DomainThrottle()`.
- */
-export function createThrottle(opts?: ThrottleOptions): DomainThrottle {
-	return new DomainThrottle(opts);
-}