@dpopsuev/web-spider 0.10.4

package/test/images-integration.test.ts

@@ -0,0 +1,359 @@
+/**
+ * End-to-end captureImages integration tests.
+ *
+ * Covers the full pipeline:
+ *   spider() → SpideredPage.images → DiskCache.flush() → DiskCache.get()
+ *   → LLM data URL → PlaywrightHttpClient-shaped stub
+ *
+ * No real network, no real browser.
+ */
+
+import { mkdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { DiskCache } from "../src/disk-cache.js";
+import { PlaywrightHttpClient } from "../src/playwright.js";
+import type { IHttpClient } from "../src/ports.js";
+import { spider } from "../src/spider.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+const FIXTURE_HTML = readFileSync(
+	join(import.meta.dirname, "../fixtures/article-with-images.html"),
+	"utf8",
+);
+const SMALL_JPG = readFileSync(join(import.meta.dirname, "../fixtures/images/small.jpg"));
+const TINY_PNG = readFileSync(join(import.meta.dirname, "../fixtures/images/tiny.png"));
+const LARGE_JPG = readFileSync(join(import.meta.dirname, "../fixtures/images/large.jpg"));
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+let testDir: string;
+
+beforeEach(() => {
+	testDir = join(tmpdir(), `wbs-integration-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+	mkdirSync(testDir, { recursive: true });
+});
+
+afterEach(() => {
+	rmSync(testDir, { recursive: true, force: true });
+});
+
+function makeCachePath() {
+	return join(testDir, "pages.json");
+}
+
+/**
+ * Stub client that serves the fixture HTML for page fetches and
+ * appropriate fixture image bytes for image fetches.
+ * `useLargeImages`: serve large.jpg (>32KB) to exercise disk-spill.
+ */
+function makeStubClient(opts: { useLargeImages?: boolean } = {}): IHttpClient {
+	return {
+		async fetch(req) {
+			const isImageReq = (req.headers?.["Accept"] ?? "").startsWith("image/");
+
+			if (!isImageReq) {
+				// Page fetch
+				return {
+					ok: true,
+					status: 200,
+					statusText: "OK",
+					headers: { get: (n) => (n === "content-type" ? "text/html" : null) },
+					text: async () => FIXTURE_HTML,
+					arrayBuffer: async () => new ArrayBuffer(0),
+				};
+			}
+
+			// Image fetch — pick fixture based on src extension / useLargeImages flag
+			const src = req.url;
+			let bytes: Buffer;
+			let mime: string;
+
+			if (opts.useLargeImages) {
+				bytes = LARGE_JPG;
+				mime = "image/jpeg";
+			} else if (src.match(/\.png(\?|$)/i)) {
+				bytes = TINY_PNG;
+				mime = "image/png";
+			} else {
+				bytes = SMALL_JPG;
+				mime = "image/jpeg";
+			}
+
+			const buf = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
+			return {
+				ok: true,
+				status: 200,
+				statusText: "OK",
+				headers: { get: (n) => (n === "content-type" ? mime : null) },
+				text: async () => "",
+				arrayBuffer: async () => buf,
+			};
+		},
+	};
+}
+
+// ---------------------------------------------------------------------------
+// 1. Full spider() → DiskCache → reload roundtrip (small images, inline)
+// ---------------------------------------------------------------------------
+
+describe("spider() → DiskCache roundtrip — small images (inline)", () => {
+	it("images survive flush + reload with correct base64", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+			captureImages: true,
+		});
+
+		expect(page.images).toBeDefined();
+		expect(page.images!.length).toBeGreaterThan(0);
+
+		const cachePath = makeCachePath();
+		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		const reloaded = cache2.get("https://example.com");
+
+		expect(reloaded).toBeDefined();
+		expect(reloaded!.images).toBeDefined();
+		expect(reloaded!.images!.length).toBe(page.images!.length);
+
+		// Every base64 must survive the roundtrip exactly
+		for (let i = 0; i < page.images!.length; i++) {
+			const orig = page.images![i];
+			const loaded = reloaded!.images![i];
+			expect(loaded.src).toBe(orig.src);
+			expect(loaded.mimeType).toBe(orig.mimeType);
+			expect(loaded.alt).toBe(orig.alt);
+			if (orig.base64) expect(loaded.base64).toBe(orig.base64);
+		}
+	});
+
+	it("page text (markdown, chunks, title) also survives the roundtrip", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+			captureImages: true,
+		});
+
+		const cachePath = makeCachePath();
+		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		const reloaded = cache2.get("https://example.com");
+
+		expect(reloaded!.title).toBe(page.title);
+		expect(reloaded!.markdown).toBe(page.markdown);
+		expect(reloaded!.chunks.length).toBe(page.chunks.length);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// 2. Full spider() → DiskCache → reload roundtrip (large images, disk-spill)
+// ---------------------------------------------------------------------------
+
+describe("spider() → DiskCache roundtrip — large images (disk-spill)", () => {
+	it("large images are spilled to disk and hydrated on reload", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient({ useLargeImages: true }),
+			captureImages: true,
+			maxImages: 3,
+		});
+
+		expect(page.images).toBeDefined();
+		const cachePath = makeCachePath();
+
+		// Use a low threshold so even SMALL_JPG spills — 100 bytes decoded
+		const cache1 = new DiskCache(cachePath, {
+			ttlMs: 60 * 60 * 1000,
+			autoFlush: false,
+			inlineImageThreshold: 100,
+		});
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		const cache2 = new DiskCache(cachePath, {
+			ttlMs: 60 * 60 * 1000,
+			autoFlush: false,
+			inlineImageThreshold: 100,
+		});
+		const reloaded = cache2.get("https://example.com");
+
+		expect(reloaded!.images).toBeDefined();
+		// All images must have base64 after hydration
+		for (const img of reloaded!.images!) {
+			if (img.filePath) {
+				expect(img.base64).toBeDefined();
+				expect(img.base64!.length).toBeGreaterThan(0);
+			}
+		}
+	});
+});
+
+// ---------------------------------------------------------------------------
+// 3. LLM wire format — every image produces a valid data URL
+// ---------------------------------------------------------------------------
+
+describe("LLM wire format", () => {
+	it("every captured image yields a valid data: URL", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+			captureImages: true,
+		});
+
+		expect(page.images!.length).toBeGreaterThan(0);
+
+		for (const img of page.images!) {
+			if (!img.base64) continue;
+			const dataUrl = `data:${img.mimeType};base64,${img.base64}`;
+			expect(dataUrl).toMatch(/^data:image\/(jpeg|png|webp|gif|svg\+xml|avif);base64,/);
+		}
+	});
+
+	it("base64 in data URL decodes to non-empty binary", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+			captureImages: true,
+		});
+
+		for (const img of page.images!) {
+			if (!img.base64) continue;
+			const decoded = Buffer.from(img.base64, "base64");
+			expect(decoded.byteLength).toBeGreaterThan(0);
+		}
+	});
+
+	it("data: URL images from fixture have correct inline base64", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+			captureImages: true,
+		});
+
+		// The fixture contains one data: URL (1x1 PNG)
+		const inlineImg = page.images!.find((i) => i.src.startsWith("data:"));
+		expect(inlineImg).toBeDefined();
+		expect(inlineImg!.mimeType).toBe("image/png");
+
+		const dataUrl = `data:${inlineImg!.mimeType};base64,${inlineImg!.base64}`;
+		expect(dataUrl).toMatch(/^data:image\/png;base64,/);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// 4. captureImages: false — no images attached, cache roundtrip clean
+// ---------------------------------------------------------------------------
+
+describe("captureImages: false — clean roundtrip", () => {
+	it("images field is absent on spider() result", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+		});
+		expect(page.images).toBeUndefined();
+	});
+
+	it("cache roundtrip without images is clean", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+		});
+
+		const cachePath = makeCachePath();
+		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		const reloaded = cache2.get("https://example.com");
+		expect(reloaded!.images).toBeUndefined();
+	});
+});
+
+// ---------------------------------------------------------------------------
+// 5. PlaywrightHttpClient-shaped stub with captureImages
+// ---------------------------------------------------------------------------
+
+describe("PlaywrightHttpClient captureImages integration", () => {
+	it("PlaywrightHttpClient constructs with captureImages:true and satisfies IHttpClient", () => {
+		const client: IHttpClient = new PlaywrightHttpClient({ captureImages: true });
+		expect(typeof client.fetch).toBe("function");
+	});
+
+	it("spider() with a Playwright-shaped stub and captureImages:true returns images", async () => {
+		// Simulate what PlaywrightHttpClient would do: a stub that looks like
+		// a Playwright client — returns HTML for page fetches, images for image fetches.
+		const playwrightShapedStub: IHttpClient = {
+			async fetch(req) {
+				const isImageReq = (req.headers?.["Accept"] ?? "").startsWith("image/");
+				if (!isImageReq) {
+					return {
+						ok: true,
+						status: 200,
+						statusText: "OK",
+						headers: { get: (n) => (n === "content-type" ? "text/html" : null) },
+						text: async () => FIXTURE_HTML,
+						arrayBuffer: async () => new ArrayBuffer(0),
+					};
+				}
+				const buf = SMALL_JPG.buffer.slice(
+					SMALL_JPG.byteOffset,
+					SMALL_JPG.byteOffset + SMALL_JPG.byteLength,
+				) as ArrayBuffer;
+				return {
+					ok: true,
+					status: 200,
+					statusText: "OK",
+					headers: { get: (n) => (n === "content-type" ? "image/jpeg" : null) },
+					text: async () => "",
+					arrayBuffer: async () => buf,
+				};
+			},
+		};
+
+		const page = await spider("https://example.com", {
+			httpClient: playwrightShapedStub,
+			captureImages: true,
+		});
+
+		expect(page.images).toBeDefined();
+		expect(page.images!.length).toBeGreaterThan(0);
+
+		for (const img of page.images!) {
+			if (img.base64) {
+				expect(`data:${img.mimeType};base64,${img.base64}`).toMatch(/^data:image\//);
+			}
+		}
+	});
+});
+
+// ---------------------------------------------------------------------------
+// 6. maxImages respected end-to-end through cache
+// ---------------------------------------------------------------------------
+
+describe("maxImages end-to-end", () => {
+	it("maxImages:2 — only 2 images in cache after roundtrip", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: makeStubClient(),
+			captureImages: true,
+			maxImages: 2,
+		});
+
+		expect(page.images!.length).toBeLessThanOrEqual(2);
+
+		const cachePath = makeCachePath();
+		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
+		const reloaded = cache2.get("https://example.com");
+		expect(reloaded!.images!.length).toBeLessThanOrEqual(2);
+	});
+});

package/test/improvements.test.ts

@@ -0,0 +1,279 @@
+/**
+ * Tests for improvement tasks: JS degradation, chunk tokenBudget, sitemap discovery, disk cache.
+ * Written before implementation — all should fail until code is in place.
+ */
+
+import { describe, expect, it } from "vitest";
+import { spider } from "../src/spider.js";
+import { crawl } from "../src/crawl.js";
+import type { IHttpClient } from "../src/ports.js";
+
+// ---------------------------------------------------------------------------
+// Shared helpers
+// ---------------------------------------------------------------------------
+
+function mockClient(responses: Record<string, { status?: number; body: string }>): IHttpClient {
+	return {
+		fetch: async (req) => {
+			const entry = responses[req.url] ?? responses["*"];
+			if (!entry) throw new Error(`Unexpected fetch: ${req.url}`);
+			const status = entry.status ?? 200;
+			return {
+				ok: status >= 200 && status < 300,
+				status,
+				statusText: status === 200 ? "OK" : "Error",
+				headers: { get: () => null },
+				text: async () => entry.body,
+				arrayBuffer: async () => new ArrayBuffer(0),
+			};
+		},
+	};
+}
+
+const articleHtml = (title: string, body: string) => `<!DOCTYPE html>
+<html lang="en">
+<head><title>${title}</title><meta name="description" content="test"></head>
+<body><article><h1>${title}</h1>${body}</article></body>
+</html>`;
+
+const LONG_BODY = `<p>${"Word ".repeat(300)}</p><h2>Section</h2><p>${"More words. ".repeat(300)}</p>`;
+
+// ---------------------------------------------------------------------------
+// Graceful degradation on JS-rendered pages
+// ---------------------------------------------------------------------------
+
+describe("JS-rendered pages degrade gracefully", () => {
+	const jsHtml = `<!DOCTYPE html><html><head><title>App</title></head>
+<body><div id="root"></div><script>/* SPA */</script></body></html>`;
+
+	it("returns a page with jsRendered:true instead of throwing", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: jsHtml } }),
+		});
+		expect((page as { jsRendered?: boolean }).jsRendered).toBe(true);
+	});
+
+	it("still returns title and links from JS page", async () => {
+		const html = `<!DOCTYPE html><html><head><title>My SPA</title></head>
+<body><div id="root"></div><a href="/about">About</a></body></html>`;
+		const page = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: html } }),
+		});
+		expect(page.title).toContain("My SPA");
+		expect(page.links.length).toBeGreaterThan(0);
+	});
+
+	it("returns empty chunks and markdown for JS page", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: jsHtml } }),
+		});
+		expect(page.chunks).toHaveLength(0);
+		expect(page.markdown).toBe("");
+	});
+
+	it("lean view also degrades instead of throwing", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: jsHtml } }),
+			view: "lean",
+		});
+		expect((page as { jsRendered?: boolean }).jsRendered).toBe(true);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Chunk-aware tokenBudget
+// ---------------------------------------------------------------------------
+
+describe("chunk-aware tokenBudget", () => {
+	it("returns whole chunks up to budget, not truncated mid-sentence", async () => {
+		const page = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
+			tokenBudget: 100,
+		});
+		// Should have at least one complete chunk
+		expect(page.chunks.length).toBeGreaterThan(0);
+		// Markdown should not end with truncation notice
+		expect(page.markdown).not.toContain("truncated to ~");
+		// Each chunk should end at a word boundary (not mid-word)
+		for (const c of page.chunks) {
+			expect(c.text.trim()).not.toMatch(/\w-$/);
+		}
+	});
+
+	it("total chunk text fits within budget (first chunk may overflow)", async () => {
+		const budget = 100;
+		const page = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
+			tokenBudget: budget,
+		});
+		const totalChars = page.chunks.reduce((sum, c) => sum + c.text.length, 0);
+		// The first chunk is always included even if it exceeds the budget.
+		// From chunk 2 onward, total must stay within budget.
+		const firstChunkLen = page.chunks[0]?.text.length ?? 0;
+		const rest = totalChars - firstChunkLen;
+		expect(rest).toBeLessThanOrEqual(budget * 4);
+	});
+
+	it("without budget, returns all chunks", async () => {
+		const withBudget = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
+			tokenBudget: 50,
+		});
+		const withoutBudget = await spider("https://example.com", {
+			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
+		});
+		expect(withBudget.chunks.length).toBeLessThan(withoutBudget.chunks.length);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Sitemap discovery
+// ---------------------------------------------------------------------------
+
+describe("sitemap.xml seeds crawl frontier", () => {
+	const sitemapXml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://example.com/page-a</loc></url>
+  <url><loc>https://example.com/page-b</loc></url>
+  <url><loc>https://example.com/page-c</loc></url>
+</urlset>`;
+
+	const pageHtml = articleHtml("Page", "<p>Content here. ".repeat(20) + "</p>");
+
+	it("fetches sitemap.xml and includes those URLs in crawl", async () => {
+		const visited: string[] = [];
+		const client = mockClient({
+			"https://example.com": { body: pageHtml },
+			"https://example.com/sitemap.xml": { body: sitemapXml },
+			"https://example.com/page-a": { body: pageHtml },
+			"https://example.com/page-b": { body: pageHtml },
+			"https://example.com/page-c": { body: pageHtml },
+		});
+
+		const result = await crawl("https://example.com", {
+			httpClient: client,
+			maxDepth: 0,
+			maxPages: 10,
+			useSitemap: true,
+			onPage: (p) => visited.push(p.url),
+		});
+
+		expect(result.pages.has("https://example.com/page-a")).toBe(true);
+		expect(result.pages.has("https://example.com/page-b")).toBe(true);
+		expect(result.pages.has("https://example.com/page-c")).toBe(true);
+	});
+
+	it("falls back to normal BFS when sitemap is missing (404)", async () => {
+		const client = mockClient({
+			"https://example.com": { body: articleHtml("Home", '<p>Text. <a href="/about">About</a></p>') },
+			"https://example.com/sitemap.xml": { status: 404, body: "" },
+			"https://example.com/about": { body: pageHtml },
+		});
+
+		const result = await crawl("https://example.com", {
+			httpClient: client,
+			maxDepth: 1,
+			useSitemap: true,
+		});
+
+		expect(result.pages.size).toBeGreaterThan(0);
+		// Should not throw even though sitemap 404d
+	});
+
+	it("sitemap disabled when useSitemap:false", async () => {
+		const sitemapFetched = { value: false };
+		const client: IHttpClient = {
+			fetch: async (req) => {
+				if (req.url.includes("sitemap")) sitemapFetched.value = true;
+				return {
+					ok: true, status: 200, statusText: "OK",
+					headers: { get: () => null },
+					text: async () => pageHtml,
+					arrayBuffer: async () => new ArrayBuffer(0),
+				};
+			},
+		};
+
+		await crawl("https://example.com", {
+			httpClient: client,
+			maxDepth: 0,
+			useSitemap: false,
+		});
+
+		expect(sitemapFetched.value).toBe(false);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Disk cache (tested via ICache contract)
+// ---------------------------------------------------------------------------
+
+import { DiskCache } from "../src/disk-cache.js";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { SpideredPage } from "../src/types.js";
+
+describe("DiskCache persists across instances", () => {
+	function makePage(url: string): SpideredPage {
+		return {
+			url, domain: "example.com", fetchedAt: new Date().toISOString(),
+			title: "Test", description: "desc", author: "", publishedAt: "",
+			lang: "en", tags: [], wordCount: 10, readingTimeMinutes: 1,
+			headings: [], chunks: [], links: [], markdown: "hello",
+		};
+	}
+
+	it("persists a page and retrieves it in a new instance", () => {
+		const dir = mkdtempSync(join(tmpdir(), "spider-cache-"));
+		const path = join(dir, "cache.json");
+		try {
+			const cache1 = new DiskCache(path);
+			const page = makePage("https://example.com/a");
+			cache1.set("https://example.com/a", page);
+			cache1.flush();
+
+			const cache2 = new DiskCache(path);
+			const retrieved = cache2.get("https://example.com/a");
+			expect(retrieved?.url).toBe("https://example.com/a");
+			expect(retrieved?.markdown).toBe("hello");
+		} finally {
+			rmSync(dir, { recursive: true });
+		}
+	});
+
+	it("returns undefined for expired entries", () => {
+		const dir = mkdtempSync(join(tmpdir(), "spider-cache-"));
+		const path = join(dir, "cache.json");
+		try {
+			const cache = new DiskCache(path, { ttlMs: 1 }); // 1ms TTL
+			cache.set("https://example.com/b", makePage("https://example.com/b"));
+			cache.flush();
+
+			// Wait for TTL to expire
+			const waited = Date.now() + 5;
+			while (Date.now() < waited) { /* spin */ }
+
+			const cache2 = new DiskCache(path, { ttlMs: 1 });
+			expect(cache2.get("https://example.com/b")).toBeUndefined();
+		} finally {
+			rmSync(dir, { recursive: true });
+		}
+	});
+
+	it("implements ICache interface", () => {
+		const dir = mkdtempSync(join(tmpdir(), "spider-cache-"));
+		const path = join(dir, "cache.json");
+		try {
+			const cache = new DiskCache(path);
+			const page = makePage("https://example.com/c");
+			expect(cache.has("https://example.com/c")).toBe(false);
+			cache.set("https://example.com/c", page);
+			expect(cache.has("https://example.com/c")).toBe(true);
+			cache.delete("https://example.com/c");
+			expect(cache.has("https://example.com/c")).toBe(false);
+		} finally {
+			rmSync(dir, { recursive: true });
+		}
+	});
+});