@dpopsuev/web-spider 0.10.4 → 0.10.5

package/test/images-integration.test.ts

@@ -1,359 +0,0 @@
-/**
- * End-to-end captureImages integration tests.
- *
- * Covers the full pipeline:
- *   spider() → SpideredPage.images → DiskCache.flush() → DiskCache.get()
- *   → LLM data URL → PlaywrightHttpClient-shaped stub
- *
- * No real network, no real browser.
- */
-
-import { mkdirSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { readFileSync } from "node:fs";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { DiskCache } from "../src/disk-cache.js";
-import { PlaywrightHttpClient } from "../src/playwright.js";
-import type { IHttpClient } from "../src/ports.js";
-import { spider } from "../src/spider.js";
-
-// ---------------------------------------------------------------------------
-// Fixtures
-// ---------------------------------------------------------------------------
-
-const FIXTURE_HTML = readFileSync(
-	join(import.meta.dirname, "../fixtures/article-with-images.html"),
-	"utf8",
-);
-const SMALL_JPG = readFileSync(join(import.meta.dirname, "../fixtures/images/small.jpg"));
-const TINY_PNG = readFileSync(join(import.meta.dirname, "../fixtures/images/tiny.png"));
-const LARGE_JPG = readFileSync(join(import.meta.dirname, "../fixtures/images/large.jpg"));
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-let testDir: string;
-
-beforeEach(() => {
-	testDir = join(tmpdir(), `wbs-integration-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-	mkdirSync(testDir, { recursive: true });
-});
-
-afterEach(() => {
-	rmSync(testDir, { recursive: true, force: true });
-});
-
-function makeCachePath() {
-	return join(testDir, "pages.json");
-}
-
-/**
- * Stub client that serves the fixture HTML for page fetches and
- * appropriate fixture image bytes for image fetches.
- * `useLargeImages`: serve large.jpg (>32KB) to exercise disk-spill.
- */
-function makeStubClient(opts: { useLargeImages?: boolean } = {}): IHttpClient {
-	return {
-		async fetch(req) {
-			const isImageReq = (req.headers?.["Accept"] ?? "").startsWith("image/");
-
-			if (!isImageReq) {
-				// Page fetch
-				return {
-					ok: true,
-					status: 200,
-					statusText: "OK",
-					headers: { get: (n) => (n === "content-type" ? "text/html" : null) },
-					text: async () => FIXTURE_HTML,
-					arrayBuffer: async () => new ArrayBuffer(0),
-				};
-			}
-
-			// Image fetch — pick fixture based on src extension / useLargeImages flag
-			const src = req.url;
-			let bytes: Buffer;
-			let mime: string;
-
-			if (opts.useLargeImages) {
-				bytes = LARGE_JPG;
-				mime = "image/jpeg";
-			} else if (src.match(/\.png(\?|$)/i)) {
-				bytes = TINY_PNG;
-				mime = "image/png";
-			} else {
-				bytes = SMALL_JPG;
-				mime = "image/jpeg";
-			}
-
-			const buf = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
-			return {
-				ok: true,
-				status: 200,
-				statusText: "OK",
-				headers: { get: (n) => (n === "content-type" ? mime : null) },
-				text: async () => "",
-				arrayBuffer: async () => buf,
-			};
-		},
-	};
-}
-
-// ---------------------------------------------------------------------------
-// 1. Full spider() → DiskCache → reload roundtrip (small images, inline)
-// ---------------------------------------------------------------------------
-
-describe("spider() → DiskCache roundtrip — small images (inline)", () => {
-	it("images survive flush + reload with correct base64", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-			captureImages: true,
-		});
-
-		expect(page.images).toBeDefined();
-		expect(page.images!.length).toBeGreaterThan(0);
-
-		const cachePath = makeCachePath();
-		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		cache1.set("https://example.com", page);
-		cache1.flush();
-
-		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		const reloaded = cache2.get("https://example.com");
-
-		expect(reloaded).toBeDefined();
-		expect(reloaded!.images).toBeDefined();
-		expect(reloaded!.images!.length).toBe(page.images!.length);
-
-		// Every base64 must survive the roundtrip exactly
-		for (let i = 0; i < page.images!.length; i++) {
-			const orig = page.images![i];
-			const loaded = reloaded!.images![i];
-			expect(loaded.src).toBe(orig.src);
-			expect(loaded.mimeType).toBe(orig.mimeType);
-			expect(loaded.alt).toBe(orig.alt);
-			if (orig.base64) expect(loaded.base64).toBe(orig.base64);
-		}
-	});
-
-	it("page text (markdown, chunks, title) also survives the roundtrip", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-			captureImages: true,
-		});
-
-		const cachePath = makeCachePath();
-		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		cache1.set("https://example.com", page);
-		cache1.flush();
-
-		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		const reloaded = cache2.get("https://example.com");
-
-		expect(reloaded!.title).toBe(page.title);
-		expect(reloaded!.markdown).toBe(page.markdown);
-		expect(reloaded!.chunks.length).toBe(page.chunks.length);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// 2. Full spider() → DiskCache → reload roundtrip (large images, disk-spill)
-// ---------------------------------------------------------------------------
-
-describe("spider() → DiskCache roundtrip — large images (disk-spill)", () => {
-	it("large images are spilled to disk and hydrated on reload", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient({ useLargeImages: true }),
-			captureImages: true,
-			maxImages: 3,
-		});
-
-		expect(page.images).toBeDefined();
-		const cachePath = makeCachePath();
-
-		// Use a low threshold so even SMALL_JPG spills — 100 bytes decoded
-		const cache1 = new DiskCache(cachePath, {
-			ttlMs: 60 * 60 * 1000,
-			autoFlush: false,
-			inlineImageThreshold: 100,
-		});
-		cache1.set("https://example.com", page);
-		cache1.flush();
-
-		const cache2 = new DiskCache(cachePath, {
-			ttlMs: 60 * 60 * 1000,
-			autoFlush: false,
-			inlineImageThreshold: 100,
-		});
-		const reloaded = cache2.get("https://example.com");
-
-		expect(reloaded!.images).toBeDefined();
-		// All images must have base64 after hydration
-		for (const img of reloaded!.images!) {
-			if (img.filePath) {
-				expect(img.base64).toBeDefined();
-				expect(img.base64!.length).toBeGreaterThan(0);
-			}
-		}
-	});
-});
-
-// ---------------------------------------------------------------------------
-// 3. LLM wire format — every image produces a valid data URL
-// ---------------------------------------------------------------------------
-
-describe("LLM wire format", () => {
-	it("every captured image yields a valid data: URL", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-			captureImages: true,
-		});
-
-		expect(page.images!.length).toBeGreaterThan(0);
-
-		for (const img of page.images!) {
-			if (!img.base64) continue;
-			const dataUrl = `data:${img.mimeType};base64,${img.base64}`;
-			expect(dataUrl).toMatch(/^data:image\/(jpeg|png|webp|gif|svg\+xml|avif);base64,/);
-		}
-	});
-
-	it("base64 in data URL decodes to non-empty binary", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-			captureImages: true,
-		});
-
-		for (const img of page.images!) {
-			if (!img.base64) continue;
-			const decoded = Buffer.from(img.base64, "base64");
-			expect(decoded.byteLength).toBeGreaterThan(0);
-		}
-	});
-
-	it("data: URL images from fixture have correct inline base64", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-			captureImages: true,
-		});
-
-		// The fixture contains one data: URL (1x1 PNG)
-		const inlineImg = page.images!.find((i) => i.src.startsWith("data:"));
-		expect(inlineImg).toBeDefined();
-		expect(inlineImg!.mimeType).toBe("image/png");
-
-		const dataUrl = `data:${inlineImg!.mimeType};base64,${inlineImg!.base64}`;
-		expect(dataUrl).toMatch(/^data:image\/png;base64,/);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// 4. captureImages: false — no images attached, cache roundtrip clean
-// ---------------------------------------------------------------------------
-
-describe("captureImages: false — clean roundtrip", () => {
-	it("images field is absent on spider() result", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-		});
-		expect(page.images).toBeUndefined();
-	});
-
-	it("cache roundtrip without images is clean", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-		});
-
-		const cachePath = makeCachePath();
-		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		cache1.set("https://example.com", page);
-		cache1.flush();
-
-		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		const reloaded = cache2.get("https://example.com");
-		expect(reloaded!.images).toBeUndefined();
-	});
-});
-
-// ---------------------------------------------------------------------------
-// 5. PlaywrightHttpClient-shaped stub with captureImages
-// ---------------------------------------------------------------------------
-
-describe("PlaywrightHttpClient captureImages integration", () => {
-	it("PlaywrightHttpClient constructs with captureImages:true and satisfies IHttpClient", () => {
-		const client: IHttpClient = new PlaywrightHttpClient({ captureImages: true });
-		expect(typeof client.fetch).toBe("function");
-	});
-
-	it("spider() with a Playwright-shaped stub and captureImages:true returns images", async () => {
-		// Simulate what PlaywrightHttpClient would do: a stub that looks like
-		// a Playwright client — returns HTML for page fetches, images for image fetches.
-		const playwrightShapedStub: IHttpClient = {
-			async fetch(req) {
-				const isImageReq = (req.headers?.["Accept"] ?? "").startsWith("image/");
-				if (!isImageReq) {
-					return {
-						ok: true,
-						status: 200,
-						statusText: "OK",
-						headers: { get: (n) => (n === "content-type" ? "text/html" : null) },
-						text: async () => FIXTURE_HTML,
-						arrayBuffer: async () => new ArrayBuffer(0),
-					};
-				}
-				const buf = SMALL_JPG.buffer.slice(
-					SMALL_JPG.byteOffset,
-					SMALL_JPG.byteOffset + SMALL_JPG.byteLength,
-				) as ArrayBuffer;
-				return {
-					ok: true,
-					status: 200,
-					statusText: "OK",
-					headers: { get: (n) => (n === "content-type" ? "image/jpeg" : null) },
-					text: async () => "",
-					arrayBuffer: async () => buf,
-				};
-			},
-		};
-
-		const page = await spider("https://example.com", {
-			httpClient: playwrightShapedStub,
-			captureImages: true,
-		});
-
-		expect(page.images).toBeDefined();
-		expect(page.images!.length).toBeGreaterThan(0);
-
-		for (const img of page.images!) {
-			if (img.base64) {
-				expect(`data:${img.mimeType};base64,${img.base64}`).toMatch(/^data:image\//);
-			}
-		}
-	});
-});
-
-// ---------------------------------------------------------------------------
-// 6. maxImages respected end-to-end through cache
-// ---------------------------------------------------------------------------
-
-describe("maxImages end-to-end", () => {
-	it("maxImages:2 — only 2 images in cache after roundtrip", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: makeStubClient(),
-			captureImages: true,
-			maxImages: 2,
-		});
-
-		expect(page.images!.length).toBeLessThanOrEqual(2);
-
-		const cachePath = makeCachePath();
-		const cache1 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		cache1.set("https://example.com", page);
-		cache1.flush();
-
-		const cache2 = new DiskCache(cachePath, { ttlMs: 60 * 60 * 1000, autoFlush: false });
-		const reloaded = cache2.get("https://example.com");
-		expect(reloaded!.images!.length).toBeLessThanOrEqual(2);
-	});
-});

package/test/improvements.test.ts

@@ -1,279 +0,0 @@
-/**
- * Tests for improvement tasks: JS degradation, chunk tokenBudget, sitemap discovery, disk cache.
- * Written before implementation — all should fail until code is in place.
- */
-
-import { describe, expect, it } from "vitest";
-import { spider } from "../src/spider.js";
-import { crawl } from "../src/crawl.js";
-import type { IHttpClient } from "../src/ports.js";
-
-// ---------------------------------------------------------------------------
-// Shared helpers
-// ---------------------------------------------------------------------------
-
-function mockClient(responses: Record<string, { status?: number; body: string }>): IHttpClient {
-	return {
-		fetch: async (req) => {
-			const entry = responses[req.url] ?? responses["*"];
-			if (!entry) throw new Error(`Unexpected fetch: ${req.url}`);
-			const status = entry.status ?? 200;
-			return {
-				ok: status >= 200 && status < 300,
-				status,
-				statusText: status === 200 ? "OK" : "Error",
-				headers: { get: () => null },
-				text: async () => entry.body,
-				arrayBuffer: async () => new ArrayBuffer(0),
-			};
-		},
-	};
-}
-
-const articleHtml = (title: string, body: string) => `<!DOCTYPE html>
-<html lang="en">
-<head><title>${title}</title><meta name="description" content="test"></head>
-<body><article><h1>${title}</h1>${body}</article></body>
-</html>`;
-
-const LONG_BODY = `<p>${"Word ".repeat(300)}</p><h2>Section</h2><p>${"More words. ".repeat(300)}</p>`;
-
-// ---------------------------------------------------------------------------
-// Graceful degradation on JS-rendered pages
-// ---------------------------------------------------------------------------
-
-describe("JS-rendered pages degrade gracefully", () => {
-	const jsHtml = `<!DOCTYPE html><html><head><title>App</title></head>
-<body><div id="root"></div><script>/* SPA */</script></body></html>`;
-
-	it("returns a page with jsRendered:true instead of throwing", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: jsHtml } }),
-		});
-		expect((page as { jsRendered?: boolean }).jsRendered).toBe(true);
-	});
-
-	it("still returns title and links from JS page", async () => {
-		const html = `<!DOCTYPE html><html><head><title>My SPA</title></head>
-<body><div id="root"></div><a href="/about">About</a></body></html>`;
-		const page = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: html } }),
-		});
-		expect(page.title).toContain("My SPA");
-		expect(page.links.length).toBeGreaterThan(0);
-	});
-
-	it("returns empty chunks and markdown for JS page", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: jsHtml } }),
-		});
-		expect(page.chunks).toHaveLength(0);
-		expect(page.markdown).toBe("");
-	});
-
-	it("lean view also degrades instead of throwing", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: jsHtml } }),
-			view: "lean",
-		});
-		expect((page as { jsRendered?: boolean }).jsRendered).toBe(true);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Chunk-aware tokenBudget
-// ---------------------------------------------------------------------------
-
-describe("chunk-aware tokenBudget", () => {
-	it("returns whole chunks up to budget, not truncated mid-sentence", async () => {
-		const page = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
-			tokenBudget: 100,
-		});
-		// Should have at least one complete chunk
-		expect(page.chunks.length).toBeGreaterThan(0);
-		// Markdown should not end with truncation notice
-		expect(page.markdown).not.toContain("truncated to ~");
-		// Each chunk should end at a word boundary (not mid-word)
-		for (const c of page.chunks) {
-			expect(c.text.trim()).not.toMatch(/\w-$/);
-		}
-	});
-
-	it("total chunk text fits within budget (first chunk may overflow)", async () => {
-		const budget = 100;
-		const page = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
-			tokenBudget: budget,
-		});
-		const totalChars = page.chunks.reduce((sum, c) => sum + c.text.length, 0);
-		// The first chunk is always included even if it exceeds the budget.
-		// From chunk 2 onward, total must stay within budget.
-		const firstChunkLen = page.chunks[0]?.text.length ?? 0;
-		const rest = totalChars - firstChunkLen;
-		expect(rest).toBeLessThanOrEqual(budget * 4);
-	});
-
-	it("without budget, returns all chunks", async () => {
-		const withBudget = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
-			tokenBudget: 50,
-		});
-		const withoutBudget = await spider("https://example.com", {
-			httpClient: mockClient({ "*": { body: articleHtml("Test", LONG_BODY) } }),
-		});
-		expect(withBudget.chunks.length).toBeLessThan(withoutBudget.chunks.length);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Sitemap discovery
-// ---------------------------------------------------------------------------
-
-describe("sitemap.xml seeds crawl frontier", () => {
-	const sitemapXml = `<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-  <url><loc>https://example.com/page-a</loc></url>
-  <url><loc>https://example.com/page-b</loc></url>
-  <url><loc>https://example.com/page-c</loc></url>
-</urlset>`;
-
-	const pageHtml = articleHtml("Page", "<p>Content here. ".repeat(20) + "</p>");
-
-	it("fetches sitemap.xml and includes those URLs in crawl", async () => {
-		const visited: string[] = [];
-		const client = mockClient({
-			"https://example.com": { body: pageHtml },
-			"https://example.com/sitemap.xml": { body: sitemapXml },
-			"https://example.com/page-a": { body: pageHtml },
-			"https://example.com/page-b": { body: pageHtml },
-			"https://example.com/page-c": { body: pageHtml },
-		});
-
-		const result = await crawl("https://example.com", {
-			httpClient: client,
-			maxDepth: 0,
-			maxPages: 10,
-			useSitemap: true,
-			onPage: (p) => visited.push(p.url),
-		});
-
-		expect(result.pages.has("https://example.com/page-a")).toBe(true);
-		expect(result.pages.has("https://example.com/page-b")).toBe(true);
-		expect(result.pages.has("https://example.com/page-c")).toBe(true);
-	});
-
-	it("falls back to normal BFS when sitemap is missing (404)", async () => {
-		const client = mockClient({
-			"https://example.com": { body: articleHtml("Home", '<p>Text. <a href="/about">About</a></p>') },
-			"https://example.com/sitemap.xml": { status: 404, body: "" },
-			"https://example.com/about": { body: pageHtml },
-		});
-
-		const result = await crawl("https://example.com", {
-			httpClient: client,
-			maxDepth: 1,
-			useSitemap: true,
-		});
-
-		expect(result.pages.size).toBeGreaterThan(0);
-		// Should not throw even though sitemap 404d
-	});
-
-	it("sitemap disabled when useSitemap:false", async () => {
-		const sitemapFetched = { value: false };
-		const client: IHttpClient = {
-			fetch: async (req) => {
-				if (req.url.includes("sitemap")) sitemapFetched.value = true;
-				return {
-					ok: true, status: 200, statusText: "OK",
-					headers: { get: () => null },
-					text: async () => pageHtml,
-					arrayBuffer: async () => new ArrayBuffer(0),
-				};
-			},
-		};
-
-		await crawl("https://example.com", {
-			httpClient: client,
-			maxDepth: 0,
-			useSitemap: false,
-		});
-
-		expect(sitemapFetched.value).toBe(false);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Disk cache (tested via ICache contract)
-// ---------------------------------------------------------------------------
-
-import { DiskCache } from "../src/disk-cache.js";
-import { mkdtempSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { SpideredPage } from "../src/types.js";
-
-describe("DiskCache persists across instances", () => {
-	function makePage(url: string): SpideredPage {
-		return {
-			url, domain: "example.com", fetchedAt: new Date().toISOString(),
-			title: "Test", description: "desc", author: "", publishedAt: "",
-			lang: "en", tags: [], wordCount: 10, readingTimeMinutes: 1,
-			headings: [], chunks: [], links: [], markdown: "hello",
-		};
-	}
-
-	it("persists a page and retrieves it in a new instance", () => {
-		const dir = mkdtempSync(join(tmpdir(), "spider-cache-"));
-		const path = join(dir, "cache.json");
-		try {
-			const cache1 = new DiskCache(path);
-			const page = makePage("https://example.com/a");
-			cache1.set("https://example.com/a", page);
-			cache1.flush();
-
-			const cache2 = new DiskCache(path);
-			const retrieved = cache2.get("https://example.com/a");
-			expect(retrieved?.url).toBe("https://example.com/a");
-			expect(retrieved?.markdown).toBe("hello");
-		} finally {
-			rmSync(dir, { recursive: true });
-		}
-	});
-
-	it("returns undefined for expired entries", () => {
-		const dir = mkdtempSync(join(tmpdir(), "spider-cache-"));
-		const path = join(dir, "cache.json");
-		try {
-			const cache = new DiskCache(path, { ttlMs: 1 }); // 1ms TTL
-			cache.set("https://example.com/b", makePage("https://example.com/b"));
-			cache.flush();
-
-			// Wait for TTL to expire
-			const waited = Date.now() + 5;
-			while (Date.now() < waited) { /* spin */ }
-
-			const cache2 = new DiskCache(path, { ttlMs: 1 });
-			expect(cache2.get("https://example.com/b")).toBeUndefined();
-		} finally {
-			rmSync(dir, { recursive: true });
-		}
-	});
-
-	it("implements ICache interface", () => {
-		const dir = mkdtempSync(join(tmpdir(), "spider-cache-"));
-		const path = join(dir, "cache.json");
-		try {
-			const cache = new DiskCache(path);
-			const page = makePage("https://example.com/c");
-			expect(cache.has("https://example.com/c")).toBe(false);
-			cache.set("https://example.com/c", page);
-			expect(cache.has("https://example.com/c")).toBe(true);
-			cache.delete("https://example.com/c");
-			expect(cache.has("https://example.com/c")).toBe(false);
-		} finally {
-			rmSync(dir, { recursive: true });
-		}
-	});
-});