@dpopsuev/web-spider 0.10.4

package/test/disk-cache-images.test.ts

@@ -0,0 +1,193 @@
+/**
+ * TDD tests for DiskCache hybrid image persistence.
+ * Uses real tmp directories — no mocking of fs.
+ */
+
+import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { DiskCache } from "../src/disk-cache.js";
+import type { ImageRef, SpideredPage } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+let testDir: string;
+
+beforeEach(() => {
+	testDir = join(tmpdir(), `wbs-disk-cache-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+	mkdirSync(testDir, { recursive: true });
+});
+
+afterEach(() => {
+	rmSync(testDir, { recursive: true, force: true });
+});
+
+function cachePath(): string {
+	return join(testDir, "pages.json");
+}
+
+function makeCache(threshold = 32 * 1024): DiskCache {
+	return new DiskCache(cachePath(), {
+		ttlMs: 60 * 60 * 1000,
+		autoFlush: false,
+		inlineImageThreshold: threshold,
+	});
+}
+
+function makeImage(base64: string, src = "https://example.com/photo.jpg"): ImageRef {
+	return { src, mimeType: "image/jpeg", alt: "Test image", base64 };
+}
+
+function makePage(images: ImageRef[]): SpideredPage {
+	return {
+		url: "https://example.com",
+		domain: "example.com",
+		fetchedAt: new Date().toISOString(),
+		title: "Test",
+		description: "",
+		author: "",
+		publishedAt: "",
+		lang: "en",
+		tags: [],
+		wordCount: 0,
+		readingTimeMinutes: 0,
+		headings: [],
+		chunks: [],
+		links: [],
+		markdown: "",
+		images,
+	};
+}
+
+/** Generate a base64 string of approximately `bytes` bytes when decoded. */
+function makeBase64(bytes: number): string {
+	return Buffer.alloc(bytes, 0xab).toString("base64");
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("DiskCache hybrid image persistence", () => {
+	it("1. small image (\u2264 threshold) is stored inline in JSON", () => {
+		const cache = makeCache(32 * 1024);
+		const b64 = makeBase64(100); // 100 bytes decoded \u226532KB threshold
+		const page = makePage([makeImage(b64)]);
+
+		cache.set("https://example.com", page);
+		cache.flush();
+
+		const raw = JSON.parse(readFileSync(cachePath(), "utf8")) as { entries: Record<string, { page: SpideredPage }> };
+		const entry = Object.values(raw.entries)[0];
+		expect(entry.page.images![0].base64).toBe(b64);
+		expect(entry.page.images![0].filePath).toBeUndefined();
+	});
+
+	it("2. large image (> threshold) is written to disk — JSON has filePath, no base64", () => {
+		const cache = makeCache(32 * 1024);
+		const b64 = makeBase64(40 * 1024); // 40KB decoded > 32KB threshold
+		const page = makePage([makeImage(b64)]);
+
+		cache.set("https://example.com", page);
+		cache.flush();
+
+		const raw = JSON.parse(readFileSync(cachePath(), "utf8")) as { entries: Record<string, { page: SpideredPage }> };
+		const entry = Object.values(raw.entries)[0];
+		const storedImg = entry.page.images![0];
+
+		expect(storedImg.base64).toBeUndefined();
+		expect(storedImg.filePath).toBeDefined();
+		expect(existsSync(storedImg.filePath!)).toBe(true);
+	});
+
+	it("3. large image is hydrated on get() after reload", () => {
+		const b64 = makeBase64(40 * 1024);
+		const page = makePage([makeImage(b64)]);
+
+		const cache1 = makeCache(32 * 1024);
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		// Fresh cache instance — simulates restart
+		const cache2 = makeCache(32 * 1024);
+		const loaded = cache2.get("https://example.com");
+
+		expect(loaded).toBeDefined();
+		expect(loaded!.images![0].base64).toBe(b64);
+	});
+
+	it("4. missing binary file degrades gracefully — no throw, filePath preserved", () => {
+		const b64 = makeBase64(40 * 1024);
+		const page = makePage([makeImage(b64)]);
+
+		const cache1 = makeCache(32 * 1024);
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		// Read the JSON to find the file path, then delete it
+		const raw = JSON.parse(readFileSync(cachePath(), "utf8")) as { entries: Record<string, { page: SpideredPage }> };
+		const entry = Object.values(raw.entries)[0];
+		const filePath = entry.page.images![0].filePath!;
+		rmSync(filePath);
+
+		const cache2 = makeCache(32 * 1024);
+		let result: SpideredPage | undefined;
+		expect(() => { result = cache2.get("https://example.com"); }).not.toThrow();
+		expect(result).toBeDefined();
+		expect(result!.images![0].filePath).toBeDefined();
+		expect(result!.images![0].base64).toBeUndefined();
+	});
+
+	it("5. images/ directory is created automatically on first large-image flush", () => {
+		const imagesDir = join(testDir, "images");
+		expect(existsSync(imagesDir)).toBe(false);
+
+		const cache = makeCache(32 * 1024);
+		const b64 = makeBase64(40 * 1024);
+		cache.set("https://example.com", makePage([makeImage(b64)]));
+		cache.flush();
+
+		expect(existsSync(imagesDir)).toBe(true);
+	});
+
+	it("page without images round-trips cleanly", () => {
+		const cache1 = makeCache();
+		const page = makePage([]);
+		// Override images to be undefined (no captureImages)
+		const noImgPage = { ...page, images: undefined };
+		cache1.set("https://example.com", noImgPage);
+		cache1.flush();
+
+		const cache2 = makeCache();
+		const loaded = cache2.get("https://example.com");
+		expect(loaded).toBeDefined();
+		expect(loaded!.images).toBeUndefined();
+	});
+
+	it("multiple images — mix of small and large — persisted correctly", () => {
+		const smallB64 = makeBase64(100);
+		const largeB64 = makeBase64(40 * 1024);
+		const page = makePage([
+			makeImage(smallB64, "https://example.com/small.jpg"),
+			makeImage(largeB64, "https://example.com/large.jpg"),
+		]);
+
+		const cache1 = makeCache(32 * 1024);
+		cache1.set("https://example.com", page);
+		cache1.flush();
+
+		const cache2 = makeCache(32 * 1024);
+		const loaded = cache2.get("https://example.com");
+
+		expect(loaded!.images).toHaveLength(2);
+		// Small: inline
+		const small = loaded!.images!.find((i) => i.src.includes("small"))!;
+		expect(small.base64).toBe(smallB64);
+		// Large: hydrated from file
+		const large = loaded!.images!.find((i) => i.src.includes("large"))!;
+		expect(large.base64).toBe(largeB64);
+	});
+});

package/test/engine-registry.test.ts

@@ -0,0 +1,114 @@
+/**
+ * TDD tests for the search engine registry.
+ *
+ * Adding a new engine must not require editing existing code —
+ * only registering a new entry.
+ */
+
+import { describe, expect, it } from "vitest";
+import type { ISearchEngine, SearchQuery, WebSearchResult } from "../src/ports.js";
+import {
+	registerSearchEngine,
+	resolveSearchEngine,
+	defaultSearchEngine,
+	BraveSearchEngine,
+	TavilySearchEngine,
+	ExaSearchEngine,
+	DdgSearchEngine,
+} from "../src/web-search.js";
+
+// ---------------------------------------------------------------------------
+// Stub engine — used to verify registration without touching real APIs
+// ---------------------------------------------------------------------------
+
+class StubEngine implements ISearchEngine {
+	readonly calls: SearchQuery[] = [];
+	constructor(private readonly results: WebSearchResult[] = []) {}
+	async search(req: SearchQuery): Promise<WebSearchResult[]> {
+		this.calls.push(req);
+		return this.results;
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Registry — registerSearchEngine / resolveSearchEngine
+// ---------------------------------------------------------------------------
+
+describe("registerSearchEngine / resolveSearchEngine", () => {
+	it("resolves a built-in engine by name without editing existing code", () => {
+		const engine = resolveSearchEngine("ddg");
+		expect(engine).toBeInstanceOf(DdgSearchEngine);
+	});
+
+	it("resolves brave when BRAVE_SEARCH_API_KEY is set", () => {
+		const engine = resolveSearchEngine("brave", "test-brave-key");
+		expect(engine).toBeInstanceOf(BraveSearchEngine);
+	});
+
+	it("resolves tavily when TAVILY_API_KEY is set", () => {
+		const engine = resolveSearchEngine("tavily", "test-tavily-key");
+		expect(engine).toBeInstanceOf(TavilySearchEngine);
+	});
+
+	it("resolves exa when EXA_API_KEY is set", () => {
+		const engine = resolveSearchEngine("exa", "test-exa-key");
+		expect(engine).toBeInstanceOf(ExaSearchEngine);
+	});
+
+	it("throws a descriptive error when key is missing for a keyed engine", () => {
+		expect(() => resolveSearchEngine("brave", undefined)).toThrow(/BRAVE_SEARCH_API_KEY/);
+		expect(() => resolveSearchEngine("tavily", undefined)).toThrow(/TAVILY_API_KEY/);
+		expect(() => resolveSearchEngine("exa", undefined)).toThrow(/EXA_API_KEY/);
+	});
+
+	it("throws for an unknown engine name", () => {
+		expect(() => resolveSearchEngine("unknown-engine" as never, undefined)).toThrow(/unknown.*engine/i);
+	});
+
+	it("a third-party engine can be registered without editing existing code", () => {
+		const stub = new StubEngine([{ url: "https://test.com", title: "Test", snippet: "ok" }]);
+		registerSearchEngine("my-custom-engine", () => stub);
+
+		const resolved = resolveSearchEngine("my-custom-engine" as never, undefined);
+		expect(resolved).toBe(stub);
+	});
+
+	it("registered engine is callable and returns results", async () => {
+		const stub = new StubEngine([{ url: "https://custom.com", title: "Custom", snippet: "result" }]);
+		registerSearchEngine("test-engine-2", () => stub);
+
+		const engine = resolveSearchEngine("test-engine-2" as never, undefined);
+		const results = await engine.search({ query: "hello", numResults: 5 });
+		expect(results[0].url).toBe("https://custom.com");
+		expect(stub.calls).toHaveLength(1);
+		expect(stub.calls[0].query).toBe("hello");
+	});
+
+	it("registered engine overwrites a previous registration for the same name", () => {
+		const first = new StubEngine();
+		const second = new StubEngine();
+		registerSearchEngine("overwrite-test", () => first);
+		registerSearchEngine("overwrite-test", () => second);
+
+		const resolved = resolveSearchEngine("overwrite-test" as never, undefined);
+		expect(resolved).toBe(second);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// defaultSearchEngine — still builds the right chain from env vars
+// ---------------------------------------------------------------------------
+
+describe("defaultSearchEngine", () => {
+	it("returns an ISearchEngine", () => {
+		const engine = defaultSearchEngine();
+		expect(typeof engine.search).toBe("function");
+	});
+
+	it("includes DdgSearchEngine as last-resort fallback (always present)", () => {
+		// defaultSearchEngine always returns a FallbackSearchEngine that ends with DDG.
+		// We verify by checking the returned engine is functional even with no API keys.
+		const engine = defaultSearchEngine();
+		expect(engine).toBeDefined();
+	});
+});

package/test/exports.test.ts

@@ -0,0 +1,124 @@
+/**
+ * Package export smoke tests.
+ *
+ * Imports every public symbol from the package entrypoint and asserts it is
+ * present and has the right shape. This catches "not a constructor" and
+ * "undefined is not a function" errors before they reach users.
+ *
+ * Run after every build: these tests operate on the compiled src/, not dist/,
+ * so they reflect what will be shipped.
+ */
+
+import { describe, expect, it } from "vitest";
+import {
+	DdgSearchEngine,
+	DomainThrottle,
+	FallbackSearchEngine,
+	PageGraph,
+	PlaywrightHttpClient,
+	RobotsCache,
+	SpiderCache,
+	braveSearch,
+	ddgSearch,
+	buildTree,
+	crawl,
+	createPlaywrightClient,
+	fuzzySearch,
+	searchPages,
+	navigateTree,
+	queryTree,
+	spider,
+	tavilySearch,
+	toLean,
+	webSearch,
+} from "../src/index.js";
+
+describe("class constructors", () => {
+	it("SpiderCache is constructable", () => {
+		expect(typeof SpiderCache).toBe("function");
+		expect(new SpiderCache()).toBeInstanceOf(SpiderCache);
+	});
+
+	it("PageGraph is constructable", () => {
+		expect(typeof PageGraph).toBe("function");
+		expect(new PageGraph()).toBeInstanceOf(PageGraph);
+	});
+
+	it("DomainThrottle is constructable", () => {
+		expect(typeof DomainThrottle).toBe("function");
+		const t = new DomainThrottle();
+		expect(t).toBeInstanceOf(DomainThrottle);
+		expect(typeof t.wait).toBe("function");
+		expect(typeof t.success).toBe("function");
+		expect(typeof t.rateLimit).toBe("function");
+	});
+
+	it("RobotsCache is constructable", () => {
+		expect(typeof RobotsCache).toBe("function");
+		const r = new RobotsCache();
+		expect(r).toBeInstanceOf(RobotsCache);
+		expect(typeof r.check).toBe("function");
+	});
+});
+
+describe("functions", () => {
+	it.each([
+		["spider", spider],
+		["crawl", crawl],
+		["fuzzySearch", fuzzySearch],
+		["searchPages", searchPages],
+		["buildTree", buildTree],
+		["navigateTree", navigateTree],
+		["queryTree", queryTree],
+		["toLean", toLean],
+		["webSearch", webSearch],
+		["braveSearch", braveSearch],
+		["tavilySearch", tavilySearch],
+		["ddgSearch", ddgSearch],
+	])("%s is a function", (_name, fn) => {
+		expect(typeof fn).toBe("function");
+	});
+});
+
+describe("PlaywrightHttpClient", () => {
+	it("is constructable", () => {
+		expect(typeof PlaywrightHttpClient).toBe("function");
+		const client = new PlaywrightHttpClient();
+		expect(typeof client.fetch).toBe("function");
+		expect(typeof client.close).toBe("function");
+	});
+
+	it("createPlaywrightClient returns an instance", () => {
+		const client = createPlaywrightClient();
+		expect(client).toBeInstanceOf(PlaywrightHttpClient);
+	});
+});
+
+describe("FallbackSearchEngine", () => {
+	it("is constructable with at least one engine", () => {
+		const stub = { search: async () => [] };
+		const fb = new FallbackSearchEngine([stub]);
+		expect(typeof fb.search).toBe("function");
+	});
+
+	it("DdgSearchEngine is constructable", () => {
+		const ddg = new DdgSearchEngine();
+		expect(typeof ddg.search).toBe("function");
+	});
+});
+
+describe("DomainThrottle defaults", () => {
+	it("has expected default values", () => {
+		const t = new DomainThrottle();
+		expect(t.minDelayMs).toBe(500);
+		expect(t.backoffBaseMs).toBe(1_000);
+		expect(t.backoffCapMs).toBe(30_000);
+		expect(t.maxRetries).toBe(3);
+	});
+
+	it("accepts custom options", () => {
+		const t = new DomainThrottle({ minDelayMs: 100, maxRetries: 1 });
+		expect(t.minDelayMs).toBe(100);
+		expect(t.maxRetries).toBe(1);
+	});
+});

package/test/get-chunk.test.ts

@@ -0,0 +1,115 @@
+/**
+ * TDD tests for getChunk(cache, url, index).
+ */
+
+import { describe, expect, it } from "vitest";
+import { SpiderCache } from "../src/cache.js";
+import { getChunk } from "../src/index.js";
+import type { Chunk, SpideredPage } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeChunk(index: number, text = `Chunk text number ${index}. `.repeat(12)): Chunk {
+	return {
+		id: `https://example.com#chunk-${index}`,
+		index,
+		heading: `Heading ${index}`,
+		text,
+		wordCount: text.split(/\s+/).filter(Boolean).length,
+		contentType: "text",
+	};
+}
+
+function makePage(chunks: Chunk[]): SpideredPage {
+	return {
+		url: "https://example.com",
+		domain: "example.com",
+		fetchedAt: new Date().toISOString(),
+		title: "Test Page",
+		description: "",
+		author: "",
+		publishedAt: "",
+		lang: "en",
+		tags: [],
+		wordCount: chunks.reduce((n, c) => n + c.wordCount, 0),
+		readingTimeMinutes: 1,
+		headings: [],
+		chunks,
+		links: [],
+		markdown: chunks.map((c) => c.text).join("\n\n"),
+	};
+}
+
+function populatedCache(): SpiderCache {
+	const cache = new SpiderCache();
+	cache.set("https://example.com", makePage([makeChunk(0), makeChunk(1), makeChunk(2)]));
+	cache.set("https://other.com", makePage([makeChunk(0), makeChunk(1)]));
+	return cache;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("getChunk(cache, url, index)", () => {
+	it("returns the correct chunk by index", () => {
+		const cache = populatedCache();
+		const chunk = getChunk(cache, "https://example.com", 1);
+		expect(chunk).toBeDefined();
+		expect(chunk!.index).toBe(1);
+		expect(chunk!.id).toBe("https://example.com#chunk-1");
+	});
+
+	it("returns chunk 0", () => {
+		const cache = populatedCache();
+		const chunk = getChunk(cache, "https://example.com", 0);
+		expect(chunk!.index).toBe(0);
+	});
+
+	it("returns the last chunk", () => {
+		const cache = populatedCache();
+		const chunk = getChunk(cache, "https://example.com", 2);
+		expect(chunk!.index).toBe(2);
+	});
+
+	it("returns undefined for an out-of-range index", () => {
+		const cache = populatedCache();
+		expect(getChunk(cache, "https://example.com", 99)).toBeUndefined();
+	});
+
+	it("returns undefined when the URL is not in the cache", () => {
+		const cache = populatedCache();
+		expect(getChunk(cache, "https://notcached.com", 0)).toBeUndefined();
+	});
+
+	it("returns undefined for a negative index", () => {
+		const cache = populatedCache();
+		expect(getChunk(cache, "https://example.com", -1)).toBeUndefined();
+	});
+
+	it("works across different cached URLs", () => {
+		const cache = populatedCache();
+		const a = getChunk(cache, "https://example.com", 2);
+		const b = getChunk(cache, "https://other.com", 1);
+		expect(a!.index).toBe(2);
+		expect(b!.index).toBe(1);
+	});
+
+	it("normalises trailing slashes in URL", () => {
+		const cache = populatedCache();
+		const chunk = getChunk(cache, "https://example.com/", 0);
+		expect(chunk).toBeDefined();
+	});
+
+	it("works with DiskCache via ICache interface", () => {
+		// getChunk accepts any ICache<string, SpideredPage> — verify it's not SpiderCache-specific
+		const cache = new SpiderCache();
+		const chunks = [makeChunk(0), makeChunk(1)];
+		cache.set("https://example.com", makePage(chunks));
+		const result = getChunk(cache, "https://example.com", 1);
+		expect(result).toBeDefined();
+		expect(result!.text).toBe(chunks[1].text);
+	});
+});