@dpopsuev/web-spider 0.10.4

package/test/tree.test.ts

@@ -0,0 +1,272 @@
+import { describe, expect, it } from "vitest";
+import { buildTree, navigateTree, queryTree } from "../src/tree.js";
+import type { DOMNode } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function _countNodes(node: DOMNode): number {
+	return 1 + (node.children ?? []).reduce((n, c) => n + _countNodes(c), 0);
+}
+
+function allPaths(node: DOMNode): string[] {
+	return [node.path, ...(node.children ?? []).flatMap(allPaths)];
+}
+
+function findByTag(node: DOMNode, tag: string): DOMNode[] {
+	const results: DOMNode[] = [];
+	if (node.tag === tag) results.push(node);
+	for (const child of node.children ?? []) results.push(...findByTag(child, tag));
+	return results;
+}
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+const SIMPLE_HTML = `
+<div>
+  <h1>Getting Started</h1>
+  <p>This is the introduction.</p>
+  <h2>Installation</h2>
+  <p>Run <code>npm install</code> to get started.</p>
+  <pre><code class="language-bash">npm install my-package</code></pre>
+  <h2>Usage</h2>
+  <ul>
+    <li>Item one</li>
+    <li>Item two</li>
+  </ul>
+</div>`;
+
+const TABLE_HTML = `
+<div>
+  <h2>Comparison</h2>
+  <table>
+    <thead><tr><th>Feature</th><th>A</th><th>B</th></tr></thead>
+    <tbody>
+      <tr><td>Speed</td><td>Fast</td><td>Slow</td></tr>
+      <tr><td>Cost</td><td>High</td><td>Low</td></tr>
+    </tbody>
+  </table>
+</div>`;
+
+const DEEP_WRAPPER_HTML = `
+<div>
+  <div>
+    <div>
+      <div>
+        <p>This is deeply nested but should collapse.</p>
+      </div>
+    </div>
+  </div>
+</div>`;
+
+// ---------------------------------------------------------------------------
+// Tree building
+// ---------------------------------------------------------------------------
+
+describe("buildTree", () => {
+	it("returns a root node tagged 'article'", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		expect(tree.tag).toBe("article");
+		expect(tree.path).toBe("article");
+	});
+
+	it("extracts headings", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const h1s = findByTag(tree, "h1");
+		const h2s = findByTag(tree, "h2");
+		expect(h1s).toHaveLength(1);
+		expect(h1s[0].text).toContain("Getting Started");
+		expect(h2s).toHaveLength(2);
+	});
+
+	it("extracts paragraphs with text", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const ps = findByTag(tree, "p");
+		expect(ps.length).toBeGreaterThanOrEqual(2);
+		expect(ps[0].text).toContain("introduction");
+	});
+
+	it("preserves pre/code blocks as atomic leaf nodes", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const pres = findByTag(tree, "pre");
+		expect(pres).toHaveLength(1);
+		expect(pres[0].text).toContain("npm install my-package");
+		// pre is a leaf — no children
+		expect(pres[0].children).toBeUndefined();
+	});
+
+	it("extracts lang attr from code blocks", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const pres = findByTag(tree, "pre");
+		expect(pres[0].attrs?.lang).toBe("bash");
+	});
+
+	it("extracts lists and list items", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const uls = findByTag(tree, "ul");
+		expect(uls).toHaveLength(1);
+		const lis = findByTag(tree, "li");
+		expect(lis).toHaveLength(2);
+		expect(lis[0].text).toBe("Item one");
+	});
+
+	it("preserves tables as structured subtrees", () => {
+		const tree = buildTree(TABLE_HTML, "https://example.com");
+		const tables = findByTag(tree, "table");
+		expect(tables).toHaveLength(1);
+		const tds = findByTag(tree, "td");
+		expect(tds.length).toBeGreaterThanOrEqual(4);
+		expect(tds[0].text).toBe("Speed");
+	});
+
+	it("collapses deep div wrappers", () => {
+		const tree = buildTree(DEEP_WRAPPER_HTML, "https://example.com");
+		const ps = findByTag(tree, "p");
+		expect(ps).toHaveLength(1);
+		expect(ps[0].text).toContain("deeply nested");
+		// should NOT have div nodes
+		const divs = findByTag(tree, "div");
+		expect(divs).toHaveLength(0);
+	});
+
+	it("generates unique paths for all nodes", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const paths = allPaths(tree);
+		const unique = new Set(paths);
+		expect(unique.size).toBe(paths.length);
+	});
+
+	it("sibling nodes of the same tag get bracket notation", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const h2s = findByTag(tree, "h2");
+		expect(h2s[0].path).not.toContain("["); // first h2 has no bracket
+		expect(h2s[1].path).toContain("[1]"); // second h2 gets [1]
+	});
+
+	it("href attr preserved on anchor tags", () => {
+		const html = `<div><p>See <a href="https://example.com/docs">the docs</a>.</p></div>`;
+		const tree = buildTree(html, "https://example.com");
+		const _links = findByTag(tree, "a");
+		// a tags inside p are inline — p is flattened to text, so link may not survive
+		// but the p text should contain "the docs"
+		const ps = findByTag(tree, "p");
+		expect(ps[0].text).toContain("the docs");
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Tree navigation
+// ---------------------------------------------------------------------------
+
+describe("navigateTree", () => {
+	it("returns the root when path is 'article'", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const node = navigateTree(tree, "article");
+		expect(node).toBe(tree);
+	});
+
+	it("returns a child node by exact path", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const h1s = findByTag(tree, "h1");
+		const h1Path = h1s[0].path;
+		const node = navigateTree(tree, h1Path);
+		expect(node).toBeDefined();
+		expect(node?.tag).toBe("h1");
+		expect(node?.text).toContain("Getting Started");
+	});
+
+	it("returns null for a non-existent path", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		expect(navigateTree(tree, "article.section[99].p")).toBeNull();
+	});
+
+	it("returns a pre node and its content is intact", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const pres = findByTag(tree, "pre");
+		const node = navigateTree(tree, pres[0].path);
+		expect(node?.tag).toBe("pre");
+		expect(node?.text).toContain("npm install");
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Tree fuzzy search
+// ---------------------------------------------------------------------------
+
+describe("queryTree", () => {
+	it("returns empty array for blank query", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		expect(queryTree(tree, "")).toEqual([]);
+		expect(queryTree(tree, "   ")).toEqual([]);
+	});
+
+	it("finds a heading by text", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "installation");
+		expect(hits.length).toBeGreaterThan(0);
+		expect(hits[0].node.tag).toBe("h2");
+		expect(hits[0].node.text).toContain("Installation");
+	});
+
+	it("finds content in a paragraph", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "introduction");
+		expect(hits.length).toBeGreaterThan(0);
+		expect(hits[0].snippet).toContain("introduction");
+	});
+
+	it("returns code blocks for code queries", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "npm install my-package");
+		expect(hits.length).toBeGreaterThan(0);
+		const codeHit = hits.find((h) => h.node.tag === "pre");
+		expect(codeHit).toBeDefined();
+		expect(codeHit!.node.text).toContain("npm install my-package");
+	});
+
+	it("sorts hits by score descending", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "npm install");
+		for (let i = 1; i < hits.length; i++) {
+			expect(hits[i].score).toBeLessThanOrEqual(hits[i - 1].score);
+		}
+	});
+
+	it("respects topN option", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "item", { topN: 1 });
+		expect(hits.length).toBeLessThanOrEqual(1);
+	});
+
+	it("returns table rows for table queries", () => {
+		const tree = buildTree(TABLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "speed fast");
+		expect(hits.length).toBeGreaterThan(0);
+		expect(hits[0].snippet.toLowerCase()).toContain("speed");
+	});
+
+	it("does not return duplicate ancestor/descendant pairs", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "npm install");
+		// No two hits where one path is a prefix of another at equal score
+		for (let i = 0; i < hits.length; i++) {
+			for (let j = 0; j < hits.length; j++) {
+				if (i === j) continue;
+				if (hits[j].path.startsWith(`${hits[i].path}.`) && hits[i].score >= hits[j].score) {
+					expect(false).toBe(true); // should have been deduplicated
+				}
+			}
+		}
+	});
+
+	it("snippet is non-empty for every hit", () => {
+		const tree = buildTree(SIMPLE_HTML, "https://example.com");
+		const hits = queryTree(tree, "install");
+		for (const h of hits) {
+			expect(h.snippet.trim().length).toBeGreaterThan(0);
+		}
+	});
+});

package/test/types.test.ts

@@ -0,0 +1,169 @@
+/**
+ * TDD tests for ImageRef and SpideredPage.images
+ */
+
+import { describe, expect, it } from "vitest";
+import type { ImageRef, SpideredPage } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Build a minimal valid SpideredPage for structural tests. */
+function makeMinimalPage(overrides: Partial<SpideredPage> = {}): SpideredPage {
+	return {
+		url: "https://example.com",
+		domain: "example.com",
+		fetchedAt: new Date().toISOString(),
+		title: "Test",
+		description: "",
+		author: "",
+		publishedAt: "",
+		lang: "en",
+		tags: [],
+		wordCount: 0,
+		readingTimeMinutes: 0,
+		headings: [],
+		chunks: [],
+		links: [],
+		markdown: "",
+		...overrides,
+	};
+}
+
+/** Runtime guard: at least one of base64 or filePath must be present. */
+function assertImageRefHasData(ref: ImageRef): void {
+	if (!ref.base64 && !ref.filePath) {
+		throw new Error("ImageRef must have at least one of base64 or filePath");
+	}
+}
+
+// ---------------------------------------------------------------------------
+// ImageRef structural tests
+// ---------------------------------------------------------------------------
+
+describe("ImageRef type", () => {
+	it("accepts a minimal ImageRef with only required fields", () => {
+		const ref: ImageRef = {
+			src: "https://example.com/photo.jpg",
+			mimeType: "image/jpeg",
+			alt: "A photo",
+		};
+		expect(ref.src).toBe("https://example.com/photo.jpg");
+		expect(ref.mimeType).toBe("image/jpeg");
+		expect(ref.alt).toBe("A photo");
+		expect(ref.base64).toBeUndefined();
+		expect(ref.filePath).toBeUndefined();
+	});
+
+	it("accepts an ImageRef with base64", () => {
+		const ref: ImageRef = {
+			src: "https://example.com/photo.jpg",
+			mimeType: "image/jpeg",
+			alt: "A photo",
+			base64: "abc123",
+		};
+		expect(ref.base64).toBe("abc123");
+	});
+
+	it("accepts an ImageRef with filePath", () => {
+		const ref: ImageRef = {
+			src: "https://example.com/large.jpg",
+			mimeType: "image/jpeg",
+			alt: "Large image",
+			filePath: "/home/user/.cache/web-spider/images/abc123.jpg",
+		};
+		expect(ref.filePath).toContain("abc123.jpg");
+	});
+
+	it("accepts an ImageRef with both base64 and filePath", () => {
+		const ref: ImageRef = {
+			src: "https://example.com/photo.jpg",
+			mimeType: "image/jpeg",
+			alt: "",
+			base64: "xyz",
+			filePath: "/tmp/xyz.jpg",
+		};
+		expect(ref.base64).toBe("xyz");
+		expect(ref.filePath).toBe("/tmp/xyz.jpg");
+	});
+
+	it("accepts empty string alt (no alt attribute)", () => {
+		const ref: ImageRef = {
+			src: "https://example.com/no-alt.jpg",
+			mimeType: "image/jpeg",
+			alt: "",
+		};
+		expect(ref.alt).toBe("");
+	});
+
+	it("accepts data: URL in src field", () => {
+		const ref: ImageRef = {
+			src: "data:image/png;base64,iVBORw0KGgo=",
+			mimeType: "image/png",
+			alt: "Inline image",
+			base64: "iVBORw0KGgo=",
+		};
+		expect(ref.src).toMatch(/^data:/);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Runtime guard: assertImageRefHasData
+// ---------------------------------------------------------------------------
+
+describe("ImageRef data guard", () => {
+	it("passes when base64 is present", () => {
+		const ref: ImageRef = { src: "https://x.com/a.jpg", mimeType: "image/jpeg", alt: "", base64: "abc" };
+		expect(() => assertImageRefHasData(ref)).not.toThrow();
+	});
+
+	it("passes when filePath is present", () => {
+		const ref: ImageRef = { src: "https://x.com/a.jpg", mimeType: "image/jpeg", alt: "", filePath: "/tmp/a.jpg" };
+		expect(() => assertImageRefHasData(ref)).not.toThrow();
+	});
+
+	it("throws when neither base64 nor filePath is present", () => {
+		const ref: ImageRef = { src: "https://x.com/a.jpg", mimeType: "image/jpeg", alt: "" };
+		expect(() => assertImageRefHasData(ref)).toThrow("at least one of base64 or filePath");
+	});
+});
+
+// ---------------------------------------------------------------------------
+// SpideredPage.images field
+// ---------------------------------------------------------------------------
+
+describe("SpideredPage.images field", () => {
+	it("is optional — SpideredPage without images is valid", () => {
+		const page = makeMinimalPage();
+		expect(page.images).toBeUndefined();
+	});
+
+	it("accepts an empty images array", () => {
+		const page = makeMinimalPage({ images: [] });
+		expect(page.images).toEqual([]);
+	});
+
+	it("accepts a populated images array", () => {
+		const images: ImageRef[] = [
+			{ src: "https://example.com/a.jpg", mimeType: "image/jpeg", alt: "A", base64: "abc" },
+			{ src: "https://example.com/b.png", mimeType: "image/png", alt: "B", base64: "def" },
+		];
+		const page = makeMinimalPage({ images });
+		expect(page.images).toHaveLength(2);
+		expect(page.images![0].src).toBe("https://example.com/a.jpg");
+		expect(page.images![1].mimeType).toBe("image/png");
+	});
+
+	it("produces a valid LLM data URL from an ImageRef", () => {
+		const ref: ImageRef = {
+			src: "https://example.com/photo.jpg",
+			mimeType: "image/jpeg",
+			alt: "Photo",
+			base64: "abc123XYZ",
+		};
+		const dataUrl = `data:${ref.mimeType};base64,${ref.base64}`;
+		expect(dataUrl).toBe("data:image/jpeg;base64,abc123XYZ");
+		expect(dataUrl).toMatch(/^data:image\//);
+	});
+});

package/test/web-search-integration.test.ts

@@ -0,0 +1,180 @@
+/**
+ * Integration tests — real network, real APIs.
+ *
+ * Each suite skips cleanly when its API key is absent so CI without secrets
+ * still passes. Run locally with keys set:
+ *
+ *   TAVILY_API_KEY=tvly-... vitest run test/web-search-integration.test.ts
+ *
+ * DDG tests always run — no key required.
+ */
+
+import { describe, expect, it } from "vitest";
+import type { WebSearchResult } from "../src/ports.js";
+import {
+	DdgSearchEngine,
+	FallbackSearchEngine,
+	TavilySearchEngine,
+	ddgSearch,
+	tavilySearch,
+	webSearch,
+} from "../src/web-search.js";
+
+// ---------------------------------------------------------------------------
+// Shared contract assertions
+// ---------------------------------------------------------------------------
+
+function assertResults(results: WebSearchResult[], minCount = 1) {
+	expect(Array.isArray(results)).toBe(true);
+	expect(results.length).toBeGreaterThanOrEqual(minCount);
+	for (const r of results) {
+		expect(typeof r.url).toBe("string");
+		expect(r.url).toMatch(/^https?:\/\//);
+		expect(typeof r.title).toBe("string");
+		expect(r.title.length).toBeGreaterThan(0);
+		expect(typeof r.snippet).toBe("string");
+	}
+}
+
+// ---------------------------------------------------------------------------
+// DuckDuckGo — no key required, always runs
+// ---------------------------------------------------------------------------
+
+describe("ddgSearch() — live DDG Instant Answer API", () => {
+	it("returns a parseable JSON response (catches Brotli/encoding issues)", async () => {
+		// This test exists specifically to catch the Node fetch + Brotli bug:
+		// DDG responds with Content-Encoding: br by default; undici won't
+		// decompress it, yielding an empty body and a JSON parse error.
+		// The fix: Accept-Encoding: gzip, deflate in the request headers.
+		const results = await ddgSearch("OpenAI", { numResults: 5 });
+		// DDG may return 0 results for some queries — just assert no throw
+		expect(Array.isArray(results)).toBe(true);
+	});
+
+	it("returns well-known entity results for an unambiguous query", async () => {
+		const results = await ddgSearch("Node.js", { numResults: 5 });
+		assertResults(results);
+	});
+
+	it("every result has a valid http(s) URL", async () => {
+		const results = await ddgSearch("TypeScript language", { numResults: 8 });
+		for (const r of results) {
+			expect(() => new URL(r.url)).not.toThrow();
+			expect(["http:", "https:"]).toContain(new URL(r.url).protocol);
+		}
+	});
+
+	it("respects numResults cap", async () => {
+		const results = await ddgSearch("JavaScript", { numResults: 3 });
+		expect(results.length).toBeLessThanOrEqual(3);
+	});
+
+	it("returns an empty array (not a throw) for a nonsense query", async () => {
+		const results = await ddgSearch("xyzzy-quux-frumious-bandersnatch-99999");
+		expect(Array.isArray(results)).toBe(true);
+	});
+
+	it("DdgSearchEngine.search() delegates correctly", async () => {
+		const engine = new DdgSearchEngine();
+		const results = await engine.search({ query: "OpenAI", numResults: 3 });
+		expect(Array.isArray(results)).toBe(true);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Tavily — skips when TAVILY_API_KEY is absent
+// ---------------------------------------------------------------------------
+
+const TAVILY_KEY = process.env["TAVILY_API_KEY"];
+const describeTavily = TAVILY_KEY ? describe : describe.skip;
+
+describeTavily("tavilySearch() — live Tavily API", () => {
+	it("returns results for a straightforward query", async () => {
+		const results = await tavilySearch("hexagonal architecture TypeScript", { numResults: 3 });
+		assertResults(results, 1);
+	});
+
+	it("every result has a valid URL and non-empty title", async () => {
+		const results = await tavilySearch("web scraping AI agents", { numResults: 5 });
+		assertResults(results, 1);
+	});
+
+	it("respects numResults", async () => {
+		const results = await tavilySearch("JavaScript", { numResults: 2 });
+		expect(results.length).toBeLessThanOrEqual(2);
+	});
+
+	it("TavilySearchEngine.search() delegates correctly", async () => {
+		const engine = new TavilySearchEngine(TAVILY_KEY!);
+		const results = await engine.search({ query: "DuckDuckGo API", numResults: 3 });
+		assertResults(results, 1);
+	});
+
+	it("throws a clear error when the key is wrong", async () => {
+		await expect(
+			tavilySearch("test", { apiKey: "tvly-invalid-key-000" }),
+		).rejects.toThrow(/tavily/i);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// FallbackSearchEngine — Tavily → DDG end-to-end
+// ---------------------------------------------------------------------------
+
+const describeFallback = TAVILY_KEY ? describe : describe.skip;
+
+describeFallback("FallbackSearchEngine — Tavily → DDG live chain", () => {
+	it("Tavily wins for a normal query (DDG never needed)", async () => {
+		const engine = new FallbackSearchEngine([
+			new TavilySearchEngine(TAVILY_KEY!),
+			new DdgSearchEngine(),
+		]);
+		const results = await engine.search({ query: "TypeScript strategy pattern", numResults: 3 });
+		assertResults(results, 1);
+	});
+
+	it("DDG provides results when Tavily is replaced with a failing stub", async () => {
+		const alwaysFails = { search: async () => { throw new Error("simulated Tavily outage"); } };
+		const engine = new FallbackSearchEngine([alwaysFails, new DdgSearchEngine()]);
+		// DDG may return empty for some queries but must not throw
+		const results = await engine.search({ query: "OpenAI", numResults: 3 });
+		expect(Array.isArray(results)).toBe(true);
+	});
+
+	it("DDG provides results when Tavily returns empty", async () => {
+		const alwaysEmpty = { search: async () => [] };
+		const engine = new FallbackSearchEngine([alwaysEmpty, new DdgSearchEngine()]);
+		const results = await engine.search({ query: "Node.js", numResults: 3 });
+		expect(Array.isArray(results)).toBe(true);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// webSearch() — auto-detect from env
+// ---------------------------------------------------------------------------
+
+const describeWebSearch = TAVILY_KEY ? describe : describe.skip;
+
+describeWebSearch("webSearch() — auto-detects Tavily from env", () => {
+	it("returns results without specifying an engine", async () => {
+		const results = await webSearch("open source web crawler", { numResults: 3 });
+		assertResults(results, 1);
+	});
+
+	it("returns results when engine is forced to 'tavily'", async () => {
+		const results = await webSearch("AI coding assistant", { engine: "tavily", numResults: 3 });
+		assertResults(results, 1);
+	});
+
+	it("returns results when engine is forced to 'ddg'", async () => {
+		const results = await webSearch("OpenAI", { engine: "ddg", numResults: 5 });
+		expect(Array.isArray(results)).toBe(true);
+	});
+
+	it("throws a descriptive error when forced to 'brave' with no key set", async () => {
+		const saved = process.env["BRAVE_SEARCH_API_KEY"];
+		delete process.env["BRAVE_SEARCH_API_KEY"];
+		await expect(webSearch("test", { engine: "brave" })).rejects.toThrow("BRAVE_SEARCH_API_KEY");
+		if (saved) process.env["BRAVE_SEARCH_API_KEY"] = saved;
+	});
+});