@dpopsuev/web-spider 0.10.4 → 0.10.5

package/test/inbound-count.test.ts

@@ -1,111 +0,0 @@
-/**
- * TDD tests for LeanPage.inboundCount via PageGraph.
- */
-
-import { describe, expect, it } from "vitest";
-import { PageGraph } from "../src/graph.js";
-import type { SpideredPage } from "../src/types.js";
-import { toLean } from "../src/views.js";
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-function makePage(url: string, linksTo: string[] = []): SpideredPage {
-	return {
-		url,
-		domain: new URL(url).hostname,
-		fetchedAt: new Date().toISOString(),
-		title: `Page at ${url}`,
-		description: "",
-		author: "",
-		publishedAt: "",
-		lang: "en",
-		tags: [],
-		wordCount: 100,
-		readingTimeMinutes: 1,
-		headings: [],
-		chunks: [],
-		links: linksTo.map((href) => ({ href, text: href, isExternal: false, rel: "body" as const })),
-		markdown: "",
-	};
-}
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-describe("LeanPage.inboundCount via PageGraph", () => {
-	it("omitted when no graph is passed to toLean()", () => {
-		const lean = toLean(makePage("https://example.com"));
-		expect(lean.inboundCount).toBeUndefined();
-	});
-
-	it("0 when page has no inbound links", () => {
-		const graph = new PageGraph();
-		const page = makePage("https://example.com");
-		graph.addPage(page);
-		const lean = toLean(page, graph);
-		expect(lean.inboundCount).toBe(0);
-	});
-
-	it("1 when one page links to this page", () => {
-		const graph = new PageGraph();
-		const home = makePage("https://example.com", ["https://example.com/about"]);
-		const about = makePage("https://example.com/about");
-		graph.addPage(home);
-		graph.addPage(about);
-		const lean = toLean(about, graph);
-		expect(lean.inboundCount).toBe(1);
-	});
-
-	it("counts multiple inbound links correctly", () => {
-		const graph = new PageGraph();
-		const target = makePage("https://example.com/popular");
-		const a = makePage("https://example.com/a", ["https://example.com/popular"]);
-		const b = makePage("https://example.com/b", ["https://example.com/popular"]);
-		const c = makePage("https://example.com/c", ["https://example.com/popular"]);
-		[target, a, b, c].forEach((p) => graph.addPage(p));
-		const lean = toLean(target, graph);
-		expect(lean.inboundCount).toBe(3);
-	});
-
-	it("inboundCount on a hub page is 0 (only outbound)", () => {
-		const graph = new PageGraph();
-		const hub = makePage("https://example.com", [
-			"https://example.com/a",
-			"https://example.com/b",
-		]);
-		graph.addPage(hub);
-		const lean = toLean(hub, graph);
-		expect(lean.inboundCount).toBe(0);
-	});
-
-	it("pages ranked by inboundCount descending matches graph.byPageRank()", () => {
-		const graph = new PageGraph();
-		const popular = makePage("https://example.com/popular");
-		const normal = makePage("https://example.com/normal");
-		const a = makePage("https://example.com/a", ["https://example.com/popular"]);
-		const b = makePage("https://example.com/b", ["https://example.com/popular"]);
-		const c = makePage("https://example.com/c", ["https://example.com/normal"]);
-		[popular, normal, a, b, c].forEach((p) => graph.addPage(p));
-
-		const popularLean = toLean(popular, graph);
-		const normalLean = toLean(normal, graph);
-
-		expect(popularLean.inboundCount).toBe(2);
-		expect(normalLean.inboundCount).toBe(1);
-		expect(popularLean.inboundCount!).toBeGreaterThan(normalLean.inboundCount!);
-	});
-
-	it("all other LeanPage fields are still populated when graph is provided", () => {
-		const graph = new PageGraph();
-		const page = makePage("https://example.com");
-		graph.addPage(page);
-		const lean = toLean(page, graph);
-		expect(lean.url).toBe("https://example.com");
-		expect(lean.title).toBeTruthy();
-		expect(lean.view).toBe("lean");
-		expect(typeof lean.wordCount).toBe("number");
-	});
-});

package/test/lean.test.ts

@@ -1,105 +0,0 @@
-import { readFileSync } from "fs";
-import { dirname, join } from "path";
-import { fileURLToPath } from "url";
-import { describe, expect, it } from "vitest";
-import type { LeanPage, SpideredPage } from "../src/types.js";
-import { toLean } from "../src/views.js";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-function loadFixture(name: string): SpideredPage {
-	const raw = readFileSync(join(__dirname, "../fixtures", name), "utf8");
-	return JSON.parse(raw) as SpideredPage;
-}
-
-const guide = loadFixture("guide-ai-agents-web-scraping.json");
-
-describe("toLean", () => {
-	let lean: LeanPage;
-
-	it("produces a lean page without error", () => {
-		lean = toLean(guide);
-		expect(lean).toBeDefined();
-	});
-
-	it("sets view discriminant to 'lean'", () => {
-		lean = toLean(guide);
-		expect(lean.view).toBe("lean");
-	});
-
-	it("preserves identity and metadata fields", () => {
-		lean = toLean(guide);
-		expect(lean.url).toBe(guide.url);
-		expect(lean.domain).toBe(guide.domain);
-		expect(lean.title).toBe(guide.title);
-		expect(lean.lang).toBe(guide.lang);
-		expect(lean.wordCount).toBe(guide.wordCount);
-		expect(lean.readingTimeMinutes).toBe(guide.readingTimeMinutes);
-		// fetchedAt is intentionally excluded from lean (noise in agent context)
-		expect(lean).not.toHaveProperty("fetchedAt");
-	});
-
-	it("converts headings to flat markdown strings", () => {
-		lean = toLean(guide);
-		// Fixture has all level-2 headings
-		expect(lean.headings).toBeInstanceOf(Array);
-		expect(lean.headings.length).toBe(guide.headings.length);
-		for (const h of lean.headings) {
-			expect(typeof h).toBe("string");
-			expect(h).toMatch(/^#{1,3} .+/);
-		}
-		// Spot-check first heading
-		const first = guide.headings[0];
-		expect(lean.headings[0]).toBe(`${"#".repeat(first.level)} ${first.text}`);
-	});
-
-	it("strips isExternal from links", () => {
-		lean = toLean(guide);
-		expect(lean.links.length).toBeGreaterThan(0);
-		for (const link of lean.links) {
-			expect(link).toHaveProperty("href");
-			expect(link).toHaveProperty("text");
-			expect(link).not.toHaveProperty("isExternal");
-		}
-	});
-
-	it("passes tags through from source page", () => {
-		lean = toLean(guide);
-		expect(Array.isArray(lean.tags)).toBe(true);
-	});
-
-	it("passes canonicalUrl when present", () => {
-		const withCanonical = { ...guide, canonicalUrl: "https://example.com/canonical" };
-		const l = toLean(withCanonical);
-		expect(l.canonicalUrl).toBe("https://example.com/canonical");
-	});
-
-	it("omits canonicalUrl when absent", () => {
-		const withoutCanonical = { ...guide };
-		delete (withoutCanonical as Partial<typeof guide>).canonicalUrl;
-		const l = toLean(withoutCanonical);
-		expect(l.canonicalUrl).toBeUndefined();
-	});
-
-	it("sets chunkCount from the source chunks array", () => {
-		lean = toLean(guide);
-		expect(lean.chunkCount).toBe(guide.chunks.length);
-		expect(lean.chunkCount).toBeGreaterThan(0);
-	});
-
-	it("omits chunks and markdown fields", () => {
-		lean = toLean(guide);
-		expect(lean).not.toHaveProperty("chunks");
-		expect(lean).not.toHaveProperty("markdown");
-	});
-
-	it("is materially smaller than the full page", () => {
-		lean = toLean(guide);
-		const fullSize = JSON.stringify(guide).length;
-		const leanSize = JSON.stringify(lean).length;
-		// Lean should be less than 30% the size of full.
-		// Links (up to 200) still carry href+text so the floor isn't zero,
-		// but chunks and markdown — the two biggest fields — are gone.
-		expect(leanSize).toBeLessThan(fullSize * 0.3);
-	});
-});

package/test/playwright.test.ts

@@ -1,128 +0,0 @@
-/**
- * TDD tests for PlaywrightHttpClient captureImages option.
- *
- * No real browser is launched. We test:
- *   1. Option wiring — captureImages is stored and readable.
- *   2. Route logic — the abort/continue decision function in isolation.
- *   3. Interface conformance — PlaywrightHttpClient still satisfies IHttpClient.
- */
-
-import { describe, expect, it } from "vitest";
-import { PlaywrightHttpClient, createPlaywrightClient } from "../src/playwright.js";
-import type { IHttpClient } from "../src/ports.js";
-
-// ---------------------------------------------------------------------------
-// Extract the routing decision as a pure function so we can test it without
-// launching a browser. Mirrors the logic in playwright.ts fetch().
-// ---------------------------------------------------------------------------
-
-function shouldAbort(
-	resourceType: string,
-	acceptHeader: string,
-	captureImages: boolean,
-): boolean {
-	const isImageFetch = acceptHeader.startsWith("image/");
-	if (resourceType === "font") return true;
-	if (["image", "media"].includes(resourceType) && !(captureImages && isImageFetch)) return true;
-	return false;
-}
-
-// ---------------------------------------------------------------------------
-// Route logic — pure unit tests, no browser
-// ---------------------------------------------------------------------------
-
-describe("Playwright route abort logic", () => {
-	describe("fonts — always aborted", () => {
-		it("aborts font with captureImages: false", () => {
-			expect(shouldAbort("font", "", false)).toBe(true);
-		});
-		it("aborts font with captureImages: true", () => {
-			expect(shouldAbort("font", "image/*", true)).toBe(true);
-		});
-	});
-
-	describe("images — aborted unless captureImages + Accept: image/*", () => {
-		it("aborts image with captureImages: false", () => {
-			expect(shouldAbort("image", "", false)).toBe(true);
-		});
-		it("aborts image with captureImages: false even if Accept: image/*", () => {
-			expect(shouldAbort("image", "image/*", false)).toBe(true);
-		});
-		it("aborts image with captureImages: true but no image Accept header", () => {
-			expect(shouldAbort("image", "text/html", true)).toBe(true);
-		});
-		it("allows image with captureImages: true AND Accept: image/*", () => {
-			expect(shouldAbort("image", "image/*", true)).toBe(false);
-		});
-		it("allows image with captureImages: true AND Accept: image/jpeg", () => {
-			expect(shouldAbort("image", "image/jpeg", true)).toBe(false);
-		});
-		it("allows image with captureImages: true AND Accept: image/png", () => {
-			expect(shouldAbort("image", "image/png", true)).toBe(false);
-		});
-	});
-
-	describe("media — same rules as image", () => {
-		it("aborts media with captureImages: false", () => {
-			expect(shouldAbort("media", "", false)).toBe(true);
-		});
-		it("aborts media with captureImages: true but no image Accept", () => {
-			expect(shouldAbort("media", "video/mp4", true)).toBe(true);
-		});
-		it("allows media with captureImages: true AND Accept: image/*", () => {
-			expect(shouldAbort("media", "image/*", true)).toBe(false);
-		});
-	});
-
-	describe("other resource types — never aborted", () => {
-		it.each(["document", "stylesheet", "script", "xhr", "fetch", "websocket"])(
-			"allows %s regardless of captureImages",
-			(type) => {
-				expect(shouldAbort(type, "", false)).toBe(false);
-				expect(shouldAbort(type, "", true)).toBe(false);
-			},
-		);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Option wiring — captureImages stored on the instance
-// ---------------------------------------------------------------------------
-
-describe("PlaywrightHttpClient option wiring", () => {
-	it("defaults captureImages to false", () => {
-		const client = new PlaywrightHttpClient();
-		// Access via cast — private field, but we verify the default behaviour
-		// through the public interface indirectly. Here we just confirm construction.
-		expect(client).toBeInstanceOf(PlaywrightHttpClient);
-	});
-
-	it("constructs with captureImages: true without throwing", () => {
-		expect(() => new PlaywrightHttpClient({ captureImages: true })).not.toThrow();
-	});
-
-	it("constructs with captureImages: false without throwing", () => {
-		expect(() => new PlaywrightHttpClient({ captureImages: false })).not.toThrow();
-	});
-
-	it("createPlaywrightClient passes captureImages through", () => {
-		const client = createPlaywrightClient({ captureImages: true });
-		expect(client).toBeInstanceOf(PlaywrightHttpClient);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Interface conformance
-// ---------------------------------------------------------------------------
-
-describe("PlaywrightHttpClient interface conformance", () => {
-	it("satisfies IHttpClient", () => {
-		const client: IHttpClient = new PlaywrightHttpClient();
-		expect(typeof client.fetch).toBe("function");
-	});
-
-	it("has a close() method", () => {
-		const client = new PlaywrightHttpClient();
-		expect(typeof client.close).toBe("function");
-	});
-});

package/test/ports.test.ts

@@ -1,161 +0,0 @@
-/**
- * TDD tests for HttpResponse.arrayBuffer()
- *
- * All tests use stub HTTP clients — no real network.
- */
-
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-import { describe, expect, it } from "vitest";
-import type { HttpResponse, IHttpClient } from "../src/ports.js";
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-function makeStubResponse(overrides: Partial<HttpResponse> = {}): HttpResponse {
-	return {
-		ok: true,
-		status: 200,
-		statusText: "OK",
-		headers: { get: () => null },
-		text: async () => "",
-		arrayBuffer: async () => new ArrayBuffer(0),
-		...overrides,
-	};
-}
-
-// ---------------------------------------------------------------------------
-// Interface conformance (TypeScript structural check via satisfies)
-// ---------------------------------------------------------------------------
-
-describe("HttpResponse interface", () => {
-	it("stub with arrayBuffer() satisfies HttpResponse", () => {
-		const stub = {
-			ok: true,
-			status: 200,
-			statusText: "OK",
-			headers: { get: (_name: string) => null as string | null },
-			text: async () => "",
-			arrayBuffer: async () => new ArrayBuffer(4),
-		} satisfies HttpResponse;
-
-		expect(typeof stub.arrayBuffer).toBe("function");
-	});
-
-	it("IHttpClient stub with arrayBuffer-returning fetch satisfies the port", () => {
-		const client: IHttpClient = {
-			fetch: async (_req) => makeStubResponse({ arrayBuffer: async () => new ArrayBuffer(8) }),
-		};
-		expect(typeof client.fetch).toBe("function");
-	});
-});
-
-// ---------------------------------------------------------------------------
-// arrayBuffer() returns correct bytes
-// ---------------------------------------------------------------------------
-
-describe("arrayBuffer() byte content", () => {
-	it("resolves to an ArrayBuffer", async () => {
-		const response = makeStubResponse({
-			arrayBuffer: async () => new ArrayBuffer(4),
-		});
-		const buf = await response.arrayBuffer();
-		expect(buf).toBeInstanceOf(ArrayBuffer);
-	});
-
-	it("returns the correct byte length", async () => {
-		const response = makeStubResponse({
-			arrayBuffer: async () => new ArrayBuffer(16),
-		});
-		const buf = await response.arrayBuffer();
-		expect(buf.byteLength).toBe(16);
-	});
-
-	it("returns correct bytes from a known fixture", async () => {
-		const tinyPng = readFileSync(
-			join(import.meta.dirname, "../fixtures/images/tiny.png"),
-		);
-		const expected = tinyPng.buffer.slice(
-			tinyPng.byteOffset,
-			tinyPng.byteOffset + tinyPng.byteLength,
-		) as ArrayBuffer;
-
-		const response = makeStubResponse({
-			arrayBuffer: async () => expected,
-		});
-
-		const buf = await response.arrayBuffer();
-		expect(buf.byteLength).toBe(tinyPng.byteLength);
-
-		const view = new Uint8Array(buf);
-		// PNG magic bytes: 0x89 0x50 0x4E 0x47
-		expect(view[0]).toBe(0x89);
-		expect(view[1]).toBe(0x50); // P
-		expect(view[2]).toBe(0x4e); // N
-		expect(view[3]).toBe(0x47); // G
-	});
-
-	it("returns zero-length buffer when resource is empty", async () => {
-		const response = makeStubResponse({
-			arrayBuffer: async () => new ArrayBuffer(0),
-		});
-		const buf = await response.arrayBuffer();
-		expect(buf.byteLength).toBe(0);
-	});
-});
-
-// ---------------------------------------------------------------------------
-// Default fetch adapter — arrayBuffer() on real tiny PNG bytes
-// ---------------------------------------------------------------------------
-
-describe("default fetch adapter arrayBuffer() via mock client", () => {
-	it("mock client that returns tiny.png bytes produces correct ArrayBuffer", async () => {
-		const tinyPng = readFileSync(
-			join(import.meta.dirname, "../fixtures/images/tiny.png"),
-		);
-
-		const client: IHttpClient = {
-			fetch: async (_req) =>
-				makeStubResponse({
-					arrayBuffer: async () =>
-						tinyPng.buffer.slice(
-							tinyPng.byteOffset,
-							tinyPng.byteOffset + tinyPng.byteLength,
-						) as ArrayBuffer,
-				}),
-		};
-
-		const res = await client.fetch({ url: "https://example.com/tiny.png" });
-		const buf = await res.arrayBuffer();
-
-		expect(buf.byteLength).toBe(tinyPng.byteLength);
-		const view = new Uint8Array(buf);
-		expect(view[0]).toBe(0x89); // PNG magic
-	});
-
-	it("base64-encoding an ArrayBuffer from mock produces correct data URL prefix", async () => {
-		const tinyPng = readFileSync(
-			join(import.meta.dirname, "../fixtures/images/tiny.png"),
-		);
-
-		const client: IHttpClient = {
-			fetch: async (_req) =>
-				makeStubResponse({
-					arrayBuffer: async () =>
-						tinyPng.buffer.slice(
-							tinyPng.byteOffset,
-							tinyPng.byteOffset + tinyPng.byteLength,
-						) as ArrayBuffer,
-				}),
-		};
-
-		const res = await client.fetch({ url: "https://example.com/tiny.png" });
-		const buf = await res.arrayBuffer();
-		const b64 = Buffer.from(buf).toString("base64");
-		const dataUrl = `data:image/png;base64,${b64}`;
-
-		expect(dataUrl).toMatch(/^data:image\/png;base64,/);
-		expect(b64.length).toBeGreaterThan(0);
-	});
-});