@sourcepress/knowledge 0.1.0

package/src/__tests__/graph-builder.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, it } from "vitest";
+import { GraphBuilder } from "../graph-builder.js";
+import type { ExtractedEntity, ExtractedRelation } from "../types.js";
+
+function makeEntities(): ExtractedEntity[] {
+	return [
+		{
+			type: "client",
+			name: "Acme Corp",
+			aliases: ["Acme", "ACME"],
+			confidence: 0.95,
+			source_file: "knowledge/clients/acme.md",
+		},
+		{
+			type: "technology",
+			name: "Next.js",
+			aliases: ["NextJS"],
+			confidence: 0.99,
+			source_file: "knowledge/clients/acme.md",
+		},
+		{
+			type: "technology",
+			name: "React",
+			aliases: [],
+			confidence: 0.9,
+			source_file: "knowledge/tech/react.md",
+		},
+		{
+			type: "service",
+			name: "Web Development",
+			aliases: ["Webbutveckling"],
+			confidence: 0.85,
+			source_file: "knowledge/services/web.md",
+		},
+	];
+}
+
+function makeRelations(): ExtractedRelation[] {
+	return [
+		{
+			from_entity: "Acme Corp",
+			to_entity: "Next.js",
+			relation_type: "uses",
+			confidence: 0.9,
+			evidence: "Acme Corp is migrating to Next.js",
+			source_file: "knowledge/clients/acme.md",
+		},
+		{
+			from_entity: "Next.js",
+			to_entity: "React",
+			relation_type: "built_on",
+			confidence: 0.99,
+			evidence: "Next.js is a React framework",
+			source_file: "knowledge/tech/react.md",
+		},
+		{
+			from_entity: "Acme Corp",
+			to_entity: "Web Development",
+			relation_type: "needs",
+			confidence: 0.85,
+			evidence: "Acme needs web development services",
+			source_file: "knowledge/clients/acme.md",
+		},
+	];
+}
+
+describe("GraphBuilder", () => {
+	it("builds a graph from entities and relations", () => {
+		const builder = new GraphBuilder();
+		builder.addEntities(makeEntities());
+		builder.addRelations(makeRelations());
+		const graph = builder.build();
+		expect(graph.entities.size).toBe(4);
+		expect(graph.relations).toHaveLength(3);
+		expect(graph.file_count).toBe(3);
+		expect(graph.built_at).toBeTruthy();
+	});
+
+	it("deduplicates entities by name", () => {
+		const builder = new GraphBuilder();
+		builder.addEntities(makeEntities());
+		builder.addEntities([
+			{
+				type: "client",
+				name: "Acme Corp",
+				aliases: ["Acme Inc"],
+				confidence: 0.8,
+				source_file: "knowledge/other.md",
+			},
+		]);
+		const graph = builder.build();
+		const acme = graph.entities.get("Acme Corp");
+		expect(acme).toBeDefined();
+		// biome-ignore lint/style/noNonNullAssertion: asserted defined on line above
+		expect(acme!.confidence).toBe(0.95);
+		// biome-ignore lint/style/noNonNullAssertion: asserted defined on line above
+		expect(acme!.aliases).toContain("Acme");
+		// biome-ignore lint/style/noNonNullAssertion: asserted defined on line above
+		expect(acme!.aliases).toContain("ACME");
+		// biome-ignore lint/style/noNonNullAssertion: asserted defined on line above
+		expect(acme!.aliases).toContain("Acme Inc");
+	});
+
+	it("filters relations below confidence threshold", () => {
+		const builder = new GraphBuilder({ minConfidence: 0.88 });
+		builder.addEntities(makeEntities());
+		builder.addRelations(makeRelations());
+		const graph = builder.build();
+		expect(graph.relations).toHaveLength(2);
+		expect(graph.relations.every((r) => r.confidence >= 0.88)).toBe(true);
+	});
+
+	it("computes clusters from connected entities", () => {
+		const builder = new GraphBuilder();
+		builder.addEntities(makeEntities());
+		builder.addRelations(makeRelations());
+		const graph = builder.build();
+		expect(graph.clusters.length).toBeGreaterThanOrEqual(1);
+		expect(graph.clusters[0].entities.length).toBeGreaterThanOrEqual(2);
+	});
+
+	it("handles empty input", () => {
+		const builder = new GraphBuilder();
+		const graph = builder.build();
+		expect(graph.entities.size).toBe(0);
+		expect(graph.relations).toHaveLength(0);
+		expect(graph.clusters).toHaveLength(0);
+	});
+});

package/src/__tests__/graph-ops.test.ts

@@ -0,0 +1,189 @@
+import type { ContentFile } from "@sourcepress/core";
+import { beforeEach, describe, expect, it } from "vitest";
+import { GraphOps } from "../graph-ops.js";
+import type { ExtractedEntity, ExtractedRelation, KnowledgeGraph } from "../types.js";
+
+function makeGraph(): KnowledgeGraph {
+	const entities = new Map<string, ExtractedEntity>([
+		[
+			"Acme Corp",
+			{
+				type: "client",
+				name: "Acme Corp",
+				aliases: ["Acme"],
+				confidence: 0.95,
+				source_file: "knowledge/clients/acme.md",
+			},
+		],
+		[
+			"Next.js",
+			{
+				type: "technology",
+				name: "Next.js",
+				aliases: ["NextJS"],
+				confidence: 0.99,
+				source_file: "knowledge/tech/nextjs.md",
+			},
+		],
+		[
+			"React",
+			{
+				type: "technology",
+				name: "React",
+				aliases: [],
+				confidence: 0.9,
+				source_file: "knowledge/tech/react.md",
+			},
+		],
+		[
+			"Orphan Entity",
+			{
+				type: "concept",
+				name: "Orphan Entity",
+				aliases: [],
+				confidence: 0.7,
+				source_file: "knowledge/misc/orphan.md",
+			},
+		],
+	]);
+	const relations: ExtractedRelation[] = [
+		{
+			from_entity: "Acme Corp",
+			to_entity: "Next.js",
+			relation_type: "uses",
+			confidence: 0.9,
+			evidence: "Acme uses Next.js",
+			source_file: "knowledge/clients/acme.md",
+		},
+		{
+			from_entity: "Next.js",
+			to_entity: "React",
+			relation_type: "built_on",
+			confidence: 0.99,
+			evidence: "Next.js is built on React",
+			source_file: "knowledge/tech/nextjs.md",
+		},
+	];
+	return {
+		entities,
+		relations,
+		clusters: [
+			{
+				id: "cluster-0",
+				name: "Acme Corp",
+				entities: ["Acme Corp", "Next.js", "React"],
+				coherence_score: 0.95,
+			},
+		],
+		built_at: "2026-04-04T10:00:00Z",
+		file_count: 4,
+	};
+}
+
+function makeContentFiles(): ContentFile[] {
+	return [
+		{
+			collection: "cases",
+			slug: "acme",
+			path: "content/cases/acme.mdx",
+			frontmatter: { title: "Acme Corp Case Study", client: "acme-corp" },
+			body: "Acme Corp migrated to Next.js with our help.",
+			provenance: {
+				generated_by: "claude-sonnet-4-5-20250514",
+				generated_at: "2026-04-01T10:00:00Z",
+				source_files: ["knowledge/clients/acme.md"],
+			},
+		},
+		{
+			collection: "posts",
+			slug: "react-guide",
+			path: "content/posts/react-guide.mdx",
+			frontmatter: { title: "React Guide" },
+			body: "A guide to React.",
+			provenance: {
+				generated_by: "claude-sonnet-4-5-20250514",
+				generated_at: "2026-04-03T10:00:00Z",
+				source_files: ["knowledge/tech/react.md"],
+			},
+		},
+	];
+}
+
+describe("GraphOps", () => {
+	let ops: GraphOps;
+	beforeEach(() => {
+		ops = new GraphOps(makeGraph());
+	});
+
+	describe("query", () => {
+		it("returns entity with relations and related entities", () => {
+			const result = ops.query("Acme Corp");
+			expect(result).not.toBeNull();
+			// biome-ignore lint/style/noNonNullAssertion: asserted not null on line above
+			expect(result!.entity.name).toBe("Acme Corp");
+			// biome-ignore lint/style/noNonNullAssertion: asserted not null on line above
+			expect(result!.relations).toHaveLength(1);
+			// biome-ignore lint/style/noNonNullAssertion: asserted not null on line above
+			expect(result!.related_entities).toHaveLength(1);
+			// biome-ignore lint/style/noNonNullAssertion: asserted not null on line above
+			expect(result!.related_entities[0].name).toBe("Next.js");
+		});
+		it("returns null for unknown entity", () => {
+			expect(ops.query("Unknown")).toBeNull();
+		});
+		it("finds entity by alias", () => {
+			const result = ops.query("Acme");
+			expect(result).not.toBeNull();
+			// biome-ignore lint/style/noNonNullAssertion: asserted not null on line above
+			expect(result!.entity.name).toBe("Acme Corp");
+		});
+		it("collects source files for entity", () => {
+			const result = ops.query("Next.js");
+			expect(result).not.toBeNull();
+			// biome-ignore lint/style/noNonNullAssertion: asserted not null on line above
+			expect(result!.files).toContain("knowledge/tech/nextjs.md");
+			// biome-ignore lint/style/noNonNullAssertion: asserted not null on line above
+			expect(result!.files).toContain("knowledge/clients/acme.md");
+		});
+	});
+
+	describe("findGaps", () => {
+		it("finds entities with knowledge but no content", () => {
+			const gaps = ops.findGaps(makeContentFiles());
+			expect(gaps.length).toBeGreaterThanOrEqual(1);
+			const orphanGap = gaps.find((g) => g.entity_name === "Orphan Entity");
+			expect(orphanGap).toBeDefined();
+			// biome-ignore lint/style/noNonNullAssertion: asserted defined on line above
+			expect(orphanGap!.content_file_count).toBe(0);
+		});
+	});
+
+	describe("findStale", () => {
+		it("detects content older than source knowledge", () => {
+			const stale = ops.findStale(makeContentFiles(), {
+				"knowledge/clients/acme.md": "2026-04-03T10:00:00Z",
+				"knowledge/tech/react.md": "2026-04-02T10:00:00Z",
+			});
+			expect(stale).toHaveLength(1);
+			expect(stale[0].content_path).toBe("content/cases/acme.mdx");
+			expect(stale[0].stale_sources).toContain("knowledge/clients/acme.md");
+		});
+		it("returns empty when all content is fresh", () => {
+			const stale = ops.findStale(makeContentFiles(), {
+				"knowledge/clients/acme.md": "2026-03-01T10:00:00Z",
+				"knowledge/tech/react.md": "2026-03-01T10:00:00Z",
+			});
+			expect(stale).toHaveLength(0);
+		});
+	});
+
+	describe("getStats", () => {
+		it("returns graph statistics", () => {
+			const stats = ops.getStats();
+			expect(stats.entity_count).toBe(4);
+			expect(stats.relation_count).toBe(2);
+			expect(stats.cluster_count).toBe(1);
+			expect(stats.file_count).toBe(4);
+		});
+	});
+});

package/src/__tests__/ingestion.test.ts

@@ -0,0 +1,127 @@
+import type { BudgetTracker, ResolvedProvider } from "@sourcepress/ai";
+import { describe, expect, it, vi } from "vitest";
+import { KnowledgeEngine } from "../knowledge-engine.js";
+import { InMemoryKnowledgeStore } from "../knowledge-store.js";
+
+// Mock AI functions
+vi.mock("@sourcepress/ai", () => ({
+	classify: vi.fn().mockResolvedValue({
+		type: "article",
+		quality: "structured",
+		quality_score: 0.8,
+	}),
+	extract: vi.fn().mockResolvedValue({
+		entities: [{ type: "company", name: "Acme Corp", confidence: 0.9 }],
+		relations: [],
+	}),
+}));
+
+const ARTICLE_HTML = `<!DOCTYPE html>
+<html><head><title>Acme Services</title></head>
+<body><article>
+<h1>Acme Corp Services</h1>
+<p>Acme Corp provides enterprise software solutions for modern businesses.
+Their platform handles everything from customer management to analytics.
+Founded in 2010, Acme has grown to serve over 500 enterprise clients
+worldwide with their innovative approach to business software.</p>
+<h2>Key Offerings</h2>
+<p>The company specializes in cloud-native applications, data pipelines,
+and AI-powered automation tools that help businesses scale efficiently.</p>
+</article></body></html>`;
+
+const SITEMAP_XML = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://acme.com/services/web</loc></url>
+  <url><loc>https://acme.com/services/cloud</loc></url>
+  <url><loc>https://acme.com/blog/post-1</loc></url>
+</urlset>`;
+
+function createMockFetcher() {
+	return vi.fn().mockImplementation(async (url: string) => {
+		if (url.endsWith("sitemap.xml")) {
+			return { ok: true, status: 200, text: () => Promise.resolve(SITEMAP_XML) };
+		}
+		return { ok: true, status: 200, text: () => Promise.resolve(ARTICLE_HTML) };
+	}) as unknown as typeof globalThis.fetch;
+}
+
+function createEngine(fetcher?: typeof globalThis.fetch) {
+	const store = new InMemoryKnowledgeStore();
+	const mockProvider = {
+		provider: "mock",
+		model: "mock",
+		apiKey: "mock",
+	} as unknown as ResolvedProvider;
+	const mockBudget = { check: vi.fn(), record: vi.fn() } as unknown as BudgetTracker;
+	return new KnowledgeEngine(store, mockProvider, mockBudget, fetcher);
+}
+
+describe("KnowledgeEngine ingestion", () => {
+	it("importUrl scrapes and stores a single URL", async () => {
+		const fetcher = createMockFetcher();
+		const engine = createEngine(fetcher);
+		const result = await engine.importUrl("https://acme.com/services/web");
+
+		expect(result.path).toBe("knowledge/acme.com/services/web.md");
+		expect(result.source).toBe("url");
+		expect(result.source_url).toBe("https://acme.com/services/web");
+		expect(result.body).toContain("Acme Corp");
+	});
+
+	it("parseSitemap returns grouped sections", async () => {
+		const fetcher = createMockFetcher();
+		const engine = createEngine(fetcher);
+		const result = await engine.parseSitemap("https://acme.com/sitemap.xml");
+
+		expect(result.total_urls).toBe(3);
+		const services = result.sections.find((s) => s.pattern === "/services/*");
+		expect(services).toBeDefined();
+		expect(services?.count).toBe(2);
+	});
+
+	it("importBatch processes multiple URLs with progress", async () => {
+		const fetcher = createMockFetcher();
+		const engine = createEngine(fetcher);
+		const progress = vi.fn();
+
+		const result = await engine.importBatch(
+			["https://acme.com/services/web", "https://acme.com/services/cloud"],
+			progress,
+		);
+
+		expect(result.completed).toBe(2);
+		expect(result.failed).toBe(0);
+		expect(result.results).toHaveLength(2);
+		expect(progress).toHaveBeenCalledTimes(2);
+	});
+
+	it("importBatch handles failures gracefully", async () => {
+		const fetcher = vi
+			.fn()
+			.mockResolvedValueOnce({ ok: true, status: 200, text: () => Promise.resolve(ARTICLE_HTML) })
+			.mockResolvedValueOnce({
+				ok: false,
+				status: 500,
+				statusText: "Server Error",
+			}) as unknown as typeof globalThis.fetch;
+
+		const engine = createEngine(fetcher);
+		const result = await engine.importBatch([
+			"https://acme.com/services/web",
+			"https://acme.com/broken",
+		]);
+
+		expect(result.completed).toBe(2);
+		expect(result.failed).toBe(1);
+		expect(result.results[0].success).toBe(true);
+		expect(result.results[1].success).toBe(false);
+		expect(result.results[1].error).toContain("500");
+	});
+
+	it("urlToPath converts URLs to knowledge paths", async () => {
+		const fetcher = createMockFetcher();
+		const engine = createEngine(fetcher);
+		const result = await engine.importUrl("https://example.com/blog/my-post");
+		expect(result.path).toBe("knowledge/example.com/blog/my-post.md");
+	});
+});

package/src/__tests__/json-file-store.test.ts

@@ -0,0 +1,206 @@
+import { existsSync, mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { KnowledgeFile } from "@sourcepress/core";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { JsonFileStore } from "../json-file-store.js";
+import type { KnowledgeGraph } from "../types.js";
+
+function makeKnowledgeFile(overrides: Partial<KnowledgeFile> = {}): KnowledgeFile {
+	return {
+		path: "knowledge/clients/acme.md",
+		type: "project-notes",
+		quality: "structured",
+		quality_score: 8,
+		entities: [
+			{ type: "client", name: "Acme Corp" },
+			{ type: "technology", name: "Next.js" },
+		],
+		ingested_at: "2026-04-04T10:00:00Z",
+		source: "manual",
+		body: "Meeting with Acme Corp about Next.js migration.",
+		...overrides,
+	};
+}
+
+function makeGraph(): KnowledgeGraph {
+	return {
+		entities: new Map([
+			[
+				"acme-corp",
+				{
+					type: "client",
+					name: "Acme Corp",
+					aliases: ["Acme"],
+					confidence: 0.95,
+					source_file: "knowledge/clients/acme.md",
+				},
+			],
+		]),
+		relations: [
+			{
+				from_entity: "acme-corp",
+				to_entity: "nextjs",
+				relation_type: "uses",
+				confidence: 0.9,
+				evidence: "Migration project",
+				source_file: "knowledge/clients/acme.md",
+			},
+		],
+		clusters: [],
+		built_at: "2026-04-04T12:00:00Z",
+		file_count: 1,
+	};
+}
+
+let testDir: string;
+
+describe("JsonFileStore", () => {
+	beforeEach(() => {
+		testDir = join(tmpdir(), `sourcepress-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+	});
+
+	afterEach(() => {
+		if (existsSync(testDir)) {
+			rmSync(testDir, { recursive: true });
+		}
+	});
+
+	it("creates the directory if it does not exist", () => {
+		new JsonFileStore(testDir);
+		expect(existsSync(testDir)).toBe(true);
+	});
+
+	describe("store and retrieve", () => {
+		it("stores and retrieves a knowledge file", async () => {
+			const store = new JsonFileStore(testDir);
+			const file = makeKnowledgeFile();
+			await store.store(file);
+			const retrieved = await store.retrieve(file.path);
+			expect(retrieved).toEqual(file);
+		});
+
+		it("returns null for non-existent file", async () => {
+			const store = new JsonFileStore(testDir);
+			const retrieved = await store.retrieve("nonexistent.md");
+			expect(retrieved).toBeNull();
+		});
+
+		it("overwrites existing file at same path", async () => {
+			const store = new JsonFileStore(testDir);
+			await store.store(makeKnowledgeFile());
+			await store.store(makeKnowledgeFile({ quality_score: 9 }));
+			const retrieved = await store.retrieve("knowledge/clients/acme.md");
+			expect(retrieved?.quality_score).toBe(9);
+		});
+	});
+
+	describe("persistence across instances", () => {
+		it("persists knowledge files to disk", async () => {
+			const store1 = new JsonFileStore(testDir);
+			await store1.store(makeKnowledgeFile({ path: "a.md" }));
+			await store1.store(makeKnowledgeFile({ path: "b.md" }));
+
+			const store2 = new JsonFileStore(testDir);
+			expect(await store2.count()).toBe(2);
+			expect(await store2.retrieve("a.md")).not.toBeNull();
+		});
+
+		it("persists graph to disk", async () => {
+			const store1 = new JsonFileStore(testDir);
+			const graph = makeGraph();
+			await store1.saveGraph(graph);
+
+			const store2 = new JsonFileStore(testDir);
+			const loaded = await store2.loadGraph();
+			expect(loaded).not.toBeNull();
+			expect(loaded?.entities.get("acme-corp")?.name).toBe("Acme Corp");
+			expect(loaded?.relations).toHaveLength(1);
+			expect(loaded?.built_at).toBe("2026-04-04T12:00:00Z");
+		});
+	});
+
+	describe("list", () => {
+		it("lists all stored files", async () => {
+			const store = new JsonFileStore(testDir);
+			await store.store(makeKnowledgeFile({ path: "a.md" }));
+			await store.store(makeKnowledgeFile({ path: "b.md" }));
+			const files = await store.list();
+			expect(files).toHaveLength(2);
+		});
+
+		it("filters by type", async () => {
+			const store = new JsonFileStore(testDir);
+			await store.store(makeKnowledgeFile({ path: "a.md", type: "project-notes" }));
+			await store.store(makeKnowledgeFile({ path: "b.md", type: "transcript" }));
+			const files = await store.list({ type: "transcript" });
+			expect(files).toHaveLength(1);
+			expect(files[0].type).toBe("transcript");
+		});
+
+		it("filters by quality", async () => {
+			const store = new JsonFileStore(testDir);
+			await store.store(makeKnowledgeFile({ path: "a.md", quality: "structured" }));
+			await store.store(makeKnowledgeFile({ path: "b.md", quality: "draft" }));
+			const files = await store.list({ quality: "draft" });
+			expect(files).toHaveLength(1);
+		});
+
+		it("filters by source", async () => {
+			const store = new JsonFileStore(testDir);
+			await store.store(makeKnowledgeFile({ path: "a.md", source: "manual" }));
+			await store.store(makeKnowledgeFile({ path: "b.md", source: "url" }));
+			const files = await store.list({ source: "url" });
+			expect(files).toHaveLength(1);
+		});
+	});
+
+	describe("delete", () => {
+		it("deletes an existing file and persists", async () => {
+			const store = new JsonFileStore(testDir);
+			await store.store(makeKnowledgeFile());
+			const deleted = await store.delete("knowledge/clients/acme.md");
+			expect(deleted).toBe(true);
+
+			const store2 = new JsonFileStore(testDir);
+			expect(await store2.retrieve("knowledge/clients/acme.md")).toBeNull();
+		});
+
+		it("returns false for non-existent file", async () => {
+			const store = new JsonFileStore(testDir);
+			expect(await store.delete("nonexistent.md")).toBe(false);
+		});
+	});
+
+	describe("count", () => {
+		it("returns 0 for empty store", async () => {
+			const store = new JsonFileStore(testDir);
+			expect(await store.count()).toBe(0);
+		});
+
+		it("returns correct count", async () => {
+			const store = new JsonFileStore(testDir);
+			await store.store(makeKnowledgeFile({ path: "a.md" }));
+			await store.store(makeKnowledgeFile({ path: "b.md" }));
+			expect(await store.count()).toBe(2);
+		});
+	});
+
+	describe("graph", () => {
+		it("returns null when no graph saved", async () => {
+			const store = new JsonFileStore(testDir);
+			expect(await store.loadGraph()).toBeNull();
+		});
+
+		it("preserves Map structure through serialization", async () => {
+			const store = new JsonFileStore(testDir);
+			const graph = makeGraph();
+			await store.saveGraph(graph);
+
+			const store2 = new JsonFileStore(testDir);
+			const loaded = await store2.loadGraph();
+			expect(loaded?.entities).toBeInstanceOf(Map);
+			expect(loaded?.entities.size).toBe(1);
+		});
+	});
+});