@kibhq/core 0.1.0

package/src/compile/compiler.test.ts

@@ -0,0 +1,278 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { existsSync } from "node:fs";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { ingestSource } from "../ingest/ingest.js";
+import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
+import { initVault, listWiki, loadManifest, readWiki } from "../vault.js";
+import { compileVault } from "./compiler.js";
+
+let tempDir: string;
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempVault() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-compile-test-"));
+	await initVault(tempDir, { name: "test" });
+	return tempDir;
+}
+
+/**
+ * Create a mock LLM provider that returns canned responses.
+ */
+function createMockProvider(responses: string[]): LLMProvider {
+	let callIndex = 0;
+	return {
+		name: "mock",
+		async complete(params: CompletionParams): Promise<CompletionResult> {
+			const content = responses[callIndex] ?? "[]";
+			callIndex++;
+			return {
+				content,
+				usage: { inputTokens: 100, outputTokens: 200 },
+				stopReason: "end_turn",
+			};
+		},
+		async *stream(): AsyncIterable<StreamChunk> {
+			yield { type: "text", text: "stream not used in tests" };
+		},
+	};
+}
+
+describe("compileVault", () => {
+	test("compiles a single source into articles", async () => {
+		const root = await makeTempVault();
+
+		// Ingest a test file
+		const testFile = join(root, "source.md");
+		await writeFile(
+			testFile,
+			"# Transformer Architecture\n\nThe transformer is a neural network architecture.\n\nIt uses self-attention mechanisms.",
+		);
+		await ingestSource(root, testFile);
+
+		// Mock provider returns a compile result
+		const mockResponse = JSON.stringify([
+			{
+				op: "create",
+				path: "wiki/concepts/transformer-architecture.md",
+				content: `---
+title: Transformer Architecture
+slug: transformer-architecture
+category: concept
+tags: [deep-learning, nlp]
+sources:
+  - raw/articles/transformer-architecture.md
+created: 2026-04-05
+updated: 2026-04-05
+summary: The transformer replaces recurrence with self-attention.
+---
+
+# Transformer Architecture
+
+The transformer is a neural network architecture that replaces recurrence with self-attention mechanisms, enabling parallel training and superior sequence modeling.
+
+## Key Features
+
+- **Self-Attention**: Allows the model to weigh the importance of different positions
+- **Parallelizable**: Unlike RNNs, transformers can process all positions simultaneously
+- **Scalable**: Performance improves predictably with more compute
+
+See also: [[attention-mechanisms]], [[positional-encoding]]`,
+			},
+		]);
+
+		const provider = createMockProvider([mockResponse]);
+		const config = (await import("../vault.js")).loadConfig;
+		const vaultConfig = await config(root);
+
+		const result = await compileVault(root, provider, vaultConfig);
+
+		expect(result.sourcesCompiled).toBe(1);
+		expect(result.articlesCreated).toBe(1);
+		expect(result.articlesUpdated).toBe(0);
+
+		// Verify the article was written
+		const article = await readWiki(root, "concepts/transformer-architecture.md");
+		expect(article).toContain("Transformer Architecture");
+		expect(article).toContain("self-attention");
+
+		// Verify manifest was updated
+		const manifest = await loadManifest(root);
+		expect(manifest.vault.lastCompiled).not.toBeNull();
+		expect(manifest.stats.totalArticles).toBeGreaterThan(0);
+		expect(manifest.articles["transformer-architecture"]).toBeDefined();
+		expect(manifest.articles["transformer-architecture"]!.category).toBe("concept");
+		expect(manifest.articles["transformer-architecture"]!.forwardLinks).toContain(
+			"attention-mechanisms",
+		);
+
+		// Verify INDEX.md was generated
+		expect(existsSync(join(root, "wiki", "INDEX.md"))).toBe(true);
+		const index = await readWiki(root, "INDEX.md");
+		expect(index).toContain("Transformer Architecture");
+		expect(index).toContain("## Concepts");
+
+		// Verify GRAPH.md was generated
+		expect(existsSync(join(root, "wiki", "GRAPH.md"))).toBe(true);
+		const graph = await readWiki(root, "GRAPH.md");
+		expect(graph).toContain("transformer-architecture");
+	});
+
+	test("skips already-compiled sources", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "source.md");
+		await writeFile(testFile, "# Test\n\nContent.");
+		await ingestSource(root, testFile);
+
+		// First compile
+		const mockResponse = JSON.stringify([
+			{
+				op: "create",
+				path: "wiki/concepts/test.md",
+				content:
+					"---\ntitle: Test\nslug: test\ncategory: concept\ntags: []\nsummary: A test.\n---\n\n# Test\n\nContent.",
+			},
+		]);
+		const provider1 = createMockProvider([mockResponse]);
+		const vaultConfig = await (await import("../vault.js")).loadConfig(root);
+		await compileVault(root, provider1, vaultConfig);
+
+		// Second compile — should skip (no new sources)
+		const provider2 = createMockProvider([]);
+		const result = await compileVault(root, provider2, vaultConfig);
+		expect(result.sourcesCompiled).toBe(0);
+	});
+
+	test("recompiles when force is true", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "source.md");
+		await writeFile(testFile, "# Test\n\nContent.");
+		await ingestSource(root, testFile);
+
+		// First compile
+		const mockResponse = JSON.stringify([
+			{
+				op: "create",
+				path: "wiki/concepts/test.md",
+				content:
+					"---\ntitle: Test\nslug: test\ncategory: concept\ntags: []\nsummary: A test.\n---\n\n# Test\n\nContent.",
+			},
+		]);
+		const provider1 = createMockProvider([mockResponse]);
+		const vaultConfig = await (await import("../vault.js")).loadConfig(root);
+		await compileVault(root, provider1, vaultConfig);
+
+		// Force recompile
+		const provider2 = createMockProvider([
+			JSON.stringify([
+				{
+					op: "update",
+					path: "wiki/concepts/test.md",
+					content:
+						"---\ntitle: Test\nslug: test\ncategory: concept\ntags: []\nsummary: Updated.\n---\n\n# Test\n\nUpdated content.",
+				},
+			]),
+		]);
+		const result = await compileVault(root, provider2, vaultConfig, {
+			force: true,
+		});
+		expect(result.sourcesCompiled).toBe(1);
+		expect(result.articlesUpdated).toBe(1);
+	});
+
+	test("dry run does not write files", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "source.md");
+		await writeFile(testFile, "# Test\n\nContent.");
+		await ingestSource(root, testFile);
+
+		const mockResponse = JSON.stringify([
+			{
+				op: "create",
+				path: "wiki/concepts/test.md",
+				content: "---\ntitle: Test\nslug: test\ncategory: concept\n---\n\n# Test",
+			},
+		]);
+		const provider = createMockProvider([mockResponse]);
+		const vaultConfig = await (await import("../vault.js")).loadConfig(root);
+
+		const result = await compileVault(root, provider, vaultConfig, {
+			dryRun: true,
+		});
+
+		expect(result.sourcesCompiled).toBe(1);
+		expect(result.articlesCreated).toBe(1);
+
+		// File should NOT exist
+		expect(existsSync(join(root, "wiki", "concepts", "test.md"))).toBe(false);
+	});
+
+	test("handles empty response from LLM", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "source.md");
+		await writeFile(testFile, "# Test\n\nContent.");
+		await ingestSource(root, testFile);
+
+		const provider = createMockProvider(["[]"]);
+		const vaultConfig = await (await import("../vault.js")).loadConfig(root);
+
+		const result = await compileVault(root, provider, vaultConfig);
+		expect(result.sourcesCompiled).toBe(1);
+		expect(result.articlesCreated).toBe(0);
+	});
+
+	test("handles no pending sources", async () => {
+		const root = await makeTempVault();
+		const provider = createMockProvider([]);
+		const vaultConfig = await (await import("../vault.js")).loadConfig(root);
+
+		const result = await compileVault(root, provider, vaultConfig);
+		expect(result.sourcesCompiled).toBe(0);
+	});
+
+	test("compiles multiple sources", async () => {
+		const root = await makeTempVault();
+
+		const file1 = join(root, "article1.md");
+		const file2 = join(root, "article2.md");
+		await writeFile(file1, "# Article One\n\nFirst article content.");
+		await writeFile(file2, "# Article Two\n\nSecond article content.");
+		await ingestSource(root, file1);
+		await ingestSource(root, file2);
+
+		const provider = createMockProvider([
+			JSON.stringify([
+				{
+					op: "create",
+					path: "wiki/concepts/article-one.md",
+					content:
+						"---\ntitle: Article One\nslug: article-one\ncategory: concept\ntags: []\nsummary: First.\n---\n\n# Article One\n\nCompiled first.",
+				},
+			]),
+			JSON.stringify([
+				{
+					op: "create",
+					path: "wiki/concepts/article-two.md",
+					content:
+						"---\ntitle: Article Two\nslug: article-two\ncategory: concept\ntags: []\nsummary: Second.\n---\n\n# Article Two\n\nCompiled second.",
+				},
+			]),
+		]);
+		const vaultConfig = await (await import("../vault.js")).loadConfig(root);
+
+		const result = await compileVault(root, provider, vaultConfig);
+		expect(result.sourcesCompiled).toBe(2);
+		expect(result.articlesCreated).toBe(2);
+
+		const manifest = await loadManifest(root);
+		expect(manifest.stats.totalArticles).toBe(2);
+	});
+});

package/src/compile/compiler.ts

@@ -0,0 +1,305 @@
+import { readFile } from "node:fs/promises";
+import { join, relative } from "node:path";
+import { GRAPH_FILE, INDEX_FILE, WIKI_DIR } from "../constants.js";
+import { hash } from "../hash.js";
+import { countWords } from "../ingest/normalize.js";
+import type { CompileResult, LLMProvider, Manifest, VaultConfig } from "../types.js";
+import {
+	deleteFile,
+	listWiki,
+	loadManifest,
+	readIndex,
+	readRaw,
+	readWiki,
+	saveManifest,
+	writeWiki,
+} from "../vault.js";
+import { buildLinkGraph, generateGraphMd } from "./backlinks.js";
+import { extractWikilinks, parseCompileOutput, parseFrontmatter } from "./diff.js";
+import { computeStats, generateIndexMd } from "./index-manager.js";
+import { compileSystemPrompt, compileUserPrompt } from "./prompts.js";
+
+export interface CompileOptions {
+	/** Recompile all sources regardless of state */
+	force?: boolean;
+	/** Only compile this specific source path */
+	sourceFilter?: string;
+	/** Max sources to compile in this pass */
+	maxSources?: number;
+	/** Don't actually write files, just return what would happen */
+	dryRun?: boolean;
+	/** Callback for progress updates */
+	onProgress?: (msg: string) => void;
+}
+
+/**
+ * Compile pending raw sources into wiki articles.
+ */
+export async function compileVault(
+	root: string,
+	provider: LLMProvider,
+	config: VaultConfig,
+	options: CompileOptions = {},
+): Promise<CompileResult> {
+	const manifest = await loadManifest(root);
+	const categories = config.compile.categories;
+
+	// Find sources that need compilation
+	const pendingSources = findPendingSources(manifest, options);
+
+	if (pendingSources.length === 0) {
+		return {
+			sourcesCompiled: 0,
+			articlesCreated: 0,
+			articlesUpdated: 0,
+			articlesDeleted: 0,
+			operations: [],
+		};
+	}
+
+	// Limit sources per pass
+	const maxSources = options.maxSources ?? config.compile.max_sources_per_pass;
+	const sourcesToCompile = pendingSources.slice(0, maxSources);
+
+	// Read current INDEX.md for context
+	const indexContent = await readIndex(root);
+
+	let totalCreated = 0;
+	let totalUpdated = 0;
+	let totalDeleted = 0;
+	const allOperations: CompileResult["operations"] = [];
+
+	for (const [sourceId, sourcePath] of sourcesToCompile) {
+		options.onProgress?.(`Compiling ${sourcePath}...`);
+
+		try {
+			// Read the raw source content
+			const sourceContent = await readRaw(root, sourcePath);
+
+			// Read existing articles this source produced (for context)
+			const existingArticles = await loadExistingArticles(root, manifest, sourceId);
+
+			// Build the compile prompt
+			const today = new Date().toISOString().split("T")[0]!;
+			const userPrompt = compileUserPrompt({
+				indexContent,
+				sourceContent,
+				sourcePath: `raw/${sourcePath}`,
+				existingArticles,
+				today,
+			});
+
+			// Call the LLM
+			const result = await provider.complete({
+				system: compileSystemPrompt(categories),
+				messages: [{ role: "user", content: userPrompt }],
+				temperature: 0,
+				maxTokens: 8192,
+			});
+
+			// Parse the response into file operations
+			const operations = parseCompileOutput(result.content);
+
+			if (options.dryRun) {
+				allOperations.push(...operations);
+				for (const op of operations) {
+					if (op.op === "create") totalCreated++;
+					else if (op.op === "update") totalUpdated++;
+					else if (op.op === "delete") totalDeleted++;
+				}
+				continue;
+			}
+
+			// Execute file operations
+			const producedArticles: string[] = [];
+
+			for (const op of operations) {
+				const wikiRelPath = op.path.replace(/^wiki\//, "");
+
+				if (op.op === "create" || op.op === "update") {
+					if (!op.content) continue;
+					await writeWiki(root, wikiRelPath, op.content);
+					producedArticles.push(op.path);
+
+					// Update article entry in manifest
+					const { frontmatter, body } = parseFrontmatter(op.content);
+					const articleSlug = (frontmatter.slug as string) ?? wikiRelPath.replace(/\.md$/, "");
+					const contentHash = await hash(op.content);
+					const wikilinks = extractWikilinks(op.content);
+					const now = new Date().toISOString();
+
+					manifest.articles[articleSlug] = {
+						hash: contentHash,
+						createdAt:
+							op.op === "create" ? now : (manifest.articles[articleSlug]?.createdAt ?? now),
+						lastUpdated: now,
+						derivedFrom: [`raw/${sourcePath}`],
+						backlinks: [], // will be computed after all sources are compiled
+						forwardLinks: wikilinks,
+						tags: Array.isArray(frontmatter.tags) ? (frontmatter.tags as string[]) : [],
+						summary: (frontmatter.summary as string) ?? "",
+						wordCount: countWords(body),
+						category: (frontmatter.category as string) ?? "topic",
+					};
+
+					if (op.op === "create") totalCreated++;
+					else totalUpdated++;
+				} else if (op.op === "delete") {
+					const fullPath = join(root, op.path);
+					await deleteFile(fullPath);
+					totalDeleted++;
+
+					// Remove from manifest
+					const slug = wikiRelPath.replace(/\.md$/, "");
+					delete manifest.articles[slug];
+				}
+
+				allOperations.push(op);
+			}
+
+			// Update source entry
+			manifest.sources[sourceId]!.lastCompiled = new Date().toISOString();
+			manifest.sources[sourceId]!.producedArticles = producedArticles;
+		} catch (err) {
+			options.onProgress?.(`Failed to compile ${sourcePath}: ${(err as Error).message}`);
+			// Continue with other sources
+		}
+	}
+
+	if (!options.dryRun) {
+		// Rebuild link graph (backlinks)
+		options.onProgress?.("Updating backlinks...");
+		const graph = await buildLinkGraph(root);
+
+		// Update backlinks in manifest
+		for (const [slug, backlinksSet] of graph.backlinks) {
+			if (manifest.articles[slug]) {
+				manifest.articles[slug]!.backlinks = [...backlinksSet];
+			}
+		}
+		for (const [slug, forwardSet] of graph.forwardLinks) {
+			if (manifest.articles[slug]) {
+				manifest.articles[slug]!.forwardLinks = [...forwardSet];
+			}
+		}
+
+		// Regenerate INDEX.md
+		if (config.compile.auto_index) {
+			options.onProgress?.("Updating INDEX.md...");
+			const indexMd = await generateIndexMd(root);
+			await writeWiki(root, INDEX_FILE, indexMd);
+		}
+
+		// Regenerate GRAPH.md
+		if (config.compile.auto_graph) {
+			options.onProgress?.("Updating GRAPH.md...");
+			const graphMd = generateGraphMd(graph);
+			await writeWiki(root, GRAPH_FILE, graphMd);
+		}
+
+		// Update manifest stats
+		const stats = await computeStats(root);
+		manifest.stats.totalArticles = stats.totalArticles;
+		manifest.stats.totalWords = stats.totalWords;
+		manifest.stats.totalSources = Object.keys(manifest.sources).length;
+		manifest.vault.lastCompiled = new Date().toISOString();
+
+		await saveManifest(root, manifest);
+	}
+
+	return {
+		sourcesCompiled: sourcesToCompile.length,
+		articlesCreated: totalCreated,
+		articlesUpdated: totalUpdated,
+		articlesDeleted: totalDeleted,
+		operations: allOperations,
+	};
+}
+
+/**
+ * Find sources that need compilation.
+ */
+function findPendingSources(manifest: Manifest, options: CompileOptions): [string, string][] {
+	const pending: [string, string][] = [];
+
+	for (const [sourceId, source] of Object.entries(manifest.sources)) {
+		// If filtering to a specific source
+		if (options.sourceFilter) {
+			const matchesId = sourceId === options.sourceFilter;
+			const matchesPath = source.producedArticles.some((p) => p.includes(options.sourceFilter!));
+			if (!matchesId && !matchesPath) continue;
+		}
+
+		// Determine the raw file path from sourceId
+		// Source was written to raw/{category}/{slug}.md
+		const rawPath = findRawPath(manifest, sourceId);
+		if (!rawPath) continue;
+
+		if (options.force || !source.lastCompiled || source.lastCompiled < source.ingestedAt) {
+			pending.push([sourceId, rawPath]);
+		}
+	}
+
+	return pending;
+}
+
+/**
+ * Find the raw file path for a source.
+ * Sources are tracked by ID; we need to find which raw/ file they correspond to.
+ */
+function findRawPath(manifest: Manifest, sourceId: string): string | null {
+	const source = manifest.sources[sourceId];
+	if (!source) return null;
+
+	// The raw path is derived from the source metadata
+	const title = source.metadata.title ?? sourceId;
+	const slug = title
+		.toLowerCase()
+		.replace(/[^a-z0-9\s-]/g, "")
+		.replace(/\s+/g, "-")
+		.replace(/-+/g, "-")
+		.replace(/^-|-$/g, "")
+		.slice(0, 80);
+
+	const category = categoryForSourceType(source.sourceType);
+	return `${category}/${slug}.md`;
+}
+
+function categoryForSourceType(sourceType: string): string {
+	switch (sourceType) {
+		case "pdf":
+			return "papers";
+		case "youtube":
+			return "transcripts";
+		case "github":
+			return "repos";
+		case "image":
+			return "images";
+		default:
+			return "articles";
+	}
+}
+
+/**
+ * Load existing wiki articles that a source previously produced.
+ */
+async function loadExistingArticles(
+	root: string,
+	manifest: Manifest,
+	sourceId: string,
+): Promise<{ path: string; content: string }[]> {
+	const source = manifest.sources[sourceId];
+	if (!source?.producedArticles.length) return [];
+
+	const articles: { path: string; content: string }[] = [];
+	for (const articlePath of source.producedArticles) {
+		try {
+			const relPath = articlePath.replace(/^wiki\//, "");
+			const content = await readWiki(root, relPath);
+			articles.push({ path: articlePath, content });
+		} catch {
+			// Article might have been deleted
+		}
+	}
+	return articles;
+}