@kibhq/core 0.1.0

package/src/compile/diff.test.ts

@@ -0,0 +1,164 @@
+import { describe, expect, test } from "bun:test";
+import { extractWikilinks, parseCompileOutput, parseFrontmatter } from "./diff.js";
+
+describe("parseCompileOutput", () => {
+	test("parses clean JSON array", () => {
+		const input = JSON.stringify([
+			{
+				op: "create",
+				path: "wiki/concepts/test.md",
+				content: "# Test\n\nContent.",
+			},
+		]);
+		const result = parseCompileOutput(input);
+		expect(result).toHaveLength(1);
+		expect(result[0]!.op).toBe("create");
+		expect(result[0]!.path).toBe("wiki/concepts/test.md");
+	});
+
+	test("strips markdown code fences", () => {
+		const input = '```json\n[{"op":"create","path":"wiki/test.md","content":"# Test"}]\n```';
+		const result = parseCompileOutput(input);
+		expect(result).toHaveLength(1);
+		expect(result[0]!.op).toBe("create");
+	});
+
+	test("strips plain code fences", () => {
+		const input = '```\n[{"op":"create","path":"wiki/test.md","content":"x"}]\n```';
+		const result = parseCompileOutput(input);
+		expect(result).toHaveLength(1);
+	});
+
+	test("extracts JSON from surrounding text", () => {
+		const input =
+			'Here are the file operations:\n\n[{"op":"create","path":"wiki/test.md","content":"x"}]\n\nHope that helps!';
+		const result = parseCompileOutput(input);
+		expect(result).toHaveLength(1);
+	});
+
+	test("parses multiple operations", () => {
+		const input = JSON.stringify([
+			{ op: "create", path: "wiki/concepts/a.md", content: "# A" },
+			{ op: "update", path: "wiki/topics/b.md", content: "# B updated" },
+			{ op: "delete", path: "wiki/references/c.md" },
+		]);
+		const result = parseCompileOutput(input);
+		expect(result).toHaveLength(3);
+		expect(result[0]!.op).toBe("create");
+		expect(result[1]!.op).toBe("update");
+		expect(result[2]!.op).toBe("delete");
+	});
+
+	test("parses empty array", () => {
+		const result = parseCompileOutput("[]");
+		expect(result).toHaveLength(0);
+	});
+
+	test("throws on invalid JSON", () => {
+		expect(() => parseCompileOutput("not json at all")).toThrow("Failed to parse");
+	});
+
+	test("throws on wrong structure", () => {
+		expect(() => parseCompileOutput('{"not": "an array"}')).toThrow();
+	});
+
+	test("throws on invalid operation type", () => {
+		expect(() => parseCompileOutput('[{"op":"rename","path":"x","content":"y"}]')).toThrow();
+	});
+
+	test("handles whitespace around JSON", () => {
+		const input = '  \n\n  [{"op":"create","path":"wiki/test.md","content":"x"}]  \n\n  ';
+		const result = parseCompileOutput(input);
+		expect(result).toHaveLength(1);
+	});
+
+	test("delete operation doesn't require content", () => {
+		const input = '[{"op":"delete","path":"wiki/old.md"}]';
+		const result = parseCompileOutput(input);
+		expect(result).toHaveLength(1);
+		expect(result[0]!.content).toBeUndefined();
+	});
+});
+
+describe("parseFrontmatter", () => {
+	test("parses standard frontmatter", () => {
+		const md = `---
+title: Test Article
+slug: test-article
+category: concept
+tags: [deep-learning, nlp]
+created: 2026-04-05
+updated: 2026-04-05
+summary: A test article about testing.
+---
+
+# Test Article
+
+Content here.`;
+
+		const { frontmatter, body } = parseFrontmatter(md);
+		expect(frontmatter.title).toBe("Test Article");
+		expect(frontmatter.slug).toBe("test-article");
+		expect(frontmatter.category).toBe("concept");
+		expect(frontmatter.tags).toEqual(["deep-learning", "nlp"]);
+		expect(frontmatter.created).toBe("2026-04-05");
+		expect(body).toContain("# Test Article");
+		expect(body).toContain("Content here.");
+	});
+
+	test("returns empty frontmatter when none exists", () => {
+		const { frontmatter, body } = parseFrontmatter("# Just Content\n\nNo frontmatter.");
+		expect(frontmatter).toEqual({});
+		expect(body).toBe("# Just Content\n\nNo frontmatter.");
+	});
+
+	test("handles quoted values", () => {
+		const md = '---\ntitle: "Quoted Title"\n---\n\nBody.';
+		const { frontmatter } = parseFrontmatter(md);
+		expect(frontmatter.title).toBe("Quoted Title");
+	});
+
+	test("handles boolean values", () => {
+		const md = "---\ndraft: true\npublished: false\n---\n\nBody.";
+		const { frontmatter } = parseFrontmatter(md);
+		expect(frontmatter.draft).toBe(true);
+		expect(frontmatter.published).toBe(false);
+	});
+
+	test("handles empty tags array", () => {
+		const md = "---\ntags: []\n---\n\nBody.";
+		const { frontmatter } = parseFrontmatter(md);
+		expect(frontmatter.tags).toEqual([]);
+	});
+});
+
+describe("extractWikilinks", () => {
+	test("extracts single wikilink", () => {
+		const content = "This relates to [[transformer-architecture]].";
+		expect(extractWikilinks(content)).toEqual(["transformer-architecture"]);
+	});
+
+	test("extracts multiple wikilinks", () => {
+		const content = "See [[attention-mechanisms]] and [[positional-encoding]] for details.";
+		expect(extractWikilinks(content)).toEqual(["attention-mechanisms", "positional-encoding"]);
+	});
+
+	test("deduplicates wikilinks", () => {
+		const content = "The [[transformer]] is great. More about [[transformer]] here.";
+		expect(extractWikilinks(content)).toEqual(["transformer"]);
+	});
+
+	test("normalizes to kebab-case", () => {
+		const content = "See [[Transformer Architecture]] for details.";
+		expect(extractWikilinks(content)).toEqual(["transformer-architecture"]);
+	});
+
+	test("returns empty array when no links", () => {
+		expect(extractWikilinks("No links here.")).toEqual([]);
+	});
+
+	test("handles links with spaces around them", () => {
+		const content = "See [[ attention mechanisms ]] here.";
+		expect(extractWikilinks(content)).toEqual(["attention-mechanisms"]);
+	});
+});

package/src/compile/diff.ts

@@ -0,0 +1,121 @@
+import { z } from "zod";
+import { FileOperationSchema } from "../schemas.js";
+import type { FileOperation } from "../types.js";
+
+const FileOperationsArraySchema = z.array(FileOperationSchema);
+
+/**
+ * Parse LLM compile output into file operations.
+ *
+ * The LLM should return a JSON array of {op, path, content} objects.
+ * This parser handles various edge cases:
+ * - JSON wrapped in markdown code fences
+ * - Extra text before/after the JSON
+ * - Minor formatting issues
+ */
+export function parseCompileOutput(raw: string): FileOperation[] {
+	const cleaned = extractJson(raw);
+
+	try {
+		const parsed = JSON.parse(cleaned);
+		return FileOperationsArraySchema.parse(parsed);
+	} catch (err) {
+		throw new Error(
+			`Failed to parse LLM compile output: ${err instanceof Error ? err.message : err}\n\nRaw output (first 500 chars):\n${raw.slice(0, 500)}`,
+		);
+	}
+}
+
+/**
+ * Extract JSON array from LLM output that may contain surrounding text.
+ */
+function extractJson(raw: string): string {
+	let text = raw.trim();
+
+	// Strip markdown code fences
+	text = text.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "");
+	text = text.trim();
+
+	// If it already starts with [, try it directly
+	if (text.startsWith("[")) {
+		return text;
+	}
+
+	// Try to find a JSON array in the text
+	const arrayStart = text.indexOf("[");
+	const arrayEnd = text.lastIndexOf("]");
+
+	if (arrayStart !== -1 && arrayEnd !== -1 && arrayEnd > arrayStart) {
+		return text.slice(arrayStart, arrayEnd + 1);
+	}
+
+	// Nothing worked, return as-is and let JSON.parse fail with a clear error
+	return text;
+}
+
+/**
+ * Extract YAML frontmatter from a markdown article string.
+ * Returns the frontmatter fields as a Record and the body content.
+ */
+export function parseFrontmatter(markdown: string): {
+	frontmatter: Record<string, unknown>;
+	body: string;
+} {
+	const match = markdown.match(/^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/);
+	if (!match) {
+		return { frontmatter: {}, body: markdown };
+	}
+
+	const rawFrontmatter = match[1]!;
+	const body = match[2]!;
+
+	// Simple YAML-like parser for frontmatter (handles common cases)
+	const frontmatter: Record<string, unknown> = {};
+
+	for (const line of rawFrontmatter.split("\n")) {
+		const trimmed = line.trim();
+		if (!trimmed || trimmed.startsWith("#")) continue;
+
+		// Handle continuation lines (for summary: >)
+		const colonIdx = trimmed.indexOf(":");
+		if (colonIdx === -1) continue;
+
+		const key = trimmed.slice(0, colonIdx).trim();
+		let value: unknown = trimmed.slice(colonIdx + 1).trim();
+
+		// Parse arrays: [tag1, tag2]
+		if (typeof value === "string" && value.startsWith("[") && value.endsWith("]")) {
+			value = value
+				.slice(1, -1)
+				.split(",")
+				.map((s) => s.trim())
+				.filter(Boolean);
+		}
+		// Parse booleans
+		else if (value === "true") value = true;
+		else if (value === "false") value = false;
+		// Remove quotes
+		else if (typeof value === "string" && value.startsWith('"') && value.endsWith('"')) {
+			value = value.slice(1, -1);
+		}
+
+		if (key) {
+			frontmatter[key] = value;
+		}
+	}
+
+	return { frontmatter, body: body.trim() };
+}
+
+/**
+ * Extract [[wikilinks]] from markdown content.
+ * Returns an array of slug strings.
+ */
+export function extractWikilinks(content: string): string[] {
+	const matches = content.matchAll(/\[\[([^\]]+)\]\]/g);
+	const links: string[] = [];
+	for (const match of matches) {
+		links.push(match[1]!.trim().toLowerCase().replace(/\s+/g, "-"));
+	}
+	return [...new Set(links)]; // deduplicate
+}

package/src/compile/index-manager.test.ts

@@ -0,0 +1,227 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { initVault, writeWiki } from "../vault.js";
+import { computeStats, generateIndexMd } from "./index-manager.js";
+
+let tempDir: string;
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempVault() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-index-test-"));
+	await initVault(tempDir, { name: "test" });
+	return tempDir;
+}
+
+function articleMd(opts: {
+	title: string;
+	slug: string;
+	category: string;
+	tags?: string[];
+	summary?: string;
+	body?: string;
+}) {
+	const tags = opts.tags ? `[${opts.tags.join(", ")}]` : "[]";
+	return `---
+title: ${opts.title}
+slug: ${opts.slug}
+category: ${opts.category}
+tags: ${tags}
+summary: ${opts.summary ?? ""}
+---
+
+# ${opts.title}
+
+${opts.body ?? "Some content here for the article."}`;
+}
+
+describe("generateIndexMd", () => {
+	test("generates index with articles grouped by category", async () => {
+		const root = await makeTempVault();
+
+		await writeWiki(
+			root,
+			"concepts/transformers.md",
+			articleMd({
+				title: "Transformer Architecture",
+				slug: "transformer-architecture",
+				category: "concept",
+				tags: ["deep-learning", "nlp"],
+				summary: "The transformer replaces recurrence with self-attention.",
+			}),
+		);
+
+		await writeWiki(
+			root,
+			"topics/scaling-laws.md",
+			articleMd({
+				title: "Scaling Laws",
+				slug: "scaling-laws",
+				category: "topic",
+				tags: ["training"],
+				summary: "Power-law relationships between compute and loss.",
+			}),
+		);
+
+		await writeWiki(
+			root,
+			"references/vaswani.md",
+			articleMd({
+				title: "Vaswani et al.",
+				slug: "vaswani-et-al",
+				category: "reference",
+				summary: "Authors of the original transformer paper.",
+			}),
+		);
+
+		const index = await generateIndexMd(root);
+
+		expect(index).toContain("# Knowledge Base Index");
+		expect(index).toContain("3 articles");
+
+		// Categories
+		expect(index).toContain("## Concepts");
+		expect(index).toContain("## Topics");
+		expect(index).toContain("## References");
+
+		// Articles
+		expect(index).toContain("[Transformer Architecture]");
+		expect(index).toContain("[Scaling Laws]");
+		expect(index).toContain("[Vaswani et al.]");
+
+		// Tags
+		expect(index).toContain("`#deep-learning`");
+		expect(index).toContain("`#nlp`");
+
+		// Summaries
+		expect(index).toContain("replaces recurrence with self-attention");
+	});
+
+	test("handles empty wiki", async () => {
+		const root = await makeTempVault();
+		const index = await generateIndexMd(root);
+		expect(index).toContain("0 articles");
+	});
+
+	test("sorts articles alphabetically within categories", async () => {
+		const root = await makeTempVault();
+
+		await writeWiki(
+			root,
+			"concepts/zebra.md",
+			articleMd({
+				title: "Zebra Concept",
+				slug: "zebra",
+				category: "concept",
+			}),
+		);
+		await writeWiki(
+			root,
+			"concepts/alpha.md",
+			articleMd({
+				title: "Alpha Concept",
+				slug: "alpha",
+				category: "concept",
+			}),
+		);
+		await writeWiki(
+			root,
+			"concepts/mid.md",
+			articleMd({
+				title: "Mid Concept",
+				slug: "mid",
+				category: "concept",
+			}),
+		);
+
+		const index = await generateIndexMd(root);
+		const conceptsIdx = index.indexOf("## Concepts");
+		const alphaIdx = index.indexOf("Alpha Concept");
+		const midIdx = index.indexOf("Mid Concept");
+		const zebraIdx = index.indexOf("Zebra Concept");
+
+		expect(alphaIdx).toBeLessThan(midIdx);
+		expect(midIdx).toBeLessThan(zebraIdx);
+	});
+
+	test("skips INDEX.md and GRAPH.md from listing", async () => {
+		const root = await makeTempVault();
+
+		await writeWiki(root, "INDEX.md", "# Index\nOld index.");
+		await writeWiki(root, "GRAPH.md", "# Graph\nOld graph.");
+		await writeWiki(
+			root,
+			"concepts/test.md",
+			articleMd({
+				title: "Test",
+				slug: "test",
+				category: "concept",
+			}),
+		);
+
+		const index = await generateIndexMd(root);
+		expect(index).toContain("1 articles"); // Only the actual article, not INDEX/GRAPH
+	});
+
+	test("only includes sections that have articles", async () => {
+		const root = await makeTempVault();
+
+		await writeWiki(
+			root,
+			"concepts/test.md",
+			articleMd({
+				title: "Test",
+				slug: "test",
+				category: "concept",
+			}),
+		);
+
+		const index = await generateIndexMd(root);
+		expect(index).toContain("## Concepts");
+		expect(index).not.toContain("## Topics");
+		expect(index).not.toContain("## References");
+		expect(index).not.toContain("## Outputs");
+	});
+});
+
+describe("computeStats", () => {
+	test("computes article count and word count", async () => {
+		const root = await makeTempVault();
+
+		await writeWiki(
+			root,
+			"concepts/a.md",
+			articleMd({
+				title: "A",
+				slug: "a",
+				category: "concept",
+				body: "one two three four five",
+			}),
+		);
+		await writeWiki(
+			root,
+			"concepts/b.md",
+			articleMd({
+				title: "B",
+				slug: "b",
+				category: "concept",
+				body: "six seven eight nine ten",
+			}),
+		);
+
+		const stats = await computeStats(root);
+		expect(stats.totalArticles).toBe(2);
+		expect(stats.totalWords).toBeGreaterThan(0);
+	});
+
+	test("returns zero for empty wiki", async () => {
+		const root = await makeTempVault();
+		const stats = await computeStats(root);
+		expect(stats.totalArticles).toBe(0);
+		expect(stats.totalWords).toBe(0);
+	});
+});

package/src/compile/index-manager.ts

@@ -0,0 +1,148 @@
+import { readFile } from "node:fs/promises";
+import { relative } from "node:path";
+import { WIKI_DIR } from "../constants.js";
+import { listWiki } from "../vault.js";
+import { parseFrontmatter } from "./diff.js";
+
+interface ArticleMeta {
+	title: string;
+	slug: string;
+	category: string;
+	tags: string[];
+	summary: string;
+	relativePath: string;
+	wordCount: number;
+}
+
+/**
+ * Generate INDEX.md content by reading all wiki articles' frontmatter.
+ * This is deterministic — no LLM needed.
+ */
+export async function generateIndexMd(root: string): Promise<string> {
+	const wikiDir = `${root}/${WIKI_DIR}`;
+	const files = await listWiki(root);
+
+	// Skip INDEX.md and GRAPH.md themselves
+	const articleFiles = files.filter((f) => !f.endsWith("INDEX.md") && !f.endsWith("GRAPH.md"));
+
+	const articles: ArticleMeta[] = [];
+
+	for (const filePath of articleFiles) {
+		const content = await readFile(filePath, "utf-8");
+		const { frontmatter, body } = parseFrontmatter(content);
+
+		const relPath = relative(wikiDir, filePath);
+		const wordCount = body.split(/\s+/).filter(Boolean).length;
+
+		articles.push({
+			title: (frontmatter.title as string) ?? relPath.replace(/\.md$/, ""),
+			slug: (frontmatter.slug as string) ?? "",
+			category: (frontmatter.category as string) ?? categorize(relPath),
+			tags: Array.isArray(frontmatter.tags) ? (frontmatter.tags as string[]) : [],
+			summary: (frontmatter.summary as string) ?? "",
+			relativePath: relPath,
+			wordCount,
+		});
+	}
+
+	// Group by category
+	const grouped = new Map<string, ArticleMeta[]>();
+	for (const article of articles) {
+		const cat = article.category;
+		if (!grouped.has(cat)) {
+			grouped.set(cat, []);
+		}
+		grouped.get(cat)!.push(article);
+	}
+
+	// Sort articles within each category alphabetically
+	for (const arts of grouped.values()) {
+		arts.sort((a, b) => a.title.localeCompare(b.title));
+	}
+
+	// Compute stats
+	const totalArticles = articles.length;
+	const totalWords = articles.reduce((sum, a) => sum + a.wordCount, 0);
+	const now = new Date().toISOString();
+
+	// Build the index
+	const lines: string[] = [
+		"# Knowledge Base Index",
+		"",
+		`> ${totalArticles} articles | ${totalWords.toLocaleString()} words | Last compiled: ${now}`,
+	];
+
+	// Ordered category display
+	const categoryOrder = ["concept", "topic", "reference", "output"];
+	const categoryLabels: Record<string, string> = {
+		concept: "Concepts",
+		topic: "Topics",
+		reference: "References",
+		output: "Outputs",
+	};
+
+	for (const cat of categoryOrder) {
+		const arts = grouped.get(cat);
+		if (!arts || arts.length === 0) continue;
+
+		lines.push("", `## ${categoryLabels[cat] ?? cat}`, "");
+
+		for (const article of arts) {
+			const tags = article.tags.map((t) => `\`#${t}\``).join(" ");
+			const summary = article.summary ? ` -- ${article.summary}` : "";
+			lines.push(
+				`- **[${article.title}](${article.relativePath})**${summary}${tags ? ` ${tags}` : ""}`,
+			);
+		}
+	}
+
+	// Any categories not in the standard order
+	for (const [cat, arts] of grouped) {
+		if (categoryOrder.includes(cat)) continue;
+		if (arts.length === 0) continue;
+
+		const label = cat.charAt(0).toUpperCase() + cat.slice(1);
+		lines.push("", `## ${label}`, "");
+
+		for (const article of arts) {
+			const tags = article.tags.map((t) => `\`#${t}\``).join(" ");
+			const summary = article.summary ? ` -- ${article.summary}` : "";
+			lines.push(
+				`- **[${article.title}](${article.relativePath})**${summary}${tags ? ` ${tags}` : ""}`,
+			);
+		}
+	}
+
+	return lines.join("\n") + "\n";
+}
+
+/**
+ * Infer category from path when not in frontmatter.
+ */
+function categorize(relPath: string): string {
+	if (relPath.startsWith("concepts/")) return "concept";
+	if (relPath.startsWith("topics/")) return "topic";
+	if (relPath.startsWith("references/")) return "reference";
+	if (relPath.startsWith("outputs/")) return "output";
+	return "topic"; // default
+}
+
+/**
+ * Compute stats from INDEX.md or articles directly.
+ */
+export async function computeStats(root: string): Promise<{
+	totalArticles: number;
+	totalWords: number;
+}> {
+	const files = await listWiki(root);
+	const articleFiles = files.filter((f) => !f.endsWith("INDEX.md") && !f.endsWith("GRAPH.md"));
+
+	let totalWords = 0;
+	for (const filePath of articleFiles) {
+		const content = await readFile(filePath, "utf-8");
+		const { body } = parseFrontmatter(content);
+		totalWords += body.split(/\s+/).filter(Boolean).length;
+	}
+
+	return { totalArticles: articleFiles.length, totalWords };
+}