@kibhq/core 0.1.0

package/package.json

@@ -0,0 +1,40 @@
+{
+	"name": "@kibhq/core",
+	"version": "0.1.0",
+	"type": "module",
+	"exports": {
+		".": "./src/index.ts"
+	},
+	"files": [
+		"src",
+		"package.json"
+	],
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/keeganthomp/kib.git",
+		"directory": "packages/core"
+	},
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"scripts": {
+		"test": "bun test"
+	},
+	"dependencies": {
+		"@anthropic-ai/sdk": "^0.82.0",
+		"@iarna/toml": "^2.2.5",
+		"cheerio": "^1.2.0",
+		"openai": "^6.33.0",
+		"pdf-parse": "^2.4.5",
+		"turndown": "^7.2.4",
+		"xxhash-wasm": "^1.1.0",
+		"yaml": "^2.7.1",
+		"zod": "^3.24.4"
+	},
+	"devDependencies": {
+		"@types/bun": "latest",
+		"@types/turndown": "^5.0.6",
+		"typescript": "^5.8.3"
+	}
+}

package/src/compile/backlinks.test.ts

@@ -0,0 +1,112 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { initVault, writeWiki } from "../vault.js";
+import { buildLinkGraph, generateGraphMd } from "./backlinks.js";
+
+let tempDir: string;
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempVault() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-backlinks-test-"));
+	await initVault(tempDir, { name: "test" });
+	return tempDir;
+}
+
+describe("buildLinkGraph", () => {
+	test("builds forward and backward links", async () => {
+		const root = await makeTempVault();
+
+		await writeWiki(
+			root,
+			"concepts/transformers.md",
+			"---\nslug: transformers\n---\n\nSee [[attention-mechanisms]] and [[positional-encoding]].",
+		);
+		await writeWiki(
+			root,
+			"concepts/attention-mechanisms.md",
+			"---\nslug: attention-mechanisms\n---\n\nPart of [[transformers]].",
+		);
+		await writeWiki(
+			root,
+			"concepts/positional-encoding.md",
+			"---\nslug: positional-encoding\n---\n\nUsed in [[transformers]].",
+		);
+
+		const graph = await buildLinkGraph(root);
+
+		// Forward links
+		expect([...graph.forwardLinks.get("transformers")!]).toEqual(
+			expect.arrayContaining(["attention-mechanisms", "positional-encoding"]),
+		);
+		expect([...graph.forwardLinks.get("attention-mechanisms")!]).toEqual(["transformers"]);
+		expect([...graph.forwardLinks.get("positional-encoding")!]).toEqual(["transformers"]);
+
+		// Backlinks
+		expect([...graph.backlinks.get("transformers")!]).toEqual(
+			expect.arrayContaining(["attention-mechanisms", "positional-encoding"]),
+		);
+		expect([...graph.backlinks.get("attention-mechanisms")!]).toEqual(["transformers"]);
+		expect([...graph.backlinks.get("positional-encoding")!]).toEqual(["transformers"]);
+	});
+
+	test("handles articles with no links", async () => {
+		const root = await makeTempVault();
+		await writeWiki(root, "concepts/orphan.md", "---\nslug: orphan\n---\n\nNo links here.");
+
+		const graph = await buildLinkGraph(root);
+		expect(graph.forwardLinks.get("orphan")!.size).toBe(0);
+		expect(graph.backlinks.get("orphan")!.size).toBe(0);
+	});
+
+	test("handles empty wiki", async () => {
+		const root = await makeTempVault();
+		const graph = await buildLinkGraph(root);
+		expect(graph.forwardLinks.size).toBe(0);
+		expect(graph.backlinks.size).toBe(0);
+	});
+});
+
+describe("generateGraphMd", () => {
+	test("generates adjacency list", async () => {
+		const root = await makeTempVault();
+		await writeWiki(root, "concepts/a.md", "---\nslug: alpha\n---\n\n[[beta]] and [[gamma]].");
+		await writeWiki(root, "concepts/b.md", "---\nslug: beta\n---\n\n[[alpha]].");
+
+		const graph = await buildLinkGraph(root);
+		const md = generateGraphMd(graph);
+
+		expect(md).toContain("# Knowledge Graph");
+		expect(md).toContain("alpha -> beta, gamma");
+		expect(md).toContain("beta -> alpha");
+	});
+
+	test("handles empty graph", () => {
+		const graph = {
+			forwardLinks: new Map<string, Set<string>>(),
+			backlinks: new Map<string, Set<string>>(),
+		};
+		const md = generateGraphMd(graph);
+		expect(md).toContain("(no connections yet)");
+	});
+
+	test("sorts slugs alphabetically", async () => {
+		const root = await makeTempVault();
+		await writeWiki(root, "concepts/c.md", "---\nslug: charlie\n---\n\n[[alpha]].");
+		await writeWiki(root, "concepts/a.md", "---\nslug: alpha\n---\n\n[[charlie]].");
+		await writeWiki(root, "concepts/b.md", "---\nslug: bravo\n---\n\n[[alpha]].");
+
+		const graph = await buildLinkGraph(root);
+		const md = generateGraphMd(graph);
+		const lines = md.split("\n").filter((l) => l.includes("->"));
+
+		// Should be alpha, bravo, charlie order
+		expect(lines[0]).toMatch(/^alpha/);
+		expect(lines[1]).toMatch(/^bravo/);
+		expect(lines[2]).toMatch(/^charlie/);
+	});
+});

package/src/compile/backlinks.ts

@@ -0,0 +1,80 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { WIKI_DIR } from "../constants.js";
+import { listWiki } from "../vault.js";
+import { extractWikilinks, parseFrontmatter } from "./diff.js";
+
+export interface LinkGraph {
+	/** slug → set of slugs this article links to */
+	forwardLinks: Map<string, Set<string>>;
+	/** slug → set of slugs that link to this article */
+	backlinks: Map<string, Set<string>>;
+}
+
+/**
+ * Scan all wiki articles and build the full link graph.
+ */
+export async function buildLinkGraph(root: string): Promise<LinkGraph> {
+	const forwardLinks = new Map<string, Set<string>>();
+	const backlinks = new Map<string, Set<string>>();
+
+	const files = await listWiki(root);
+
+	for (const filePath of files) {
+		const content = await readFile(filePath, "utf-8");
+		const { frontmatter } = parseFrontmatter(content);
+		const slug = (frontmatter.slug as string) ?? extractSlugFromPath(filePath);
+
+		if (!slug) continue;
+
+		// Extract forward links
+		const links = extractWikilinks(content);
+		forwardLinks.set(slug, new Set(links));
+
+		// Initialize backlinks set for this article
+		if (!backlinks.has(slug)) {
+			backlinks.set(slug, new Set());
+		}
+
+		// Add backlinks for each target
+		for (const target of links) {
+			if (!backlinks.has(target)) {
+				backlinks.set(target, new Set());
+			}
+			backlinks.get(target)!.add(slug);
+		}
+	}
+
+	return { forwardLinks, backlinks };
+}
+
+function extractSlugFromPath(filePath: string): string {
+	// Extract slug from path like .../wiki/concepts/my-article.md → my-article
+	const parts = filePath.split("/");
+	const filename = parts[parts.length - 1] ?? "";
+	return filename.replace(/\.md$/, "");
+}
+
+/**
+ * Generate GRAPH.md content from the link graph.
+ */
+export function generateGraphMd(graph: LinkGraph): string {
+	const lines: string[] = ["# Knowledge Graph", ""];
+
+	// Sort by slug for consistent output
+	const slugs = [...graph.forwardLinks.keys()].sort();
+
+	for (const slug of slugs) {
+		const targets = graph.forwardLinks.get(slug);
+		if (targets && targets.size > 0) {
+			const sorted = [...targets].sort();
+			lines.push(`${slug} -> ${sorted.join(", ")}`);
+		}
+	}
+
+	if (lines.length === 2) {
+		lines.push("(no connections yet)");
+	}
+
+	return lines.join("\n") + "\n";
+}

package/src/compile/cache.test.ts

@@ -0,0 +1,126 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { initVault } from "../vault.js";
+import { CompileCache } from "./cache.js";
+
+let tempDir: string;
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempVault() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-cache-test-"));
+	await initVault(tempDir, { name: "test" });
+	return tempDir;
+}
+
+describe("CompileCache", () => {
+	test("set and get round-trip", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root);
+
+		const key = await cache.key("system prompt", "user content");
+		await cache.set(key, "LLM response content", { inputTokens: 100, outputTokens: 200 });
+
+		const entry = await cache.get(key);
+		expect(entry).not.toBeNull();
+		expect(entry!.content).toBe("LLM response content");
+		expect(entry!.usage.inputTokens).toBe(100);
+		expect(entry!.usage.outputTokens).toBe(200);
+	});
+
+	test("returns null for missing key", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root);
+
+		const entry = await cache.get("nonexistent-key");
+		expect(entry).toBeNull();
+	});
+
+	test("returns null when disabled", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root, { enabled: false });
+
+		const key = await cache.key("system", "user");
+		await cache.set(key, "content", { inputTokens: 0, outputTokens: 0 });
+
+		const entry = await cache.get(key);
+		expect(entry).toBeNull();
+	});
+
+	test("same inputs produce same key", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root);
+
+		const key1 = await cache.key("system", "user", "model", 0);
+		const key2 = await cache.key("system", "user", "model", 0);
+		expect(key1).toBe(key2);
+	});
+
+	test("different inputs produce different keys", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root);
+
+		const key1 = await cache.key("system", "user A");
+		const key2 = await cache.key("system", "user B");
+		expect(key1).not.toBe(key2);
+	});
+
+	test("expires old entries", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root, { ttlHours: 0 }); // Expire immediately
+
+		const key = await cache.key("system", "user");
+		await cache.set(key, "content", { inputTokens: 0, outputTokens: 0 });
+
+		// Wait a tiny bit for the TTL to expire
+		await new Promise((r) => setTimeout(r, 10));
+
+		const entry = await cache.get(key);
+		expect(entry).toBeNull();
+	});
+
+	test("clear removes all entries", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root);
+
+		const key1 = await cache.key("system", "user1");
+		const key2 = await cache.key("system", "user2");
+		await cache.set(key1, "content1", { inputTokens: 0, outputTokens: 0 });
+		await cache.set(key2, "content2", { inputTokens: 0, outputTokens: 0 });
+
+		const cleared = await cache.clear();
+		expect(cleared).toBe(2);
+
+		const entry1 = await cache.get(key1);
+		const entry2 = await cache.get(key2);
+		expect(entry1).toBeNull();
+		expect(entry2).toBeNull();
+	});
+
+	test("stats returns correct counts", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root);
+
+		const key1 = await cache.key("system", "user1");
+		const key2 = await cache.key("system", "user2");
+		await cache.set(key1, "content1", { inputTokens: 0, outputTokens: 0 });
+		await cache.set(key2, "content2", { inputTokens: 0, outputTokens: 0 });
+
+		const s = await cache.stats();
+		expect(s.entries).toBe(2);
+		expect(s.sizeBytes).toBeGreaterThan(0);
+	});
+
+	test("stats returns zeros for empty cache", async () => {
+		const root = await makeTempVault();
+		const cache = new CompileCache(root);
+
+		const s = await cache.stats();
+		expect(s.entries).toBe(0);
+		expect(s.sizeBytes).toBe(0);
+	});
+});

package/src/compile/cache.ts

@@ -0,0 +1,125 @@
+import { existsSync } from "node:fs";
+import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { CACHE_DIR, VAULT_DIR } from "../constants.js";
+import { hash } from "../hash.js";
+
+interface CacheEntry {
+	content: string;
+	usage: { inputTokens: number; outputTokens: number };
+	cachedAt: string;
+	ttlHours: number;
+}
+
+/**
+ * LLM response cache.
+ * Keys are xxhash of (system + user content + model + temperature).
+ * Values are stored as JSON files in .kb/cache/responses/.
+ */
+export class CompileCache {
+	private cacheDir: string;
+	private ttlHours: number;
+	private enabled: boolean;
+
+	constructor(root: string, opts: { enabled?: boolean; ttlHours?: number } = {}) {
+		this.cacheDir = join(root, VAULT_DIR, CACHE_DIR, "responses");
+		this.ttlHours = opts.ttlHours ?? 168; // 7 days
+		this.enabled = opts.enabled ?? true;
+	}
+
+	/**
+	 * Generate a cache key from prompt parameters.
+	 */
+	async key(system: string, user: string, model?: string, temperature?: number): Promise<string> {
+		const input = `${system}\n---\n${user}\n---\n${model ?? ""}\n---\n${temperature ?? 0}`;
+		return hash(input);
+	}
+
+	/**
+	 * Get a cached response if it exists and isn't expired.
+	 */
+	async get(cacheKey: string): Promise<CacheEntry | null> {
+		if (!this.enabled) return null;
+
+		const filePath = join(this.cacheDir, `${cacheKey}.json`);
+		if (!existsSync(filePath)) return null;
+
+		try {
+			const raw = await readFile(filePath, "utf-8");
+			const entry = JSON.parse(raw) as CacheEntry;
+
+			// Check TTL
+			const cachedAt = new Date(entry.cachedAt);
+			const expiresAt = new Date(cachedAt.getTime() + entry.ttlHours * 60 * 60 * 1000);
+			if (new Date() > expiresAt) {
+				// Expired — delete it
+				await rm(filePath, { force: true });
+				return null;
+			}
+
+			return entry;
+		} catch {
+			return null;
+		}
+	}
+
+	/**
+	 * Store a response in the cache.
+	 */
+	async set(
+		cacheKey: string,
+		content: string,
+		usage: { inputTokens: number; outputTokens: number },
+	): Promise<void> {
+		if (!this.enabled) return;
+
+		await mkdir(this.cacheDir, { recursive: true });
+
+		const entry: CacheEntry = {
+			content,
+			usage,
+			cachedAt: new Date().toISOString(),
+			ttlHours: this.ttlHours,
+		};
+
+		await writeFile(
+			join(this.cacheDir, `${cacheKey}.json`),
+			JSON.stringify(entry, null, 2),
+			"utf-8",
+		);
+	}
+
+	/**
+	 * Clear all cached responses.
+	 */
+	async clear(): Promise<number> {
+		if (!existsSync(this.cacheDir)) return 0;
+
+		const files = await readdir(this.cacheDir);
+		const jsonFiles = files.filter((f) => f.endsWith(".json"));
+
+		for (const file of jsonFiles) {
+			await rm(join(this.cacheDir, file), { force: true });
+		}
+
+		return jsonFiles.length;
+	}
+
+	/**
+	 * Get cache statistics.
+	 */
+	async stats(): Promise<{ entries: number; sizeBytes: number }> {
+		if (!existsSync(this.cacheDir)) return { entries: 0, sizeBytes: 0 };
+
+		const files = await readdir(this.cacheDir);
+		const jsonFiles = files.filter((f) => f.endsWith(".json"));
+
+		let sizeBytes = 0;
+		for (const file of jsonFiles) {
+			const s = await stat(join(this.cacheDir, file));
+			sizeBytes += s.size;
+		}
+
+		return { entries: jsonFiles.length, sizeBytes };
+	}
+}