@kibhq/core 0.1.0

package/src/lint/rules.ts

@@ -0,0 +1,178 @@
+import { readFile } from "node:fs/promises";
+import { relative } from "node:path";
+import { extractWikilinks, parseFrontmatter } from "../compile/diff.js";
+import { WIKI_DIR } from "../constants.js";
+import type { LintDiagnostic, Manifest } from "../types.js";
+import { listWiki } from "../vault.js";
+
+type LintRuleFn = (root: string, manifest: Manifest) => Promise<LintDiagnostic[]>;
+
+/**
+ * Find articles with no backlinks from other articles (orphans).
+ */
+export const orphanRule: LintRuleFn = async (root, manifest) => {
+	const diagnostics: LintDiagnostic[] = [];
+
+	for (const [slug, article] of Object.entries(manifest.articles)) {
+		if (article.backlinks.length === 0 && article.category !== "output") {
+			diagnostics.push({
+				rule: "orphan",
+				severity: "warning",
+				message: `No backlinks from other articles`,
+				path: `wiki/${article.category}s/${slug}.md`,
+				fixable: false,
+			});
+		}
+	}
+
+	return diagnostics;
+};
+
+/**
+ * Find [[wikilinks]] that point to non-existent articles.
+ */
+export const brokenLinkRule: LintRuleFn = async (root, manifest) => {
+	const diagnostics: LintDiagnostic[] = [];
+	const wikiDir = `${root}/${WIKI_DIR}`;
+	const files = await listWiki(root);
+	const articleFiles = files.filter((f) => !f.endsWith("INDEX.md") && !f.endsWith("GRAPH.md"));
+
+	// Build set of known slugs
+	const knownSlugs = new Set(Object.keys(manifest.articles));
+
+	for (const filePath of articleFiles) {
+		const content = await readFile(filePath, "utf-8");
+		const links = extractWikilinks(content);
+		const relPath = relative(wikiDir, filePath);
+
+		for (const link of links) {
+			if (!knownSlugs.has(link)) {
+				diagnostics.push({
+					rule: "broken-link",
+					severity: "error",
+					message: `Broken wikilink [[${link}]] — no article with this slug exists`,
+					path: relPath,
+					fixable: false,
+				});
+			}
+		}
+	}
+
+	return diagnostics;
+};
+
+/**
+ * Find sources whose content hash changed but article hasn't been recompiled.
+ */
+export const staleRule: LintRuleFn = async (_root, manifest) => {
+	const diagnostics: LintDiagnostic[] = [];
+
+	for (const [sourceId, source] of Object.entries(manifest.sources)) {
+		if (!source.lastCompiled || source.lastCompiled < source.ingestedAt) {
+			diagnostics.push({
+				rule: "stale",
+				severity: "warning",
+				message: `Source "${source.metadata.title ?? sourceId}" has not been compiled since last ingest`,
+				path: source.producedArticles[0],
+				fixable: true,
+			});
+		}
+	}
+
+	return diagnostics;
+};
+
+/**
+ * Find articles with missing or malformed YAML frontmatter.
+ */
+export const frontmatterRule: LintRuleFn = async (root) => {
+	const diagnostics: LintDiagnostic[] = [];
+	const wikiDir = `${root}/${WIKI_DIR}`;
+	const files = await listWiki(root);
+	const articleFiles = files.filter((f) => !f.endsWith("INDEX.md") && !f.endsWith("GRAPH.md"));
+
+	const requiredFields = ["title", "slug", "category"];
+
+	for (const filePath of articleFiles) {
+		const content = await readFile(filePath, "utf-8");
+		const relPath = relative(wikiDir, filePath);
+
+		// Check if frontmatter exists at all
+		if (!content.startsWith("---")) {
+			diagnostics.push({
+				rule: "frontmatter",
+				severity: "error",
+				message: "Missing YAML frontmatter",
+				path: relPath,
+				fixable: false,
+			});
+			continue;
+		}
+
+		const { frontmatter } = parseFrontmatter(content);
+
+		for (const field of requiredFields) {
+			if (!frontmatter[field]) {
+				diagnostics.push({
+					rule: "frontmatter",
+					severity: "error",
+					message: `Missing required frontmatter field: ${field}`,
+					path: relPath,
+					fixable: false,
+				});
+			}
+		}
+	}
+
+	return diagnostics;
+};
+
+/**
+ * Find concepts/topics mentioned across multiple articles that don't have their own article.
+ */
+export const missingRule: LintRuleFn = async (root, manifest) => {
+	const diagnostics: LintDiagnostic[] = [];
+	const wikiDir = `${root}/${WIKI_DIR}`;
+	const files = await listWiki(root);
+	const articleFiles = files.filter((f) => !f.endsWith("INDEX.md") && !f.endsWith("GRAPH.md"));
+
+	// Collect all wikilinks across all articles
+	const linkCounts = new Map<string, number>();
+	const knownSlugs = new Set(Object.keys(manifest.articles));
+
+	for (const filePath of articleFiles) {
+		const content = await readFile(filePath, "utf-8");
+		const links = extractWikilinks(content);
+
+		for (const link of links) {
+			if (!knownSlugs.has(link)) {
+				linkCounts.set(link, (linkCounts.get(link) ?? 0) + 1);
+			}
+		}
+	}
+
+	// Report topics mentioned in 3+ articles without a dedicated article
+	for (const [slug, count] of linkCounts) {
+		if (count >= 3) {
+			diagnostics.push({
+				rule: "missing",
+				severity: "info",
+				message: `"${slug}" is referenced in ${count} articles but has no dedicated article`,
+				fixable: true,
+			});
+		}
+	}
+
+	return diagnostics;
+};
+
+/**
+ * All lint rules.
+ */
+export const ALL_RULES: { name: string; fn: LintRuleFn }[] = [
+	{ name: "orphan", fn: orphanRule },
+	{ name: "broken-link", fn: brokenLinkRule },
+	{ name: "stale", fn: staleRule },
+	{ name: "frontmatter", fn: frontmatterRule },
+	{ name: "missing", fn: missingRule },
+];

package/src/providers/anthropic.ts

@@ -0,0 +1,107 @@
+import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
+
+// Lazy-loaded SDK
+let AnthropicClass: any = null;
+
+async function getClient() {
+	if (!AnthropicClass) {
+		const mod = await import("@anthropic-ai/sdk");
+		AnthropicClass = mod.default;
+	}
+	return new AnthropicClass();
+}
+
+export function createAnthropicProvider(model: string): LLMProvider {
+	return {
+		name: "anthropic",
+
+		async complete(params: CompletionParams): Promise<CompletionResult> {
+			const client = await getClient();
+			const response = await client.messages.create({
+				model,
+				max_tokens: params.maxTokens ?? 4096,
+				temperature: params.temperature ?? 0,
+				system: params.system,
+				messages: params.messages.map((m) => ({
+					role: m.role,
+					content: m.content,
+				})),
+			});
+
+			const content = response.content
+				.filter((b: any) => b.type === "text")
+				.map((b: any) => b.text)
+				.join("");
+
+			return {
+				content,
+				usage: {
+					inputTokens: response.usage.input_tokens,
+					outputTokens: response.usage.output_tokens,
+				},
+				stopReason: response.stop_reason === "end_turn" ? "end_turn" : "max_tokens",
+			};
+		},
+
+		async *stream(params: CompletionParams): AsyncIterable<StreamChunk> {
+			const client = await getClient();
+			const stream = await client.messages.stream({
+				model,
+				max_tokens: params.maxTokens ?? 4096,
+				temperature: params.temperature ?? 0,
+				system: params.system,
+				messages: params.messages.map((m) => ({
+					role: m.role,
+					content: m.content,
+				})),
+			});
+
+			for await (const event of stream) {
+				if (event.type === "content_block_delta" && event.delta.type === "text_delta") {
+					yield { type: "text", text: event.delta.text };
+				}
+				if (event.type === "message_delta" && event.usage) {
+					yield {
+						type: "usage",
+						usage: {
+							inputTokens: event.usage.input_tokens ?? 0,
+							outputTokens: event.usage.output_tokens ?? 0,
+						},
+					};
+				}
+			}
+		},
+
+		async vision(params: { image: Buffer; prompt: string }): Promise<string> {
+			const client = await getClient();
+			const response = await client.messages.create({
+				model,
+				max_tokens: 4096,
+				messages: [
+					{
+						role: "user",
+						content: [
+							{
+								type: "image",
+								source: {
+									type: "base64",
+									media_type: "image/png",
+									data: params.image.toString("base64"),
+								},
+							},
+							{
+								type: "text",
+								text: params.prompt,
+							},
+						],
+					},
+				],
+			});
+
+			return response.content
+				.filter((b: any) => b.type === "text")
+				.map((b: any) => b.text)
+				.join("");
+		},
+	};
+}

package/src/providers/index.ts

@@ -0,0 +1,4 @@
+export { createAnthropicProvider } from "./anthropic.js";
+export { createOllamaProvider } from "./ollama.js";
+export { createOpenAIProvider } from "./openai.js";
+export { createProvider, detectProvider } from "./router.js";

package/src/providers/ollama.ts

@@ -0,0 +1,101 @@
+import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
+
+const OLLAMA_BASE = "http://localhost:11434";
+
+export function createOllamaProvider(model: string): LLMProvider {
+	return {
+		name: "ollama",
+
+		async complete(params: CompletionParams): Promise<CompletionResult> {
+			const messages = [
+				{ role: "system", content: params.system },
+				...params.messages.map((m) => ({ role: m.role, content: m.content })),
+			];
+
+			const response = await fetch(`${OLLAMA_BASE}/api/chat`, {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					model,
+					messages,
+					stream: false,
+					options: {
+						temperature: params.temperature ?? 0,
+						num_predict: params.maxTokens ?? 4096,
+					},
+				}),
+			});
+
+			if (!response.ok) {
+				throw new Error(`Ollama request failed: ${response.status} ${response.statusText}`);
+			}
+
+			const data = (await response.json()) as any;
+			return {
+				content: data.message?.content ?? "",
+				usage: {
+					inputTokens: data.prompt_eval_count ?? 0,
+					outputTokens: data.eval_count ?? 0,
+				},
+				stopReason: "end_turn",
+			};
+		},
+
+		async *stream(params: CompletionParams): AsyncIterable<StreamChunk> {
+			const messages = [
+				{ role: "system", content: params.system },
+				...params.messages.map((m) => ({ role: m.role, content: m.content })),
+			];
+
+			const response = await fetch(`${OLLAMA_BASE}/api/chat`, {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					model,
+					messages,
+					stream: true,
+					options: {
+						temperature: params.temperature ?? 0,
+						num_predict: params.maxTokens ?? 4096,
+					},
+				}),
+			});
+
+			if (!response.ok) {
+				throw new Error(`Ollama request failed: ${response.status} ${response.statusText}`);
+			}
+
+			const reader = response.body?.getReader();
+			if (!reader) return;
+
+			const decoder = new TextDecoder();
+			let buffer = "";
+
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				buffer += decoder.decode(value, { stream: true });
+				const lines = buffer.split("\n");
+				buffer = lines.pop() ?? "";
+
+				for (const line of lines) {
+					if (!line.trim()) continue;
+					const data = JSON.parse(line) as any;
+					if (data.message?.content) {
+						yield { type: "text", text: data.message.content };
+					}
+					if (data.done && data.eval_count) {
+						yield {
+							type: "usage",
+							usage: {
+								inputTokens: data.prompt_eval_count ?? 0,
+								outputTokens: data.eval_count ?? 0,
+							},
+						};
+					}
+				}
+			}
+		},
+	};
+}

package/src/providers/openai.ts

@@ -0,0 +1,67 @@
+import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
+
+// Lazy-loaded SDK
+let OpenAIClass: any = null;
+
+async function getClient() {
+	if (!OpenAIClass) {
+		const mod = await import("openai");
+		OpenAIClass = mod.default;
+	}
+	return new OpenAIClass();
+}
+
+export function createOpenAIProvider(model: string): LLMProvider {
+	return {
+		name: "openai",
+
+		async complete(params: CompletionParams): Promise<CompletionResult> {
+			const client = await getClient();
+			const response = await client.chat.completions.create({
+				model,
+				max_tokens: params.maxTokens ?? 4096,
+				temperature: params.temperature ?? 0,
+				messages: [
+					{ role: "system", content: params.system },
+					...params.messages.map((m) => ({
+						role: m.role as "user" | "assistant",
+						content: m.content,
+					})),
+				],
+			});
+
+			return {
+				content: response.choices[0]?.message?.content ?? "",
+				usage: {
+					inputTokens: response.usage?.prompt_tokens ?? 0,
+					outputTokens: response.usage?.completion_tokens ?? 0,
+				},
+				stopReason: response.choices[0]?.finish_reason === "stop" ? "end_turn" : "max_tokens",
+			};
+		},
+
+		async *stream(params: CompletionParams): AsyncIterable<StreamChunk> {
+			const client = await getClient();
+			const stream = await client.chat.completions.create({
+				model,
+				max_tokens: params.maxTokens ?? 4096,
+				temperature: params.temperature ?? 0,
+				stream: true,
+				messages: [
+					{ role: "system", content: params.system },
+					...params.messages.map((m) => ({
+						role: m.role as "user" | "assistant",
+						content: m.content,
+					})),
+				],
+			});
+
+			for await (const chunk of stream) {
+				const delta = chunk.choices[0]?.delta?.content;
+				if (delta) {
+					yield { type: "text", text: delta };
+				}
+			}
+		},
+	};
+}

package/src/providers/router.ts

@@ -0,0 +1,62 @@
+import { DEFAULTS } from "../constants.js";
+import { NoProviderError } from "../errors.js";
+import type { LLMProvider } from "../types.js";
+
+interface DetectedProvider {
+	name: string;
+	model: string;
+}
+
+/**
+ * Auto-detect LLM provider from environment variables.
+ */
+export function detectProvider(): DetectedProvider {
+	if (process.env.ANTHROPIC_API_KEY) {
+		return { name: "anthropic", model: DEFAULTS.model };
+	}
+	if (process.env.OPENAI_API_KEY) {
+		return { name: "openai", model: "gpt-4o" };
+	}
+	// Ollama detection is async — handled in createProvider
+	return { name: "ollama", model: "llama3" };
+}
+
+/**
+ * Create an LLM provider instance.
+ * Lazy-loads the SDK for the selected provider.
+ */
+export async function createProvider(providerName?: string, model?: string): Promise<LLMProvider> {
+	const detected = detectProvider();
+	const name = providerName ?? detected.name;
+	const selectedModel = model ?? detected.model;
+
+	switch (name) {
+		case "anthropic": {
+			if (!process.env.ANTHROPIC_API_KEY) {
+				throw new NoProviderError();
+			}
+			const { createAnthropicProvider } = await import("./anthropic.js");
+			return createAnthropicProvider(selectedModel);
+		}
+		case "openai": {
+			if (!process.env.OPENAI_API_KEY) {
+				throw new NoProviderError();
+			}
+			const { createOpenAIProvider } = await import("./openai.js");
+			return createOpenAIProvider(selectedModel);
+		}
+		case "ollama": {
+			// Check if Ollama is running
+			try {
+				const res = await fetch("http://localhost:11434/api/tags");
+				if (!res.ok) throw new Error("Not running");
+			} catch {
+				throw new NoProviderError();
+			}
+			const { createOllamaProvider } = await import("./ollama.js");
+			return createOllamaProvider(selectedModel);
+		}
+		default:
+			throw new NoProviderError();
+	}
+}

package/src/query/query.test.ts

@@ -0,0 +1,165 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { SearchIndex } from "../search/engine.js";
+import type { CompletionParams, CompletionResult, LLMProvider, StreamChunk } from "../types.js";
+import { initVault, writeWiki } from "../vault.js";
+import { queryVault } from "./query.js";
+
+let tempDir: string;
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempVault() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-query-test-"));
+	await initVault(tempDir, { name: "test" });
+	return tempDir;
+}
+
+function mockProvider(response: string): LLMProvider {
+	return {
+		name: "mock",
+		async complete(): Promise<CompletionResult> {
+			return {
+				content: response,
+				usage: { inputTokens: 500, outputTokens: 100 },
+				stopReason: "end_turn",
+			};
+		},
+		async *stream(): AsyncIterable<StreamChunk> {
+			for (const char of response) {
+				yield { type: "text", text: char };
+			}
+			yield { type: "usage", usage: { inputTokens: 500, outputTokens: 100 } };
+		},
+	};
+}
+
+function articleMd(title: string, slug: string, content: string): string {
+	return `---
+title: ${title}
+slug: ${slug}
+category: concept
+tags: []
+summary: ""
+---
+
+# ${title}
+
+${content}`;
+}
+
+describe("queryVault", () => {
+	test("queries relevant articles and returns answer", async () => {
+		const root = await makeTempVault();
+
+		await writeWiki(
+			root,
+			"concepts/transformers.md",
+			articleMd(
+				"Transformer Architecture",
+				"transformer-architecture",
+				"The transformer is a neural network architecture based on self-attention.",
+			),
+		);
+		await writeWiki(
+			root,
+			"concepts/attention.md",
+			articleMd(
+				"Attention Mechanisms",
+				"attention-mechanisms",
+				"Attention computes weighted sums over value vectors.",
+			),
+		);
+
+		// Build search index so query can find articles
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+		await index.save(root);
+
+		const provider = mockProvider(
+			"The transformer architecture uses self-attention [Transformer Architecture].",
+		);
+
+		const result = await queryVault(root, "How do transformers work?", provider);
+
+		expect(result.answer).toContain("transformer");
+		expect(result.sourcePaths.length).toBeGreaterThan(0);
+		expect(result.usage.inputTokens).toBeGreaterThan(0);
+	});
+
+	test("returns answer even with empty wiki", async () => {
+		const root = await makeTempVault();
+
+		const provider = mockProvider("No relevant articles found in the knowledge base.");
+
+		const result = await queryVault(root, "What is a transformer?", provider);
+		expect(result.answer).toBeTruthy();
+	});
+
+	test("supports streaming mode", async () => {
+		const root = await makeTempVault();
+		await writeWiki(
+			root,
+			"concepts/test.md",
+			articleMd("Test", "test", "Test content for streaming."),
+		);
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+		await index.save(root);
+
+		const provider = mockProvider("Streamed answer.");
+
+		const chunks: string[] = [];
+		const result = await queryVault(root, "test", provider, {
+			onChunk: (text) => chunks.push(text),
+		});
+
+		expect(chunks.length).toBeGreaterThan(0);
+		expect(chunks.join("")).toBe("Streamed answer.");
+		expect(result.answer).toBe("Streamed answer.");
+	});
+
+	test("supports conversation history", async () => {
+		const root = await makeTempVault();
+		await writeWiki(root, "concepts/test.md", articleMd("Test", "test", "Test content."));
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+		await index.save(root);
+
+		// Track what gets sent to the provider
+		let receivedMessages: any[] = [];
+		const provider: LLMProvider = {
+			name: "mock",
+			async complete(params: CompletionParams): Promise<CompletionResult> {
+				receivedMessages = params.messages;
+				return {
+					content: "Answer with history.",
+					usage: { inputTokens: 100, outputTokens: 50 },
+					stopReason: "end_turn",
+				};
+			},
+			async *stream(): AsyncIterable<StreamChunk> {
+				yield { type: "text", text: "stream" };
+			},
+		};
+
+		await queryVault(root, "follow up question", provider, {
+			history: [
+				{ role: "user", content: "previous question" },
+				{ role: "assistant", content: "previous answer" },
+			],
+		});
+
+		// Should include history + new question
+		expect(receivedMessages.length).toBe(3);
+		expect(receivedMessages[0]!.content).toBe("previous question");
+		expect(receivedMessages[1]!.content).toBe("previous answer");
+		expect(receivedMessages[2]!.content).toContain("follow up question");
+	});
+});