@kibhq/core 0.1.0

package/src/query/query.ts

@@ -0,0 +1,136 @@
+import { readFile } from "node:fs/promises";
+import { parseFrontmatter } from "../compile/diff.js";
+import { SearchIndex } from "../search/engine.js";
+import type { CompletionResult, LLMProvider, Message } from "../types.js";
+import { listWiki, readIndex } from "../vault.js";
+
+export interface QueryOptions {
+	/** Maximum articles to include as context */
+	maxArticles?: number;
+	/** Existing conversation history (for chat mode) */
+	history?: Message[];
+	/** Callback for streaming chunks */
+	onChunk?: (text: string) => void;
+}
+
+export interface QueryResult {
+	answer: string;
+	sourcePaths: string[];
+	usage: { inputTokens: number; outputTokens: number };
+}
+
+const QUERY_SYSTEM_PROMPT = `You are a knowledge assistant for a personal wiki. Answer questions using ONLY the information provided in the articles below.
+
+RULES:
+- Base your answer strictly on the provided articles
+- Cite sources using [Article Title] notation when referencing specific information
+- If the answer is not in the provided articles, say so clearly
+- Be concise and direct
+- Use markdown formatting for readability`;
+
+/**
+ * Query the knowledge base using RAG:
+ * 1. Search for relevant articles
+ * 2. Load top articles into context
+ * 3. Send to LLM with query
+ * 4. Return answer with citations
+ */
+export async function queryVault(
+	root: string,
+	question: string,
+	provider: LLMProvider,
+	options: QueryOptions = {},
+): Promise<QueryResult> {
+	const maxArticles = options.maxArticles ?? 5;
+
+	// Build or load search index
+	const index = new SearchIndex();
+	const loaded = await index.load(root);
+	if (!loaded) {
+		await index.build(root, "wiki");
+	}
+
+	// Search for relevant articles
+	const searchResults = index.search(question, { limit: maxArticles });
+
+	// Load the full articles
+	const articles: { title: string; path: string; content: string }[] = [];
+
+	for (const result of searchResults) {
+		try {
+			const content = await readFile(result.path, "utf-8");
+			const { frontmatter, body } = parseFrontmatter(content);
+			articles.push({
+				title: (frontmatter.title as string) ?? result.title ?? result.path,
+				path: result.path,
+				content: body,
+			});
+		} catch {
+			// File might have been deleted
+		}
+	}
+
+	// If no articles found, try using INDEX.md as fallback context
+	if (articles.length === 0) {
+		const indexContent = await readIndex(root);
+		if (indexContent) {
+			articles.push({
+				title: "Knowledge Base Index",
+				path: "wiki/INDEX.md",
+				content: indexContent,
+			});
+		}
+	}
+
+	// Build context from articles
+	const articleContext = articles
+		.map((a) => `--- ${a.title} (${a.path}) ---\n${a.content}`)
+		.join("\n\n");
+
+	const userMessage =
+		articles.length > 0
+			? `RELEVANT ARTICLES:\n\n${articleContext}\n\n---\n\nQUESTION: ${question}`
+			: `No relevant articles found in the knowledge base.\n\nQUESTION: ${question}`;
+
+	// Build message history
+	const messages: Message[] = [...(options.history ?? []), { role: "user", content: userMessage }];
+
+	// Call LLM
+	let result: CompletionResult;
+
+	if (options.onChunk) {
+		// Streaming mode
+		let fullContent = "";
+		let usage = { inputTokens: 0, outputTokens: 0 };
+
+		for await (const chunk of provider.stream({
+			system: QUERY_SYSTEM_PROMPT,
+			messages,
+		})) {
+			if (chunk.type === "text" && chunk.text) {
+				fullContent += chunk.text;
+				options.onChunk(chunk.text);
+			}
+			if (chunk.type === "usage" && chunk.usage) {
+				usage = chunk.usage;
+			}
+		}
+
+		result = {
+			content: fullContent,
+			usage,
+			stopReason: "end_turn",
+		};
+	} else {
+		result = await provider.complete({
+			system: QUERY_SYSTEM_PROMPT,
+			messages,
+		});
+	}
+
+	return {
+		answer: result.content,
+		sourcePaths: articles.map((a) => a.path),
+		usage: result.usage,
+	};
+}

package/src/schemas.ts

@@ -0,0 +1,193 @@
+import { z } from "zod";
+import { DEFAULT_CATEGORIES, DEFAULTS, MANIFEST_VERSION, RAW_CATEGORIES } from "./constants.js";
+
+// ─── Source Types ────────────────────────────────────────────────
+
+export const SourceTypeSchema = z.enum(["web", "pdf", "youtube", "github", "image", "file"]);
+
+// ─── Article Categories ──────────────────────────────────────────
+
+export const ArticleCategorySchema = z.enum(["concept", "topic", "reference", "output"]);
+
+// ─── Source Entry (in manifest) ──────────────────────────────────
+
+export const SourceEntrySchema = z.object({
+	hash: z.string(),
+	ingestedAt: z.string().datetime(),
+	lastCompiled: z.string().datetime().nullable(),
+	sourceType: SourceTypeSchema,
+	originalUrl: z.string().optional(),
+	producedArticles: z.array(z.string()),
+	metadata: z.object({
+		title: z.string().optional(),
+		author: z.string().optional(),
+		date: z.string().optional(),
+		wordCount: z.number().int().nonnegative(),
+	}),
+});
+
+// ─── Article Entry (in manifest) ─────────────────────────────────
+
+export const ArticleEntrySchema = z.object({
+	hash: z.string(),
+	createdAt: z.string().datetime(),
+	lastUpdated: z.string().datetime(),
+	derivedFrom: z.array(z.string()),
+	backlinks: z.array(z.string()),
+	forwardLinks: z.array(z.string()),
+	tags: z.array(z.string()),
+	summary: z.string(),
+	wordCount: z.number().int().nonnegative(),
+	category: ArticleCategorySchema,
+});
+
+// ─── Manifest ────────────────────────────────────────────────────
+
+export const ManifestSchema = z.object({
+	version: z.literal(MANIFEST_VERSION),
+	vault: z.object({
+		name: z.string(),
+		created: z.string().datetime(),
+		lastCompiled: z.string().datetime().nullable(),
+		provider: z.string(),
+		model: z.string(),
+	}),
+	sources: z.record(z.string(), SourceEntrySchema),
+	articles: z.record(z.string(), ArticleEntrySchema),
+	stats: z.object({
+		totalSources: z.number().int().nonnegative(),
+		totalArticles: z.number().int().nonnegative(),
+		totalWords: z.number().int().nonnegative(),
+		lastLintAt: z.string().datetime().nullable(),
+	}),
+});
+
+// ─── Vault Config ────────────────────────────────────────────────
+
+export const VaultConfigSchema = z.object({
+	provider: z.object({
+		default: z.string().default(DEFAULTS.provider),
+		model: z.string().default(DEFAULTS.model),
+		fast_model: z.string().default(DEFAULTS.fastModel),
+	}),
+	compile: z.object({
+		auto_index: z.boolean().default(true),
+		auto_graph: z.boolean().default(true),
+		max_sources_per_pass: z.number().int().positive().default(DEFAULTS.maxSourcesPerPass),
+		categories: z.array(z.string()).default([...DEFAULT_CATEGORIES]),
+	}),
+	ingest: z.object({
+		download_images: z.boolean().default(true),
+		max_file_size_mb: z.number().positive().default(DEFAULTS.maxFileSizeMb),
+		default_category: z.string().default("articles"),
+	}),
+	watch: z.object({
+		enabled: z.boolean().default(false),
+		inbox_path: z.string().default("inbox"),
+		auto_compile: z.boolean().default(true),
+		poll_interval_ms: z.number().int().positive().default(DEFAULTS.watchPollIntervalMs),
+	}),
+	search: z.object({
+		engine: z.string().default("builtin"),
+		max_results: z.number().int().positive().default(DEFAULTS.searchMaxResults),
+	}),
+	query: z.object({
+		file_output: z.boolean().default(true),
+		auto_file: z.boolean().default(false),
+	}),
+	cache: z.object({
+		enabled: z.boolean().default(true),
+		ttl_hours: z.number().int().positive().default(DEFAULTS.cacheTtlHours),
+		max_size_mb: z.number().positive().default(DEFAULTS.cacheMaxSizeMb),
+	}),
+});
+
+// ─── Article Frontmatter ─────────────────────────────────────────
+
+export const ArticleFrontmatterSchema = z.object({
+	title: z.string(),
+	slug: z.string(),
+	category: ArticleCategorySchema,
+	tags: z.array(z.string()),
+	sources: z.array(z.string()),
+	created: z.string(),
+	updated: z.string(),
+	summary: z.string(),
+});
+
+// ─── LLM Provider Types ─────────────────────────────────────────
+
+export const MessageRoleSchema = z.enum(["user", "assistant"]);
+
+export const MessageSchema = z.object({
+	role: MessageRoleSchema,
+	content: z.string(),
+});
+
+export const CompletionParamsSchema = z.object({
+	system: z.string(),
+	messages: z.array(MessageSchema),
+	maxTokens: z.number().int().positive().optional(),
+	temperature: z.number().min(0).max(2).optional(),
+});
+
+export const CompletionResultSchema = z.object({
+	content: z.string(),
+	usage: z.object({
+		inputTokens: z.number().int().nonnegative(),
+		outputTokens: z.number().int().nonnegative(),
+	}),
+	stopReason: z.enum(["end_turn", "max_tokens", "tool_use"]),
+});
+
+// ─── Compile File Operation ──────────────────────────────────────
+
+export const FileOperationSchema = z.object({
+	op: z.enum(["create", "update", "delete"]),
+	path: z.string(),
+	content: z.string().optional(),
+});
+
+// ─── Search Result ───────────────────────────────────────────────
+
+export const SearchResultSchema = z.object({
+	path: z.string(),
+	score: z.number(),
+	snippet: z.string(),
+	title: z.string().optional(),
+});
+
+// ─── Ingest Result ───────────────────────────────────────────────
+
+export const IngestResultSchema = z.object({
+	sourceId: z.string(),
+	path: z.string(),
+	sourceType: SourceTypeSchema,
+	title: z.string(),
+	wordCount: z.number().int().nonnegative(),
+	skipped: z.boolean(),
+	skipReason: z.string().optional(),
+});
+
+// ─── Compile Result ──────────────────────────────────────────────
+
+export const CompileResultSchema = z.object({
+	sourcesCompiled: z.number().int().nonnegative(),
+	articlesCreated: z.number().int().nonnegative(),
+	articlesUpdated: z.number().int().nonnegative(),
+	articlesDeleted: z.number().int().nonnegative(),
+	operations: z.array(FileOperationSchema),
+});
+
+// ─── Lint Diagnostic ─────────────────────────────────────────────
+
+export const LintSeveritySchema = z.enum(["error", "warning", "info"]);
+export const LintRuleSchema = z.enum(["orphan", "stale", "missing", "broken-link", "frontmatter"]);
+
+export const LintDiagnosticSchema = z.object({
+	rule: LintRuleSchema,
+	severity: LintSeveritySchema,
+	message: z.string(),
+	path: z.string().optional(),
+	fixable: z.boolean(),
+});

package/src/search/engine.test.ts

@@ -0,0 +1,230 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { initVault, writeWiki } from "../vault.js";
+import { SearchIndex } from "./engine.js";
+
+let tempDir: string;
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempVault() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-search-test-"));
+	await initVault(tempDir, { name: "test" });
+	return tempDir;
+}
+
+function articleMd(title: string, content: string): string {
+	return `---\ntitle: ${title}\nslug: ${title.toLowerCase().replace(/\s+/g, "-")}\n---\n\n# ${title}\n\n${content}`;
+}
+
+describe("SearchIndex", () => {
+	test("builds index from wiki files", async () => {
+		const root = await makeTempVault();
+		await writeWiki(
+			root,
+			"concepts/transformers.md",
+			articleMd(
+				"Transformer Architecture",
+				"The transformer is a neural network architecture based on self-attention mechanisms used in deep learning.",
+			),
+		);
+		await writeWiki(
+			root,
+			"topics/scaling.md",
+			articleMd(
+				"Scaling Laws",
+				"Scaling laws describe power-law relationships between compute, data, and model performance in neural networks.",
+			),
+		);
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		expect(index.documentCount).toBe(2);
+	});
+
+	test("returns relevant results for a query", async () => {
+		const root = await makeTempVault();
+		await writeWiki(
+			root,
+			"concepts/transformers.md",
+			articleMd(
+				"Transformer Architecture",
+				"The transformer is a neural network architecture based on self-attention mechanisms. It was introduced in 2017 by Vaswani et al.",
+			),
+		);
+		await writeWiki(
+			root,
+			"concepts/attention.md",
+			articleMd(
+				"Attention Mechanisms",
+				"Attention mechanisms compute weighted sums over value vectors using query-key compatibility scores. Self-attention is a special case.",
+			),
+		);
+		await writeWiki(
+			root,
+			"topics/cnn.md",
+			articleMd(
+				"Convolutional Neural Networks",
+				"CNNs use convolutional layers to detect spatial patterns in images and other grid-structured data. They are unrelated to attention.",
+			),
+		);
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		const results = index.search("self-attention transformer");
+		expect(results.length).toBeGreaterThan(0);
+
+		// Transformer article should rank highest (has both terms)
+		expect(results[0]!.title).toBe("Transformer Architecture");
+
+		// Attention article should also appear
+		expect(results.some((r) => r.title === "Attention Mechanisms")).toBe(true);
+	});
+
+	test("returns empty results for unmatched query", async () => {
+		const root = await makeTempVault();
+		await writeWiki(root, "concepts/test.md", articleMd("Test", "Some content about testing."));
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		const results = index.search("quantum computing blockchain");
+		expect(results).toHaveLength(0);
+	});
+
+	test("respects limit parameter", async () => {
+		const root = await makeTempVault();
+
+		// Create many articles that all match "neural"
+		for (let i = 0; i < 10; i++) {
+			await writeWiki(
+				root,
+				`concepts/article-${i}.md`,
+				articleMd(`Neural Network ${i}`, `Article ${i} about neural networks and deep learning.`),
+			);
+		}
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		const results = index.search("neural", { limit: 3 });
+		expect(results).toHaveLength(3);
+	});
+
+	test("returns results with scores and snippets", async () => {
+		const root = await makeTempVault();
+		await writeWiki(
+			root,
+			"concepts/test.md",
+			articleMd("Test Article", "This is a test article about knowledge compilation."),
+		);
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		const results = index.search("knowledge compilation");
+		expect(results.length).toBeGreaterThan(0);
+		expect(results[0]!.score).toBeGreaterThan(0);
+		expect(results[0]!.snippet).toBeTruthy();
+		expect(results[0]!.path).toContain("test.md");
+	});
+
+	test("handles empty index gracefully", async () => {
+		const root = await makeTempVault();
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		const results = index.search("anything");
+		expect(results).toHaveLength(0);
+	});
+
+	test("handles empty query gracefully", async () => {
+		const root = await makeTempVault();
+		await writeWiki(root, "concepts/test.md", articleMd("Test", "Content."));
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		const results = index.search("");
+		expect(results).toHaveLength(0);
+	});
+
+	test("save and load round-trip preserves search ability", async () => {
+		const root = await makeTempVault();
+		await writeWiki(
+			root,
+			"concepts/ai.md",
+			articleMd(
+				"Artificial Intelligence",
+				"AI is the simulation of human intelligence by machines.",
+			),
+		);
+
+		const index1 = new SearchIndex();
+		await index1.build(root, "wiki");
+		await index1.save(root);
+
+		const index2 = new SearchIndex();
+		const loaded = await index2.load(root);
+		expect(loaded).toBe(true);
+		expect(index2.documentCount).toBe(1);
+
+		const results = index2.search("artificial intelligence");
+		expect(results.length).toBeGreaterThan(0);
+		expect(results[0]!.title).toBe("Artificial Intelligence");
+	});
+
+	test("load returns false for missing index", async () => {
+		const root = await makeTempVault();
+		const index = new SearchIndex();
+		const loaded = await index.load(root);
+		expect(loaded).toBe(false);
+	});
+
+	test("skips INDEX.md and GRAPH.md", async () => {
+		const root = await makeTempVault();
+		await writeWiki(root, "INDEX.md", "# Index\nindex content");
+		await writeWiki(root, "GRAPH.md", "# Graph\ngraph content");
+		await writeWiki(root, "concepts/real.md", articleMd("Real Article", "Actual content."));
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		expect(index.documentCount).toBe(1);
+	});
+
+	test("title gets boosted in ranking", async () => {
+		const root = await makeTempVault();
+
+		// Article with "transformer" in title
+		await writeWiki(
+			root,
+			"concepts/transformer.md",
+			articleMd("Transformer", "A neural network architecture."),
+		);
+
+		// Article with "transformer" only in body
+		await writeWiki(
+			root,
+			"concepts/overview.md",
+			articleMd(
+				"Deep Learning Overview",
+				"Various architectures include the transformer and others.",
+			),
+		);
+
+		const index = new SearchIndex();
+		await index.build(root, "wiki");
+
+		const results = index.search("transformer");
+		expect(results.length).toBe(2);
+		// Title match should rank higher
+		expect(results[0]!.title).toBe("Transformer");
+	});
+});