@kibhq/core 0.1.0

package/src/compile/prompts.ts

@@ -0,0 +1,124 @@
+/**
+ * System prompt for compiling raw sources into wiki articles.
+ */
+export function compileSystemPrompt(categories: string[]): string {
+	return `You are a knowledge compiler. You receive raw source material and an existing wiki index. Your job is to:
+
+1. Extract key concepts, topics, and entities from the source
+2. Create new wiki articles OR update existing ones
+3. Add [[wiki-style]] links for cross-references between articles
+4. Maintain consistent style and depth across the wiki
+
+RULES:
+- Each article should cover ONE concept or topic clearly
+- Articles should be 200-1000 words
+- Use YAML frontmatter with these fields: title, slug, category, tags, sources, created, updated, summary
+- Use [[wiki-style]] links to reference other articles (use the slug as the link target)
+- Prefer updating existing articles over creating duplicates
+- If a concept already has a wiki article, update it with new information rather than creating a new one
+- Categories: ${categories.join(", ")}
+- Slugs should be kebab-case (e.g., "transformer-architecture")
+- Tags should be lowercase, hyphenated (e.g., "deep-learning")
+- Summary should be 1-2 sentences
+
+OUTPUT FORMAT:
+Respond with ONLY a JSON array of file operations. No other text, no markdown code fences, just the raw JSON array:
+[
+  {
+    "op": "create",
+    "path": "wiki/concepts/example-concept.md",
+    "content": "---\\ntitle: Example Concept\\nslug: example-concept\\ncategory: concept\\ntags: [example, demo]\\nsources:\\n  - raw/articles/source-file.md\\ncreated: 2026-04-05\\nupdated: 2026-04-05\\nsummary: >\\n  A brief summary of this concept.\\n---\\n\\n# Example Concept\\n\\nArticle content here..."
+  },
+  {
+    "op": "update",
+    "path": "wiki/topics/existing-topic.md",
+    "content": "full updated content including frontmatter"
+  }
+]
+
+Valid operations:
+- "create": Create a new article at the given path
+- "update": Replace an existing article's content
+- "delete": Remove an article (use sparingly)`;
+}
+
+/**
+ * Build the user message for a compile pass.
+ */
+export function compileUserPrompt(params: {
+	indexContent: string;
+	sourceContent: string;
+	sourcePath: string;
+	existingArticles: { path: string; content: string }[];
+	today: string;
+}): string {
+	const parts: string[] = [];
+
+	parts.push("CURRENT WIKI INDEX:");
+	if (params.indexContent) {
+		parts.push(params.indexContent);
+	} else {
+		parts.push("(empty — this is the first compilation)");
+	}
+
+	if (params.existingArticles.length > 0) {
+		parts.push("\n\nEXISTING ARTICLES THAT MAY NEED UPDATES:");
+		for (const article of params.existingArticles) {
+			parts.push(`\n--- ${article.path} ---`);
+			parts.push(article.content);
+		}
+	}
+
+	parts.push(`\n\nNEW SOURCE TO COMPILE (from ${params.sourcePath}):`);
+	parts.push(params.sourceContent);
+
+	parts.push(`\n\nToday's date: ${params.today}`);
+
+	return parts.join("\n");
+}
+
+/**
+ * System prompt for generating INDEX.md from article metadata.
+ */
+export function indexSystemPrompt(): string {
+	return `You are a wiki index generator. Given a list of articles with their metadata, generate a clean INDEX.md file.
+
+Format:
+# Knowledge Base Index
+
+> {count} articles | {total_words} words | Last compiled: {timestamp}
+
+## Concepts
+- **[Title](path)** — Summary. \`#tag1\` \`#tag2\`
+
+## Topics
+- **[Title](path)** — Summary. \`#tag1\` \`#tag2\`
+
+## References
+- **[Title](path)** — Summary. \`#tag1\` \`#tag2\`
+
+## Outputs
+- **[Title](path)** — Summary. \`#tag1\` \`#tag2\`
+
+Only include sections that have articles. Sort articles alphabetically within each section.
+Output ONLY the markdown content, no JSON, no code fences.`;
+}
+
+/**
+ * System prompt for generating GRAPH.md from article links.
+ */
+export function graphSystemPrompt(): string {
+	return `You are a knowledge graph generator. Given articles and their forward/backward links, generate a GRAPH.md adjacency list.
+
+Format:
+# Knowledge Graph
+
+slug-a -> slug-b, slug-c, slug-d
+slug-b -> slug-a, slug-e
+slug-c -> slug-a
+
+Each line is: source-slug -> comma-separated target slugs
+Only include articles that have at least one connection.
+Sort alphabetically by source slug.
+Output ONLY the markdown content, no JSON, no code fences.`;
+}

package/src/constants.ts

@@ -0,0 +1,40 @@
+/** Directory names within a vault */
+export const VAULT_DIR = ".kb";
+export const RAW_DIR = "raw";
+export const WIKI_DIR = "wiki";
+export const INBOX_DIR = "inbox";
+export const CACHE_DIR = "cache";
+export const SKILLS_DIR = "skills";
+export const LOGS_DIR = "logs";
+
+/** Files within .kb/ */
+export const MANIFEST_FILE = "manifest.json";
+export const CONFIG_FILE = "config.toml";
+
+/** Files within wiki/ */
+export const INDEX_FILE = "INDEX.md";
+export const GRAPH_FILE = "GRAPH.md";
+
+/** Wiki categories (subdirectories of wiki/) */
+export const DEFAULT_CATEGORIES = ["concepts", "topics", "references", "outputs"] as const;
+
+/** Raw source categories (subdirectories of raw/) */
+export const RAW_CATEGORIES = ["articles", "papers", "repos", "images", "transcripts"] as const;
+
+/** Default config values */
+export const DEFAULTS = {
+	provider: "anthropic",
+	model: "claude-sonnet-4-20250514",
+	fastModel: "claude-haiku-4-5-20251001",
+	maxSourcesPerPass: 10,
+	searchMaxResults: 20,
+	cacheTtlHours: 168, // 7 days
+	cacheMaxSizeMb: 500,
+	watchPollIntervalMs: 2000,
+	maxFileSizeMb: 50,
+	compileArticleMinWords: 200,
+	compileArticleMaxWords: 1000,
+} as const;
+
+/** Manifest version */
+export const MANIFEST_VERSION = "1" as const;

package/src/errors.ts

@@ -0,0 +1,66 @@
+export class KibError extends Error {
+	constructor(
+		message: string,
+		public readonly code: string,
+	) {
+		super(message);
+		this.name = "KibError";
+	}
+}
+
+export class VaultNotFoundError extends KibError {
+	constructor(path?: string) {
+		super(
+			path
+				? `No vault found at ${path}. Run 'kib init' to create one.`
+				: "No vault found. Run 'kib init' to create one.",
+			"VAULT_NOT_FOUND",
+		);
+		this.name = "VaultNotFoundError";
+	}
+}
+
+export class VaultExistsError extends KibError {
+	constructor(path: string) {
+		super(`Vault already exists at ${path}. Use --force to reinitialize.`, "VAULT_EXISTS");
+		this.name = "VaultExistsError";
+	}
+}
+
+export class ProviderError extends KibError {
+	constructor(message: string) {
+		super(message, "PROVIDER_ERROR");
+		this.name = "ProviderError";
+	}
+}
+
+export class NoProviderError extends KibError {
+	constructor() {
+		super(
+			"No LLM provider found. Set ANTHROPIC_API_KEY, OPENAI_API_KEY, or start Ollama.",
+			"NO_PROVIDER",
+		);
+		this.name = "NoProviderError";
+	}
+}
+
+export class IngestError extends KibError {
+	constructor(message: string) {
+		super(message, "INGEST_ERROR");
+		this.name = "IngestError";
+	}
+}
+
+export class CompileError extends KibError {
+	constructor(message: string) {
+		super(message, "COMPILE_ERROR");
+		this.name = "CompileError";
+	}
+}
+
+export class ManifestError extends KibError {
+	constructor(message: string) {
+		super(message, "MANIFEST_ERROR");
+		this.name = "ManifestError";
+	}
+}

package/src/hash.test.ts

@@ -0,0 +1,21 @@
+import { describe, expect, test } from "bun:test";
+import { hash } from "./hash.js";
+
+describe("hash", () => {
+	test("returns a hex string", async () => {
+		const h = await hash("hello world");
+		expect(h).toMatch(/^[0-9a-f]+$/);
+	});
+
+	test("same input produces same hash", async () => {
+		const h1 = await hash("test content");
+		const h2 = await hash("test content");
+		expect(h1).toBe(h2);
+	});
+
+	test("different input produces different hash", async () => {
+		const h1 = await hash("content A");
+		const h2 = await hash("content B");
+		expect(h1).not.toBe(h2);
+	});
+});

package/src/hash.ts

@@ -0,0 +1,24 @@
+let h64: ((input: string) => bigint) | null = null;
+
+async function init() {
+	if (h64) return;
+	const xxhash = await import("xxhash-wasm");
+	const hasher = await xxhash.default();
+	h64 = hasher.h64;
+}
+
+/**
+ * Fast content hash using xxhash64.
+ * Returns a hex string.
+ */
+export async function hash(content: string): Promise<string> {
+	try {
+		await init();
+		return h64!(content).toString(16);
+	} catch {
+		// Fallback to Bun's built-in hasher
+		const hasher = new Bun.CryptoHasher("sha256");
+		hasher.update(content);
+		return hasher.digest("hex").slice(0, 16);
+	}
+}

package/src/index.ts

@@ -0,0 +1,22 @@
+export { buildLinkGraph, generateGraphMd } from "./compile/backlinks.js";
+export { CompileCache } from "./compile/cache.js";
+export { compileVault } from "./compile/compiler.js";
+export { extractWikilinks, parseCompileOutput, parseFrontmatter } from "./compile/diff.js";
+export { computeStats, generateIndexMd } from "./compile/index-manager.js";
+export * from "./constants.js";
+export * from "./errors.js";
+export * from "./hash.js";
+export { ingestSource } from "./ingest/ingest.js";
+export { countWords, slugify } from "./ingest/normalize.js";
+export { detectSourceType } from "./ingest/router.js";
+export { lintVault } from "./lint/lint.js";
+export { ALL_RULES } from "./lint/rules.js";
+export { createProvider, detectProvider } from "./providers/router.js";
+export { queryVault } from "./query/query.js";
+export * from "./schemas.js";
+export { SearchIndex } from "./search/engine.js";
+export { findSkill, loadSkills } from "./skills/loader.js";
+export { runSkill } from "./skills/runner.js";
+export { SkillDefinitionSchema } from "./skills/schema.js";
+export * from "./types.js";
+export * from "./vault.js";

package/src/ingest/extractors/file.test.ts

@@ -0,0 +1,129 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createFileExtractor } from "./file.js";
+
+let tempDir: string;
+const extractor = createFileExtractor();
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempDir() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-file-test-"));
+	return tempDir;
+}
+
+describe("file extractor", () => {
+	test("extracts markdown files as-is", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "notes.md");
+		await writeFile(path, "# My Notes\n\nSome content here.");
+
+		const result = await extractor.extract(path);
+		expect(result.title).toBe("My Notes"); // Extracted from H1 heading
+		expect(result.content).toBe("# My Notes\n\nSome content here.");
+		expect(result.metadata.fileType).toBe(".md");
+	});
+
+	test("extracts plain text files as-is", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "readme.txt");
+		await writeFile(path, "Plain text content.");
+
+		const result = await extractor.extract(path);
+		expect(result.title).toBe("Readme");
+		expect(result.content).toBe("Plain text content.");
+	});
+
+	test("wraps code files in fenced code blocks", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "index.ts");
+		await writeFile(path, "const x = 1;\nconsole.log(x);");
+
+		const result = await extractor.extract(path);
+		expect(result.title).toBe("Index");
+		expect(result.content).toContain("```typescript");
+		expect(result.content).toContain("const x = 1;");
+		expect(result.content).toContain("```");
+		expect(result.metadata.language).toBe("typescript");
+	});
+
+	test("wraps Python files with python language tag", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "script.py");
+		await writeFile(path, "def hello():\n    print('hello')");
+
+		const result = await extractor.extract(path);
+		expect(result.content).toContain("```python");
+		expect(result.metadata.language).toBe("python");
+	});
+
+	test("wraps JSON files in code blocks", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "config.json");
+		await writeFile(path, '{"key": "value"}');
+
+		const result = await extractor.extract(path);
+		expect(result.content).toContain("```json");
+		expect(result.content).toContain('"key": "value"');
+	});
+
+	test("wraps YAML files in code blocks", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "config.yaml");
+		await writeFile(path, "key: value");
+
+		const result = await extractor.extract(path);
+		expect(result.content).toContain("```yaml");
+	});
+
+	test("uses custom title from options", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "data.md");
+		await writeFile(path, "# Content");
+
+		const result = await extractor.extract(path, { title: "Custom Title" });
+		expect(result.title).toBe("Custom Title");
+	});
+
+	test("formats filename with dashes into title", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "my-cool-notes.md");
+		await writeFile(path, "content");
+
+		const result = await extractor.extract(path);
+		expect(result.title).toBe("My Cool Notes");
+	});
+
+	test("formats filename with underscores into title", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "project_ideas.md");
+		await writeFile(path, "content");
+
+		const result = await extractor.extract(path);
+		expect(result.title).toBe("Project Ideas");
+	});
+
+	test("handles Rust files", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "main.rs");
+		await writeFile(path, 'fn main() {\n    println!("Hello");\n}');
+
+		const result = await extractor.extract(path);
+		expect(result.content).toContain("```rust");
+		expect(result.metadata.language).toBe("rust");
+	});
+
+	test("handles Go files", async () => {
+		const dir = await makeTempDir();
+		const path = join(dir, "main.go");
+		await writeFile(path, "package main\n\nfunc main() {}");
+
+		const result = await extractor.extract(path);
+		expect(result.content).toContain("```go");
+		expect(result.metadata.language).toBe("go");
+	});
+});

package/src/ingest/extractors/file.ts

@@ -0,0 +1,136 @@
+import { readFile } from "node:fs/promises";
+import { basename, extname } from "node:path";
+import type { ExtractOptions, Extractor, ExtractResult } from "./interface.js";
+
+const CODE_EXTENSIONS = new Set([
+	".ts",
+	".js",
+	".tsx",
+	".jsx",
+	".py",
+	".go",
+	".rs",
+	".java",
+	".c",
+	".cpp",
+	".h",
+	".hpp",
+	".rb",
+	".sh",
+	".bash",
+	".zsh",
+	".sql",
+	".swift",
+	".kt",
+	".scala",
+	".cs",
+	".php",
+	".r",
+	".lua",
+	".zig",
+	".hs",
+	".ex",
+	".exs",
+	".clj",
+	".ml",
+	".fs",
+]);
+
+const LANGUAGE_MAP: Record<string, string> = {
+	".ts": "typescript",
+	".js": "javascript",
+	".tsx": "tsx",
+	".jsx": "jsx",
+	".py": "python",
+	".go": "go",
+	".rs": "rust",
+	".java": "java",
+	".c": "c",
+	".cpp": "cpp",
+	".h": "c",
+	".hpp": "cpp",
+	".rb": "ruby",
+	".sh": "bash",
+	".bash": "bash",
+	".zsh": "zsh",
+	".sql": "sql",
+	".swift": "swift",
+	".kt": "kotlin",
+	".scala": "scala",
+	".cs": "csharp",
+	".php": "php",
+	".r": "r",
+	".lua": "lua",
+	".zig": "zig",
+	".hs": "haskell",
+	".ex": "elixir",
+	".exs": "elixir",
+	".clj": "clojure",
+	".ml": "ocaml",
+	".fs": "fsharp",
+};
+
+export function createFileExtractor(): Extractor {
+	return {
+		type: "file",
+
+		async extract(filePath: string, options?: ExtractOptions): Promise<ExtractResult> {
+			const content = await readFile(filePath, "utf-8");
+			const ext = extname(filePath).toLowerCase();
+			const name = basename(filePath, ext);
+			const title = options?.title ?? extractMarkdownTitle(content) ?? formatTitle(name);
+
+			// Code files get wrapped in fenced code blocks
+			if (CODE_EXTENSIONS.has(ext)) {
+				const lang = LANGUAGE_MAP[ext] ?? "";
+				const wrappedContent = `# ${title}\n\nSource: \`${basename(filePath)}\`\n\n\`\`\`${lang}\n${content}\n\`\`\``;
+				return {
+					title,
+					content: wrappedContent,
+					metadata: { fileType: ext, language: lang },
+				};
+			}
+
+			// HTML files get a basic strip
+			if (ext === ".html" || ext === ".htm") {
+				const { extractFromHtml } = await import("./web.js");
+				return extractFromHtml(content, `file://${filePath}`, options);
+			}
+
+			// JSON/YAML/TOML get wrapped in code blocks
+			if (
+				ext === ".json" ||
+				ext === ".yaml" ||
+				ext === ".yml" ||
+				ext === ".toml" ||
+				ext === ".xml" ||
+				ext === ".csv"
+			) {
+				const lang = ext.replace(".", "");
+				const wrappedContent = `# ${title}\n\nSource: \`${basename(filePath)}\`\n\n\`\`\`${lang}\n${content}\n\`\`\``;
+				return {
+					title,
+					content: wrappedContent,
+					metadata: { fileType: ext },
+				};
+			}
+
+			// Markdown and text files pass through directly
+			return {
+				title,
+				content,
+				metadata: { fileType: ext },
+			};
+		},
+	};
+}
+
+function formatTitle(filename: string): string {
+	return filename.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+}
+
+/** Extract the first H1 heading from markdown content. */
+function extractMarkdownTitle(content: string): string | null {
+	const match = content.match(/^#\s+(.+)$/m);
+	return match?.[1]?.trim() ?? null;
+}

package/src/ingest/extractors/github.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from "bun:test";
+import { parseGithubUrl } from "./github.js";
+
+describe("github extractor", () => {
+	describe("parseGithubUrl", () => {
+		test("parses owner/repo URL", () => {
+			const result = parseGithubUrl("https://github.com/anthropics/claude-code");
+			expect(result).toEqual({ owner: "anthropics", repo: "claude-code", branch: undefined });
+		});
+
+		test("parses URL with tree/branch", () => {
+			const result = parseGithubUrl("https://github.com/anthropics/claude-code/tree/main/src");
+			expect(result).toEqual({ owner: "anthropics", repo: "claude-code", branch: "main" });
+		});
+
+		test("parses URL with www", () => {
+			const result = parseGithubUrl("https://www.github.com/user/repo");
+			expect(result).toEqual({ owner: "user", repo: "repo", branch: undefined });
+		});
+
+		test("returns null for non-github URL", () => {
+			expect(parseGithubUrl("https://gitlab.com/user/repo")).toBeNull();
+		});
+
+		test("returns null for github.com with only user", () => {
+			expect(parseGithubUrl("https://github.com/user")).toBeNull();
+		});
+
+		test("returns null for github.com root", () => {
+			expect(parseGithubUrl("https://github.com")).toBeNull();
+		});
+
+		test("returns null for invalid URL", () => {
+			expect(parseGithubUrl("not a url")).toBeNull();
+		});
+
+		test("handles trailing slashes", () => {
+			const result = parseGithubUrl("https://github.com/user/repo/");
+			expect(result).toEqual({ owner: "user", repo: "repo", branch: undefined });
+		});
+
+		test("handles whitespace", () => {
+			const result = parseGithubUrl("  https://github.com/user/repo  ");
+			expect(result).toEqual({ owner: "user", repo: "repo", branch: undefined });
+		});
+	});
+});