llm-kb 0.4.0 → 0.4.2

package/src/watcher.ts

@@ -1,87 +0,0 @@
-import { watch } from "chokidar";
-import { extname, join, basename } from "node:path";
-import { parsePDF } from "./pdf.js";
-import { buildIndex } from "./indexer.js";
-import { AuthStorage } from "@mariozechner/pi-coding-agent";
-import chalk from "chalk";
-
-interface WatcherOptions {
-  folder: string;
-  sourcesDir: string;
-  debounceMs?: number;
-  authStorage?: AuthStorage;
-  indexModel?: string;
-}
-
-export function startWatcher({ folder, sourcesDir, debounceMs = 2000, authStorage, indexModel }: WatcherOptions) {
-  let pendingFiles: string[] = [];
-  let debounceTimer: ReturnType<typeof setTimeout> | null = null;
-
-  async function processBatch() {
-    const files = [...pendingFiles];
-    pendingFiles = [];
-
-    if (files.length === 0) return;
-
-    console.log();
-    for (const filePath of files) {
-      const name = basename(filePath);
-      process.stdout.write(`  Parsing ${name}...`);
-      try {
-        const result = await parsePDF(filePath, sourcesDir);
-        if (result.skipped) {
-          console.log(chalk.dim(` skipped (up to date)`));
-        } else {
-          console.log(chalk.green(` ✓ ${result.totalPages} pages`));
-        }
-      } catch (err: any) {
-        console.log(chalk.red(` ✗ ${err.message}`));
-      }
-    }
-
-    // Re-index
-    process.stdout.write(`  Re-indexing...`);
-    try {
-      await buildIndex(folder, sourcesDir, undefined, authStorage, indexModel);
-      console.log(chalk.green(` ✓ index.md updated`));
-    } catch (err: any) {
-      console.log(chalk.red(` ✗ ${err.message}`));
-    }
-  }
-
-  function queueFile(filePath: string) {
-    if (!pendingFiles.includes(filePath)) {
-      pendingFiles.push(filePath);
-    }
-    if (debounceTimer) clearTimeout(debounceTimer);
-    debounceTimer = setTimeout(processBatch, debounceMs);
-  }
-
-  const watcher = watch(folder, {
-    ignoreInitial: true,
-    ignored: [
-      "**/node_modules/**",
-      "**/.llm-kb/**",
-      "**/.git/**",
-    ],
-    depth: 10,
-  });
-
-  watcher.on("add", (filePath) => {
-    const ext = extname(filePath).toLowerCase();
-    if (ext === ".pdf") {
-      console.log(chalk.dim(`\n  New file: ${basename(filePath)}`));
-      queueFile(filePath);
-    }
-  });
-
-  watcher.on("change", (filePath) => {
-    const ext = extname(filePath).toLowerCase();
-    if (ext === ".pdf") {
-      console.log(chalk.dim(`\n  Changed: ${basename(filePath)}`));
-      queueFile(filePath);
-    }
-  });
-
-  return watcher;
-}

package/src/wiki-updater.ts

@@ -1,136 +0,0 @@
-import { getModels, completeSimple } from "@mariozechner/pi-ai";
-import { AuthStorage } from "@mariozechner/pi-coding-agent";
-import { existsSync } from "node:fs";
-import { readFile, writeFile, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { homedir } from "node:os";
-import type { KBTrace } from "./trace-builder.js";
-
-async function resolveApiKey(authStorage?: AuthStorage): Promise<string | undefined> {
-  if (authStorage) {
-    return authStorage.getApiKey("anthropic");
-  }
-  const piAuthPath = join(homedir(), ".pi", "agent", "auth.json");
-  if (existsSync(piAuthPath)) {
-    const storage = AuthStorage.create(piAuthPath);
-    return storage.getApiKey("anthropic");
-  }
-  return process.env.ANTHROPIC_API_KEY;
-}
-
-function buildPrompt(
-  question: string,
-  answer: string,
-  sources: string,
-  date: string,
-  currentWiki: string
-): string {
-  const rules = `Rules for wiki structure:
-- Use ## for CONCEPTS and TOPICS — NOT source file names
-  Good: "## Electronic Evidence", "## Mob Lynching", "## Burden of Proof"
-  Bad: "## Indian Evidence Act.md", "## indian penal code - new.md"
-- Use ### for subtopics within a concept
-- A concept can draw from MULTIPLE source files — synthesize, don't separate by file
-- If knowledge from this Q&A fits an existing concept, ADD to it — never duplicate
-- If it's a genuinely new concept, create a new ## section
-- Be concise: bullet points for lists, short prose for explanations
-- Include source citations inline: (Source: filename, p.X)
-- Add cross-references where concepts relate: See also: [[Other Concept]]
-- End each ## section with: *Sources: file1, file2 · date*
-- Separate ## sections with: ---`;
-
-  if (currentWiki.trim()) {
-    return `You are maintaining a concept-organized knowledge wiki.
-
-## Current wiki
-${currentWiki}
-
-## New Q&A to integrate
-**Question:** ${question}
-**Sources used:** ${sources}
-**Date:** ${date}
-
-**Answer:**
-${answer}
-
----
-
-Update the wiki to integrate this new knowledge.
-${rules}
-
-Return ONLY the complete updated wiki markdown. No explanation.`;
-  }
-
-  return `You are creating a concept-organized knowledge wiki.
-
-## First Q&A to add
-**Question:** ${question}
-**Sources used:** ${sources}
-**Date:** ${date}
-
-**Answer:**
-${answer}
-
----
-
-Create a clean wiki from this Q&A.
-- Start with: # Knowledge Wiki\\n\\n> Concept-organized knowledge base. Updated after each query.\\n\\n---
-${rules}
-
-Return ONLY the wiki markdown. No explanation.`;
-}
-
-/**
- * Update .llm-kb/wiki/wiki.md using a direct Haiku call.
- * Organizes knowledge by CONCEPT (cross-cutting topics),
- * not by source file.
- */
-export async function updateWiki(
-  kbRoot: string,
-  trace: KBTrace,
-  authStorage?: AuthStorage,
-  indexModelId = "claude-haiku-4-5"
-): Promise<void> {
-  if (trace.mode !== "query" || !trace.question || !trace.answer) return;
-
-  const wikiDir = join(kbRoot, ".llm-kb", "wiki");
-  await mkdir(wikiDir, { recursive: true });
-  const wikiPath = join(wikiDir, "wiki.md");
-
-  const currentWiki = existsSync(wikiPath)
-    ? await readFile(wikiPath, "utf-8").catch(() => "")
-    : "";
-
-  const sources = trace.filesRead
-    .map((f) => f.split(/[\\/]/).pop() ?? f)
-    .filter((f) => f.endsWith(".md") && f !== "index.md" && f !== "wiki.md")
-    .join(", ") || "unknown";
-
-  const date = new Date(trace.timestamp).toISOString().slice(0, 10);
-  const prompt = buildPrompt(trace.question, trace.answer, sources, date, currentWiki);
-
-  const apiKey = await resolveApiKey(authStorage);
-  if (!apiKey) return;
-
-  const model = getModels("anthropic").find((m) => m.id === indexModelId);
-  if (!model) return;
-
-  const result = await completeSimple(
-    model,
-    {
-      systemPrompt: "You are a precise knowledge librarian. Organize information by CONCEPT, not by source file. Synthesize knowledge from multiple sources into unified topic articles. Return only clean markdown.",
-      messages: [{ role: "user", content: prompt, timestamp: Date.now() }],
-    },
-    { apiKey }
-  );
-
-  const text = result.content
-    .filter((b) => b.type === "text")
-    .map((b) => (b as any).text)
-    .join("")
-    .trim();
-
-  if (text) {
-    await writeFile(wikiPath, text + "\n", "utf-8");
-  }
-}

package/test/auth.test.ts

@@ -1,65 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
-
-// Mock existsSync before importing auth module
-vi.mock("node:fs", async () => {
-  const actual = await vi.importActual<typeof import("node:fs")>("node:fs");
-  return { ...actual, existsSync: vi.fn() };
-});
-
-import { checkAuth } from "../src/auth.js";
-import { existsSync } from "node:fs";
-
-const mockExistsSync = vi.mocked(existsSync);
-
-describe("checkAuth", () => {
-  const origEnv = process.env.ANTHROPIC_API_KEY;
-
-  afterEach(() => {
-    if (origEnv !== undefined) process.env.ANTHROPIC_API_KEY = origEnv;
-    else delete process.env.ANTHROPIC_API_KEY;
-    vi.restoreAllMocks();
-  });
-
-  it("returns pi-sdk when auth.json exists", () => {
-    mockExistsSync.mockReturnValue(true);
-    delete process.env.ANTHROPIC_API_KEY;
-
-    const result = checkAuth();
-    expect(result.ok).toBe(true);
-    if (result.ok) {
-      expect(result.method).toBe("pi-sdk");
-      expect(result.authStorage).toBeUndefined(); // uses default
-    }
-  });
-
-  it("returns api-key when ANTHROPIC_API_KEY is set", () => {
-    mockExistsSync.mockReturnValue(false);
-    process.env.ANTHROPIC_API_KEY = "sk-ant-test";
-
-    const result = checkAuth();
-    expect(result.ok).toBe(true);
-    if (result.ok) {
-      expect(result.method).toBe("api-key");
-      expect(result.authStorage).toBeDefined();
-    }
-  });
-
-  it("returns failure when neither is available", () => {
-    mockExistsSync.mockReturnValue(false);
-    delete process.env.ANTHROPIC_API_KEY;
-
-    const result = checkAuth();
-    expect(result.ok).toBe(false);
-  });
-
-  it("prefers pi-sdk over api-key when both exist", () => {
-    mockExistsSync.mockReturnValue(true);
-    process.env.ANTHROPIC_API_KEY = "sk-ant-test";
-
-    const result = checkAuth();
-    expect(result.ok).toBe(true);
-    if (result.ok) {
-      expect(result.method).toBe("pi-sdk");
-    }
-  });
-});

package/test/config.test.ts

@@ -1,96 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { loadConfig, ensureConfig, DEFAULT_INDEX_MODEL, DEFAULT_QUERY_MODEL } from "../src/config.js";
-import { mkdtemp, rm, readFile } from "node:fs/promises";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import { existsSync } from "node:fs";
-
-describe("config", () => {
-  let tempDir: string;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), "llm-kb-test-"));
-  });
-
-  afterEach(async () => {
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  describe("loadConfig", () => {
-    it("returns defaults when no config file exists", async () => {
-      const config = await loadConfig(tempDir);
-      expect(config.indexModel).toBe(DEFAULT_INDEX_MODEL);
-      expect(config.queryModel).toBe(DEFAULT_QUERY_MODEL);
-    });
-
-    it("reads config from .llm-kb/config.json", async () => {
-      const { mkdir, writeFile } = await import("node:fs/promises");
-      await mkdir(join(tempDir, ".llm-kb"), { recursive: true });
-      await writeFile(
-        join(tempDir, ".llm-kb", "config.json"),
-        JSON.stringify({ indexModel: "custom-haiku", queryModel: "custom-sonnet" })
-      );
-
-      const config = await loadConfig(tempDir);
-      expect(config.indexModel).toBe("custom-haiku");
-      expect(config.queryModel).toBe("custom-sonnet");
-    });
-
-    it("env vars override config file", async () => {
-      const { mkdir, writeFile } = await import("node:fs/promises");
-      await mkdir(join(tempDir, ".llm-kb"), { recursive: true });
-      await writeFile(
-        join(tempDir, ".llm-kb", "config.json"),
-        JSON.stringify({ indexModel: "from-file", queryModel: "from-file" })
-      );
-
-      process.env.LLM_KB_INDEX_MODEL = "from-env";
-      process.env.LLM_KB_QUERY_MODEL = "from-env";
-
-      try {
-        const config = await loadConfig(tempDir);
-        expect(config.indexModel).toBe("from-env");
-        expect(config.queryModel).toBe("from-env");
-      } finally {
-        delete process.env.LLM_KB_INDEX_MODEL;
-        delete process.env.LLM_KB_QUERY_MODEL;
-      }
-    });
-
-    it("handles malformed config gracefully", async () => {
-      const { mkdir, writeFile } = await import("node:fs/promises");
-      await mkdir(join(tempDir, ".llm-kb"), { recursive: true });
-      await writeFile(join(tempDir, ".llm-kb", "config.json"), "not json{{{");
-
-      const config = await loadConfig(tempDir);
-      expect(config.indexModel).toBe(DEFAULT_INDEX_MODEL);
-      expect(config.queryModel).toBe(DEFAULT_QUERY_MODEL);
-    });
-  });
-
-  describe("ensureConfig", () => {
-    it("creates config.json with defaults when none exists", async () => {
-      const config = await ensureConfig(tempDir);
-      expect(config.indexModel).toBe(DEFAULT_INDEX_MODEL);
-      expect(config.queryModel).toBe(DEFAULT_QUERY_MODEL);
-
-      const path = join(tempDir, ".llm-kb", "config.json");
-      expect(existsSync(path)).toBe(true);
-
-      const raw = JSON.parse(await readFile(path, "utf-8"));
-      expect(raw.indexModel).toBe(DEFAULT_INDEX_MODEL);
-    });
-
-    it("does not overwrite existing config", async () => {
-      const { mkdir, writeFile } = await import("node:fs/promises");
-      await mkdir(join(tempDir, ".llm-kb"), { recursive: true });
-      await writeFile(
-        join(tempDir, ".llm-kb", "config.json"),
-        JSON.stringify({ indexModel: "my-model", queryModel: "my-model" })
-      );
-
-      const config = await ensureConfig(tempDir);
-      expect(config.indexModel).toBe("my-model");
-    });
-  });
-});

package/test/md-stream.test.ts

@@ -1,98 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { MarkdownStream } from "../src/md-stream.js";
-
-describe("MarkdownStream", () => {
-  describe("non-TTY (passthrough)", () => {
-    it("returns chunks unchanged", () => {
-      const md = new MarkdownStream(false);
-      expect(md.push("**bold**")).toBe("**bold**");
-      expect(md.push("# Header\n")).toBe("# Header\n");
-    });
-  });
-
-  describe("TTY mode", () => {
-    it("renders bold text", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("**hello**\n");
-      expect(out).not.toContain("**");
-      expect(out).toContain("hello");
-    });
-
-    it("renders italic text", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("*world*\n");
-      expect(out).not.toContain("*world*");
-      expect(out).toContain("world");
-    });
-
-    it("renders inline code", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("`code`\n");
-      expect(out).not.toContain("`");
-      expect(out).toContain("code");
-    });
-
-    it("renders headers without # symbols", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("## Section Title\n");
-      expect(out).not.toContain("##");
-      expect(out).toContain("Section Title");
-    });
-
-    it("renders bullet points with •", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("- item one\n");
-      expect(out).toContain("•");
-      expect(out).toContain("item one");
-      expect(out).not.toMatch(/^- /m);
-    });
-
-    it("renders horizontal rules as ─", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("---\n");
-      expect(out).toContain("─");
-      expect(out).not.toContain("---");
-    });
-
-    it("renders block quotes with │", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("> quoted text\n");
-      expect(out).toContain("│");
-      expect(out).toContain("quoted text");
-    });
-
-    it("handles incomplete patterns by buffering", () => {
-      const md = new MarkdownStream(true);
-      // Push "**bo" — bold is incomplete, should buffer
-      const out1 = md.push("**bo");
-      // Push "ld**\n" — completes the bold
-      const out2 = md.push("ld**\n");
-      const combined = out1 + out2;
-      expect(combined).toContain("bold");
-      expect(combined).not.toContain("**");
-    });
-
-    it("flushes remaining buffer on end()", () => {
-      const md = new MarkdownStream(true);
-      // Push incomplete bold — gets buffered
-      const out1 = md.push("**unfin");
-      expect(out1).toBe(""); // buffered, not yet flushed
-      const out2 = md.end();
-      expect(out2).toContain("unfin"); // force-flushed
-    });
-
-    it("renders table separator rows as dim", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("|---|---|\n");
-      expect(out).toContain("---|---");
-    });
-
-    it("renders links by showing label and dim URL", () => {
-      const md = new MarkdownStream(true);
-      const out = md.push("[Click here](https://example.com)\n");
-      expect(out).toContain("Click here");
-      expect(out).toContain("example.com");
-      expect(out).not.toContain("[Click here]");
-    });
-  });
-});

package/test/resolve-kb.test.ts

@@ -1,33 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { resolveKnowledgeBase } from "../src/resolve-kb.js";
-import { mkdtemp, rm, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-
-describe("resolveKnowledgeBase", () => {
-  let tempDir: string;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), "llm-kb-test-"));
-  });
-
-  afterEach(async () => {
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  it("returns null when no .llm-kb directory exists", () => {
-    expect(resolveKnowledgeBase(tempDir)).toBeNull();
-  });
-
-  it("finds .llm-kb in the current directory", async () => {
-    await mkdir(join(tempDir, ".llm-kb"), { recursive: true });
-    expect(resolveKnowledgeBase(tempDir)).toBe(tempDir);
-  });
-
-  it("walks up to find .llm-kb in parent directory", async () => {
-    const child = join(tempDir, "subdir", "deep");
-    await mkdir(child, { recursive: true });
-    await mkdir(join(tempDir, ".llm-kb"), { recursive: true });
-    expect(resolveKnowledgeBase(child)).toBe(tempDir);
-  });
-});

package/test/scan.test.ts

@@ -1,65 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { scan, summarize } from "../src/scan.js";
-import { mkdtemp, rm, writeFile, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-
-describe("scan", () => {
-  let tempDir: string;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), "llm-kb-test-"));
-  });
-
-  afterEach(async () => {
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  it("finds supported files", async () => {
-    await writeFile(join(tempDir, "doc.pdf"), "");
-    await writeFile(join(tempDir, "data.xlsx"), "");
-    await writeFile(join(tempDir, "notes.md"), "");
-    await writeFile(join(tempDir, "ignore.exe"), "");
-
-    const files = await scan(tempDir);
-    expect(files.length).toBe(3);
-    expect(files.map((f) => f.ext).sort()).toEqual([".md", ".pdf", ".xlsx"]);
-  });
-
-  it("skips .llm-kb internal directory", async () => {
-    await mkdir(join(tempDir, ".llm-kb"), { recursive: true });
-    await writeFile(join(tempDir, ".llm-kb", "index.md"), "");
-    await writeFile(join(tempDir, "real.pdf"), "");
-
-    const files = await scan(tempDir);
-    expect(files.length).toBe(1);
-    expect(files[0].name).toBe("real.pdf");
-  });
-
-  it("returns empty for empty directory", async () => {
-    const files = await scan(tempDir);
-    expect(files.length).toBe(0);
-  });
-
-  it("scans subdirectories", async () => {
-    await mkdir(join(tempDir, "sub"), { recursive: true });
-    await writeFile(join(tempDir, "sub", "nested.pdf"), "");
-
-    const files = await scan(tempDir);
-    expect(files.length).toBe(1);
-    expect(files[0].path).toContain("sub");
-  });
-});
-
-describe("summarize", () => {
-  it("summarizes file counts by extension", () => {
-    const files = [
-      { name: "a.pdf", path: "a.pdf", ext: ".pdf" },
-      { name: "b.pdf", path: "b.pdf", ext: ".pdf" },
-      { name: "c.xlsx", path: "c.xlsx", ext: ".xlsx" },
-    ];
-    const result = summarize(files);
-    expect(result).toContain("2 PDF");
-    expect(result).toContain("1 XLSX");
-  });
-});