llm-kb 0.2.0 → 0.4.1

package/bin/chunk-MYQ36JJB.js

@@ -1,118 +0,0 @@
-// src/indexer.ts
-import {
-  createAgentSession,
-  createBashTool,
-  createReadTool,
-  createWriteTool,
-  DefaultResourceLoader,
-  SessionManager,
-  SettingsManager
-} from "@mariozechner/pi-coding-agent";
-import { readdir, readFile } from "fs/promises";
-import { join, dirname } from "path";
-import { fileURLToPath } from "url";
-var __filename = fileURLToPath(import.meta.url);
-var __dirname = dirname(__filename);
-function getNodeModulesPath() {
-  let dir = __dirname;
-  for (let i = 0; i < 5; i++) {
-    const candidate = join(dir, "node_modules");
-    try {
-      return candidate;
-    } catch {
-      dir = dirname(dir);
-    }
-  }
-  return join(process.cwd(), "node_modules");
-}
-function buildAgentsContent(sourcesDir, files) {
-  const sourceList = files.filter((f) => f.endsWith(".md")).map((f) => `  - ${f}`).join("\n");
-  return `# llm-kb Knowledge Base
-
-## How to access documents
-
-### PDFs (pre-parsed)
-PDFs have been parsed to markdown with bounding boxes.
-Read the markdown versions in \`.llm-kb/wiki/sources/\` instead of the raw PDFs.
-
-Available parsed sources:
-${sourceList}
-
-### Other file types (Excel, Word, PowerPoint, CSV, images)
-You have bash and read tools. These libraries are pre-installed and available:
-- **exceljs** \u2014 for .xlsx/.xls files
-- **mammoth** \u2014 for .docx files  
-- **officeparser** \u2014 for .pptx files
-- **csv-parse** \u2014 built into Node.js, use fs + split for .csv
-
-Write a quick Node.js script to extract content when needed.
-
-## Index file
-Write the index to \`.llm-kb/wiki/index.md\`.
-
-The index should be a markdown file with:
-1. A title and last-updated timestamp
-2. A summary table with columns: Source, Type, Pages/Size, Summary, Key Topics
-3. Each source gets a one-line summary (read the first ~500 chars of each file to generate it)
-4. Total word count across all sources
-`;
-}
-async function buildIndex(folder, sourcesDir, onOutput) {
-  const files = await readdir(sourcesDir);
-  const mdFiles = files.filter((f) => f.endsWith(".md"));
-  if (mdFiles.length === 0) {
-    throw new Error("No source files found to index");
-  }
-  const agentsContent = buildAgentsContent(sourcesDir, files);
-  const nodeModulesPath = getNodeModulesPath();
-  process.env.NODE_PATH = nodeModulesPath;
-  const loader = new DefaultResourceLoader({
-    cwd: folder,
-    agentsFilesOverride: (current) => ({
-      agentsFiles: [
-        ...current.agentsFiles,
-        { path: ".llm-kb/AGENTS.md", content: agentsContent }
-      ]
-    })
-  });
-  await loader.reload();
-  const { session } = await createAgentSession({
-    cwd: folder,
-    resourceLoader: loader,
-    tools: [
-      createReadTool(folder),
-      createBashTool(folder),
-      createWriteTool(folder)
-    ],
-    sessionManager: SessionManager.inMemory(),
-    settingsManager: SettingsManager.inMemory({
-      compaction: { enabled: false }
-    })
-  });
-  if (onOutput) {
-    session.subscribe((event) => {
-      if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") {
-        onOutput(event.assistantMessageEvent.delta);
-      }
-    });
-  }
-  const prompt = `Read each file in .llm-kb/wiki/sources/ (one at a time, just the first 500 characters of each).
-Then write .llm-kb/wiki/index.md with a summary table of all sources.
-
-Include: Source filename, Type (PDF/Excel/Word/etc), Pages (from the JSON if available), a one-line summary, and key topics.
-Add a total word count estimate at the bottom.`;
-  await session.prompt(prompt);
-  const indexPath = join(sourcesDir, "..", "index.md");
-  try {
-    const content = await readFile(indexPath, "utf-8");
-    session.dispose();
-    return content;
-  } catch {
-    session.dispose();
-    throw new Error("Agent did not create index.md");
-  }
-}
-
-export {
-  buildIndex
-};

package/bin/indexer-LSYSZXZX.js

@@ -1,6 +0,0 @@
-import {
-  buildIndex
-} from "./chunk-MYQ36JJB.js";
-export {
-  buildIndex
-};

package/plan.md

@@ -1,55 +0,0 @@
-# llm-kb — Phase 1 Build Plan
-
-> Emergent design. Each slice is a thin vertical slice that works end-to-end, is demoable, and informs the next step. Decisions are made at the last responsible moment.
-
-## Key Learnings
-- **PDF is the only adapter we build.** Everything else (Excel, Word, PPT, CSV, images) handled dynamically by Pi SDK agent at query time.
-- **`@llamaindex/liteparse`** proven (from parser-study). Extracts text + bounding boxes locally.
-- **Two-output pattern**: `.md` (spatial text) + `.json` (bounding boxes for citations).
-- **OCR off by default.** Most PDFs have native text. Enable via `OCR_SERVER_URL` or `OCR_ENABLED=true`.
-- **Pi SDK `createAgentSession()`** with defaults — no auth/model config needed. Uses Pi's existing auth.
-- **AGENTS.md injected via `agentsFilesOverride`** — user's folder stays clean.
-- **NODE_PATH** set so agent's bash scripts can use bundled libraries (exceljs, mammoth, officeparser).
-- **Config file skipped** — nothing reads it yet. Add when Phase 2/3 needs it.
-
----
-
-## Slice 1: "Hello World" CLI ✅
-Commander CLI with `run <folder>`. Scans folder, lists files by extension.
-
-## Slice 2: PDF → markdown + bounding boxes ✅
-LiteParse parses PDFs → `.md` + `.json` in `.llm-kb/wiki/sources/`. Tested on 9 real PDFs (1000+ pages).
-
-## Slice 3: Scanned PDF handling (OCR) ✅
-LiteParse has Tesseract.js built-in. `ocrEnabled` + `ocrServerUrl` config. OCR off by default. Azure OCR bridge tested on 16 legal PDFs (3000+ pages).
-
-## Slice 4: Progress + error handling ✅
-Inline progress. Stderr suppression. Corrupt file skip + warning. Mtime check — re-runs instant.
-
-## Slice 5: Indexer (Pi SDK) ✅
-`createAgentSession` with cwd = user's folder. AGENTS.md injected. Agent reads sources, writes `index.md` with summary table.
-
-## Slice 6: File watcher ✅
-chokidar watches folder. New/changed PDFs → parse → re-index. 2s debounce for batch drops.
-
-## Slice 7: Config + polish → Skipped
-Config file has no readers yet. Deferred to Phase 2/3. README updated instead.
-
----
-
-## Phase 1 Complete ✅
-
-**What ships:**
-- `llm-kb run ./folder` — scan, parse PDFs, build index, watch for new files
-- Pre-bundled libraries for agent to handle Excel, Word, PowerPoint at query time
-- OCR via env var (local Tesseract or remote Azure bridge)
-- Auth via Pi SDK (zero config)
-
-**Phase 2 complete ✅:**
-- `llm-kb query "question"` — auto-detects KB, streams cited answers
-- `--save` flag — research mode, saves to `outputs/`, re-indexes
-- Query mode is read-only (read tool only). Research mode adds bash + write.
-
-**Deferred to Phase 4:**
-- Trace logging (JSON per query: question, filesRead, citations, tokens, duration)
-- Needed for eval, but no eval system yet to consume traces

package/src/cli.ts

@@ -1,132 +0,0 @@
-#!/usr/bin/env node
-
-import { Command } from "commander";
-import { scan, summarize } from "./scan.js";
-import { parsePDF } from "./pdf.js";
-import { buildIndex } from "./indexer.js";
-import { startWatcher } from "./watcher.js";
-import { query } from "./query.js";
-import { resolveKnowledgeBase } from "./resolve-kb.js";
-import { existsSync } from "node:fs";
-import { mkdir } from "node:fs/promises";
-import { resolve, join } from "node:path";
-import chalk from "chalk";
-
-const program = new Command();
-
-program
-  .name("llm-kb")
-  .description("Drop files into a folder. Get a knowledge base you can query.")
-  .version("0.2.0");
-
-program
-  .command("run")
-  .description("Scan, parse, index, and watch a folder")
-  .argument("<folder>", "Path to your documents folder")
-  .action(async (folder: string) => {
-    console.log(`\n${chalk.bold("llm-kb")} v0.2.0\n`);
-
-    if (!existsSync(folder)) {
-      console.error(chalk.red(`Error: Folder not found: ${folder}`));
-      process.exit(1);
-    }
-
-    console.log(`Scanning ${folder}...`);
-
-    const files = await scan(folder);
-
-    if (files.length === 0) {
-      console.log(chalk.yellow("  No supported files found."));
-      return;
-    }
-
-    const pdfs = files.filter((f) => f.ext === ".pdf");
-    console.log(`  Found ${chalk.bold(files.length.toString())} files (${summarize(files)})`);
-    if (pdfs.length === 0) return;
-
-    // Set up .llm-kb folder structure
-    const root = resolve(folder);
-    const sourcesDir = join(root, ".llm-kb", "wiki", "sources");
-    await mkdir(sourcesDir, { recursive: true });
-
-    // Parse PDFs with inline progress
-    let parsed = 0;
-    let skipped = 0;
-    let failed = 0;
-    const errors: { name: string; message: string }[] = [];
-
-    for (let i = 0; i < pdfs.length; i++) {
-      const pdf = pdfs[i];
-      const fullPath = join(root, pdf.path);
-
-      // Inline progress — overwrite same line
-      const progress = `  Parsing... ${i + 1}/${pdfs.length} — ${pdf.name}`;
-      process.stdout.write(`\r${progress.padEnd(80)}`);
-
-      try {
-        const result = await parsePDF(fullPath, sourcesDir);
-        if (result.skipped) {
-          skipped++;
-        } else {
-          parsed++;
-        }
-      } catch (err: any) {
-        failed++;
-        errors.push({ name: pdf.name, message: err.message });
-      }
-    }
-
-    // Clear progress line
-    process.stdout.write(`\r${"".padEnd(80)}\r`);
-
-    // Summary
-    const parts: string[] = [];
-    if (parsed > 0) parts.push(chalk.green(`${parsed} parsed`));
-    if (skipped > 0) parts.push(chalk.dim(`${skipped} skipped (up to date)`));
-    if (failed > 0) parts.push(chalk.red(`${failed} failed`));
-    console.log(`  ${parts.join(", ")}`);
-
-    // Show errors
-    for (const err of errors) {
-      console.log(chalk.red(`    ✗ ${err.name} — ${err.message}`));
-    }
-
-    // Build index
-    console.log(`\n  Building index...`);
-    try {
-      await buildIndex(root, sourcesDir);
-      console.log(chalk.green(`  Index built: .llm-kb/wiki/index.md`));
-    } catch (err: any) {
-      console.error(chalk.red(`  Index failed: ${err.message}`));
-    }
-
-    console.log(`\n  ${chalk.dim("Output:")} ${sourcesDir}`);
-
-    // Start watching for new files
-    console.log(chalk.dim(`\n  Watching for new files... (Ctrl+C to stop)`));
-    startWatcher({ folder: root, sourcesDir });
-  });
-
-program
-  .command("query")
-  .description("Ask a question across your knowledge base")
-  .argument("<question>", "Your question")
-  .option("--folder <path>", "Path to document folder (auto-detects if omitted)")
-  .option("--save", "Save the answer to wiki/outputs/ (research mode)")
-  .action(async (question: string, options: { folder?: string; save?: boolean }) => {
-    const root = resolveKnowledgeBase(options.folder || process.cwd());
-
-    if (!root) {
-      console.error(chalk.red("No knowledge base found. Run 'llm-kb run <folder>' first."));
-      process.exit(1);
-    }
-
-    try {
-      await query(root, question, { save: options.save });
-    } catch (err: any) {
-      console.error(chalk.red(err.message));
-      process.exit(1);
-    }
-  });
-
-program.parse();

package/src/indexer.ts

@@ -1,148 +0,0 @@
-import {
-  createAgentSession,
-  createBashTool,
-  createReadTool,
-  createWriteTool,
-  DefaultResourceLoader,
-  SessionManager,
-  SettingsManager,
-} from "@mariozechner/pi-coding-agent";
-import { readdir, readFile } from "node:fs/promises";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-
-/**
- * Find the node_modules directory for llm-kb's bundled libraries.
- * When running from bin/cli.js, node_modules is at ../node_modules.
- */
-function getNodeModulesPath(): string {
-  // Walk up from this file to find node_modules
-  let dir = __dirname;
-  for (let i = 0; i < 5; i++) {
-    const candidate = join(dir, "node_modules");
-    try {
-      return candidate;
-    } catch {
-      dir = dirname(dir);
-    }
-  }
-  return join(process.cwd(), "node_modules");
-}
-
-function buildAgentsContent(sourcesDir: string, files: string[]): string {
-  const sourceList = files
-    .filter((f) => f.endsWith(".md"))
-    .map((f) => `  - ${f}`)
-    .join("\n");
-
-  return `# llm-kb Knowledge Base
-
-## How to access documents
-
-### PDFs (pre-parsed)
-PDFs have been parsed to markdown with bounding boxes.
-Read the markdown versions in \`.llm-kb/wiki/sources/\` instead of the raw PDFs.
-
-Available parsed sources:
-${sourceList}
-
-### Other file types (Excel, Word, PowerPoint, CSV, images)
-You have bash and read tools. These libraries are pre-installed and available:
-- **exceljs** — for .xlsx/.xls files
-- **mammoth** — for .docx files  
-- **officeparser** — for .pptx files
-- **csv-parse** — built into Node.js, use fs + split for .csv
-
-Write a quick Node.js script to extract content when needed.
-
-## Index file
-Write the index to \`.llm-kb/wiki/index.md\`.
-
-The index should be a markdown file with:
-1. A title and last-updated timestamp
-2. A summary table with columns: Source, Type, Pages/Size, Summary, Key Topics
-3. Each source gets a one-line summary (read the first ~500 chars of each file to generate it)
-4. Total word count across all sources
-`;
-}
-
-export async function buildIndex(
-  folder: string,
-  sourcesDir: string,
-  onOutput?: (text: string) => void
-): Promise<string> {
-  // List source files
-  const files = await readdir(sourcesDir);
-  const mdFiles = files.filter((f) => f.endsWith(".md"));
-
-  if (mdFiles.length === 0) {
-    throw new Error("No source files found to index");
-  }
-
-  // Build AGENTS.md content
-  const agentsContent = buildAgentsContent(sourcesDir, files);
-
-  // Set NODE_PATH so agent's bash scripts can use bundled libraries
-  const nodeModulesPath = getNodeModulesPath();
-  process.env.NODE_PATH = nodeModulesPath;
-
-  const loader = new DefaultResourceLoader({
-    cwd: folder,
-    agentsFilesOverride: (current) => ({
-      agentsFiles: [
-        ...current.agentsFiles,
-        { path: ".llm-kb/AGENTS.md", content: agentsContent },
-      ],
-    }),
-  });
-  await loader.reload();
-
-  const { session } = await createAgentSession({
-    cwd: folder,
-    resourceLoader: loader,
-    tools: [
-      createReadTool(folder),
-      createBashTool(folder),
-      createWriteTool(folder),
-    ],
-    sessionManager: SessionManager.inMemory(),
-    settingsManager: SettingsManager.inMemory({
-      compaction: { enabled: false },
-    }),
-  });
-
-  // Subscribe to streaming output
-  if (onOutput) {
-    session.subscribe((event) => {
-      if (
-        event.type === "message_update" &&
-        event.assistantMessageEvent.type === "text_delta"
-      ) {
-        onOutput(event.assistantMessageEvent.delta);
-      }
-    });
-  }
-
-  // Build the prompt
-  const prompt = `Read each file in .llm-kb/wiki/sources/ (one at a time, just the first 500 characters of each).
-Then write .llm-kb/wiki/index.md with a summary table of all sources.
-
-Include: Source filename, Type (PDF/Excel/Word/etc), Pages (from the JSON if available), a one-line summary, and key topics.
-Add a total word count estimate at the bottom.`;
-
-  await session.prompt(prompt);
-
-  // Read the generated index
-  const indexPath = join(sourcesDir, "..", "index.md");
-  try {
-    const content = await readFile(indexPath, "utf-8");
-    session.dispose();
-    return content;
-  } catch {
-    session.dispose();
-    throw new Error("Agent did not create index.md");
-  }
-}

package/src/pdf.ts

@@ -1,119 +0,0 @@
-import { LiteParse } from "@llamaindex/liteparse";
-import { writeFile, mkdir, stat } from "node:fs/promises";
-import { join, basename } from "node:path";
-import { cpus } from "node:os";
-
-export interface ParsedPDF {
-  name: string;
-  mdPath: string;
-  jsonPath: string;
-  totalPages: number;
-  textLength: number;
-  skipped: boolean;
-}
-
-/**
- * Check if source PDF is newer than the parsed output.
- * Returns true if we can skip parsing.
- */
-async function isUpToDate(
-  pdfPath: string,
-  mdPath: string,
-  jsonPath: string
-): Promise<boolean> {
-  try {
-    const [pdfStat, mdStat, jsonStat] = await Promise.all([
-      stat(pdfPath),
-      stat(mdPath),
-      stat(jsonPath),
-    ]);
-    return pdfStat.mtimeMs <= mdStat.mtimeMs && pdfStat.mtimeMs <= jsonStat.mtimeMs;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Suppress stderr temporarily to hide noisy library warnings.
- */
-function suppressStderr(): () => void {
-  const originalWrite = process.stderr.write.bind(process.stderr);
-  process.stderr.write = (() => true) as any;
-  return () => {
-    process.stderr.write = originalWrite;
-  };
-}
-
-export async function parsePDF(
-  pdfPath: string,
-  outputDir: string
-): Promise<ParsedPDF> {
-  const name = basename(pdfPath, ".pdf");
-  await mkdir(outputDir, { recursive: true });
-
-  const mdPath = join(outputDir, `${name}.md`);
-  const jsonPath = join(outputDir, `${name}.json`);
-
-  // Skip if already parsed and source hasn't changed
-  if (await isUpToDate(pdfPath, mdPath, jsonPath)) {
-    return { name, mdPath, jsonPath, totalPages: 0, textLength: 0, skipped: true };
-  }
-
-  const ocrServerUrl = process.env.OCR_SERVER_URL;
-  const ocrEnabled = ocrServerUrl ? true : process.env.OCR_ENABLED === "true";
-
-  const parser = new LiteParse({
-    ocrEnabled,
-    outputFormat: "json",
-    numWorkers: cpus().length,
-    ...(ocrServerUrl ? { ocrServerUrl } : {}),
-  });
-
-  // Suppress noisy Tesseract/PDF.js warnings during parse
-  const restore = suppressStderr();
-  let result;
-  try {
-    result = await parser.parse(pdfPath, true);
-  } finally {
-    restore();
-  }
-
-  // Build markdown — spatial text per page
-  const markdown = result.pages
-    .map((p: any) => `# Page ${p.pageNum}\n\n${p.text}`)
-    .join("\n\n---\n\n");
-
-  // Build bounding box JSON
-  const bboxData = {
-    source: basename(pdfPath),
-    totalPages: result.pages.length,
-    pages: result.pages.map((p: any) => ({
-      page: p.pageNum,
-      width: p.width,
-      height: p.height,
-      textItems: p.textItems.map((item: any) => ({
-        text: (item.str ?? item.text ?? "").trim(),
-        x: Math.round(item.x * 100) / 100,
-        y: Math.round(item.y * 100) / 100,
-        width: Math.round((item.width ?? item.w ?? 0) * 100) / 100,
-        height: Math.round((item.height ?? item.h ?? 0) * 100) / 100,
-        fontName: item.fontName,
-        fontSize: item.fontSize
-          ? Math.round(item.fontSize * 100) / 100
-          : undefined,
-      })),
-    })),
-  };
-
-  await writeFile(mdPath, markdown);
-  await writeFile(jsonPath, JSON.stringify(bboxData, null, 2));
-
-  return {
-    name,
-    mdPath,
-    jsonPath,
-    totalPages: result.pages.length,
-    textLength: markdown.length,
-    skipped: false,
-  };
-}