llm-kb 0.2.0 → 0.4.0

package/src/indexer.ts

@@ -6,31 +6,13 @@ import {
   DefaultResourceLoader,
   SessionManager,
   SettingsManager,
+  AuthStorage,
 } from "@mariozechner/pi-coding-agent";
+import { getModels } from "@mariozechner/pi-ai";
 import { readdir, readFile } from "node:fs/promises";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-
-/**
- * Find the node_modules directory for llm-kb's bundled libraries.
- * When running from bin/cli.js, node_modules is at ../node_modules.
- */
-function getNodeModulesPath(): string {
-  // Walk up from this file to find node_modules
-  let dir = __dirname;
-  for (let i = 0; i < 5; i++) {
-    const candidate = join(dir, "node_modules");
-    try {
-      return candidate;
-    } catch {
-      dir = dirname(dir);
-    }
-  }
-  return join(process.cwd(), "node_modules");
-}
+import { createKBSession } from "./session-store.js";
+import { getNodeModulesPath } from "./utils.js";
+import { join } from "node:path";
 
 function buildAgentsContent(sourcesDir: string, files: string[]): string {
   const sourceList = files
@@ -49,14 +31,25 @@ Read the markdown versions in \`.llm-kb/wiki/sources/\` instead of the raw PDFs.
 Available parsed sources:
 ${sourceList}
 
-### Other file types (Excel, Word, PowerPoint, CSV, images)
-You have bash and read tools. These libraries are pre-installed and available:
-- **exceljs** — for .xlsx/.xls files
-- **mammoth** — for .docx files  
-- **officeparser** — for .pptx files
-- **csv-parse** — built into Node.js, use fs + split for .csv
+### Other file types (Excel, Word, PowerPoint)
+You have bash and read tools. Use bash to run Node.js scripts.
+Libraries are pre-installed via require().
+
+For .docx (structured XML — ZIP containing word/document.xml):
+  const AdmZip = require('adm-zip');
+  const zip = new AdmZip('file.docx');
+  const xml = zip.readAsText('word/document.xml');
+  // Parse XML to extract headings and first paragraphs for summary
 
-Write a quick Node.js script to extract content when needed.
+For .xlsx use exceljs:
+  const ExcelJS = require('exceljs');
+  const wb = new ExcelJS.Workbook();
+  await wb.xlsx.readFile('file.xlsx');
+  const sheet = wb.getWorksheet(1);
+
+For .pptx use officeparser:
+  const officeparser = require('officeparser');
+  const text = await officeparser.parseOfficeAsync('file.pptx');
 
 ## Index file
 Write the index to \`.llm-kb/wiki/index.md\`.
@@ -72,7 +65,9 @@ The index should be a markdown file with:
 export async function buildIndex(
   folder: string,
   sourcesDir: string,
-  onOutput?: (text: string) => void
+  onOutput?: (text: string) => void,
+  authStorage?: AuthStorage,
+  modelId?: string
 ): Promise<string> {
   // List source files
   const files = await readdir(sourcesDir);
@@ -100,6 +95,10 @@ export async function buildIndex(
   });
   await loader.reload();
 
+  const model = modelId
+    ? getModels("anthropic").find((m) => m.id === modelId)
+    : undefined;
+
   const { session } = await createAgentSession({
     cwd: folder,
     resourceLoader: loader,
@@ -108,10 +107,12 @@ export async function buildIndex(
       createBashTool(folder),
       createWriteTool(folder),
     ],
-    sessionManager: SessionManager.inMemory(),
+    sessionManager: await createKBSession(folder),
     settingsManager: SettingsManager.inMemory({
       compaction: { enabled: false },
     }),
+    ...(authStorage ? { authStorage } : {}),
+    ...(model ? { model } : {}),
   });
 
   // Subscribe to streaming output
@@ -126,6 +127,9 @@ export async function buildIndex(
     });
   }
 
+  // Tag the session so the session-watcher can identify it as an index run
+  session.setSessionName(`index: ${new Date().toISOString()}`);
+
   // Build the prompt
   const prompt = `Read each file in .llm-kb/wiki/sources/ (one at a time, just the first 500 characters of each).
 Then write .llm-kb/wiki/index.md with a summary table of all sources.

package/src/md-stream.ts

@@ -0,0 +1,133 @@
+import chalk from "chalk";
+
+/**
+ * Streaming markdown renderer for terminal output.
+ *
+ * Processes text_delta chunks and applies ANSI styling as patterns complete.
+ * Handles: **bold**, *italic*, `code`, ## headers, --- hr, • bullets,
+ * > blockquotes, [links](url), ~~strikethrough~~, | tables.
+ */
+export class MarkdownStream {
+  private buffer = "";
+  private isTTY: boolean;
+
+  constructor(isTTY = false) {
+    this.isTTY = isTTY;
+  }
+
+  /** Feed a text_delta chunk. Returns styled string ready for stdout. */
+  push(chunk: string): string {
+    if (!this.isTTY) return chunk;
+
+    this.buffer += chunk;
+    return this.drain(false);
+  }
+
+  /** Flush remaining buffer (call on text_end). */
+  end(): string {
+    if (!this.isTTY) return "";
+    const out = this.drain(true);
+    this.buffer = "";
+    return out;
+  }
+
+  private drain(final: boolean): string {
+    let out = "";
+
+    while (true) {
+      const nlIdx = this.buffer.indexOf("\n");
+
+      if (nlIdx === -1) {
+        if (final && this.buffer.length > 0) {
+          // Final flush — render whatever's left
+          out += this.renderLine(this.buffer);
+          this.buffer = "";
+        }
+        // else: wait for more data (incomplete line)
+        break;
+      }
+
+      // Complete line found — render it
+      const line = this.buffer.slice(0, nlIdx);
+      this.buffer = this.buffer.slice(nlIdx + 1);
+      out += this.renderLine(line) + "\n";
+    }
+
+    return out;
+  }
+
+  /** Render a single complete line with block + inline styling. */
+  private renderLine(line: string): string {
+    const trimmed = line.trimStart();
+
+    // Horizontal rule
+    if (/^-{3,}\s*$/.test(trimmed) || /^\*{3,}\s*$/.test(trimmed)) {
+      const cols = process.stdout.columns || 80;
+      return chalk.dim("\u2500".repeat(Math.min(cols, 60)));
+    }
+
+    // Headers — strip # prefix, render bold
+    const headerMatch = trimmed.match(/^(#{1,6})\s+(.*)$/);
+    if (headerMatch) {
+      const text = this.inline(headerMatch[2]);
+      return "\n" + chalk.bold(text);
+    }
+
+    // Bullet points
+    const bulletMatch = trimmed.match(/^[-*+]\s+(.*)$/);
+    if (bulletMatch) {
+      const indent = line.length - trimmed.length;
+      return " ".repeat(indent) + chalk.dim("\u2022") + " " + this.inline(bulletMatch[1]);
+    }
+
+    // Numbered lists
+    const numMatch = trimmed.match(/^(\d+)[.)]\s+(.*)$/);
+    if (numMatch) {
+      const indent = line.length - trimmed.length;
+      return " ".repeat(indent) + chalk.dim(numMatch[1] + ".") + " " + this.inline(numMatch[2]);
+    }
+
+    // Table separator row
+    if (/^\|[\s\-:|]+\|$/.test(trimmed)) {
+      return chalk.dim(trimmed);
+    }
+
+    // Table data row
+    if (trimmed.startsWith("|") && trimmed.endsWith("|")) {
+      return this.inline(line);
+    }
+
+    // Block quotes — support nested > > and inline formatting
+    if (trimmed.startsWith(">")) {
+      const content = trimmed.replace(/^>+\s*/, "");
+      return chalk.dim("\u2502 ") + chalk.italic(this.inline(content));
+    }
+
+    return this.inline(line);
+  }
+
+  /** Apply inline markdown styling to text. */
+  private inline(text: string): string {
+    // Code spans (before bold/italic to avoid conflicts inside backticks)
+    text = text.replace(/`([^`]+)`/g, (_, c) => chalk.cyan(c));
+
+    // Bold + italic
+    text = text.replace(/\*\*\*(.+?)\*\*\*/g, (_, t) => chalk.bold.italic(t));
+
+    // Bold
+    text = text.replace(/\*\*(.+?)\*\*/g, (_, t) => chalk.bold(t));
+
+    // Italic (single * not adjacent to another *)
+    text = text.replace(/(?<!\*)\*(.+?)\*(?!\*)/g, (_, t) => chalk.italic(t));
+
+    // Strikethrough
+    text = text.replace(/~~(.+?)~~/g, (_, t) => chalk.strikethrough(t));
+
+    // Links
+    text = text.replace(/\[([^\]]+)\]\(([^)]+)\)/g, (_, label, url) =>
+      `${label} ${chalk.dim(`(${url})`)}`
+    );
+
+    return text;
+  }
+}