llm-kb 0.4.0 → 0.4.2

package/src/scan.ts

@@ -1,59 +0,0 @@
-import { readdir } from "node:fs/promises";
-import { resolve, extname, relative } from "node:path";
-
-export interface ScannedFile {
-  name: string;
-  path: string;
-  ext: string;
-}
-
-const SUPPORTED_EXTENSIONS = new Set([
-  ".pdf",
-  ".xlsx",
-  ".xls",
-  ".docx",
-  ".pptx",
-  ".jpg",
-  ".jpeg",
-  ".png",
-  ".txt",
-  ".md",
-  ".csv",
-]);
-
-export async function scan(folder: string): Promise<ScannedFile[]> {
-  const root = resolve(folder);
-  const entries = await readdir(root, { recursive: true, withFileTypes: true });
-
-  const files: ScannedFile[] = [];
-
-  for (const entry of entries) {
-    if (!entry.isFile()) continue;
-
-    const fullPath = resolve(entry.parentPath, entry.name);
-    const rel = relative(root, fullPath);
-
-    // Skip .llm-kb internal folder
-    if (rel.startsWith(".llm-kb")) continue;
-
-    const ext = extname(entry.name).toLowerCase();
-    if (!SUPPORTED_EXTENSIONS.has(ext)) continue;
-
-    files.push({ name: entry.name, path: rel, ext });
-  }
-
-  return files;
-}
-
-export function summarize(files: ScannedFile[]): string {
-  const counts = new Map<string, number>();
-  for (const f of files) {
-    counts.set(f.ext, (counts.get(f.ext) || 0) + 1);
-  }
-
-  const parts = Array.from(counts.entries())
-    .sort((a, b) => b[1] - a[1])
-    .map(([ext, count]) => `${count} ${ext.toUpperCase().slice(1)}`);
-
-  return parts.join(", ");
-}

package/src/session-store.ts

@@ -1,22 +0,0 @@
-import { SessionManager } from "@mariozechner/pi-coding-agent";
-import { mkdir } from "node:fs/promises";
-import { join } from "node:path";
-
-/**
- * Continue the most recent session, or create a new one if none exists.
- * Sessions persist in .llm-kb/sessions/ — conversation history survives restarts.
- */
-export async function continueKBSession(kbRoot: string): Promise<SessionManager> {
-  const sessionDir = join(kbRoot, ".llm-kb", "sessions");
-  await mkdir(sessionDir, { recursive: true });
-  return SessionManager.continueRecent(kbRoot, sessionDir);
-}
-
-/**
- * Always create a fresh session (for one-shot `llm-kb query` or indexing).
- */
-export async function createKBSession(kbRoot: string): Promise<SessionManager> {
-  const sessionDir = join(kbRoot, ".llm-kb", "sessions");
-  await mkdir(sessionDir, { recursive: true });
-  return SessionManager.create(kbRoot, sessionDir);
-}

package/src/session-watcher.ts

@@ -1,89 +0,0 @@
-import { watch } from "chokidar";
-import { join, basename } from "node:path";
-import { readdir, readFile, writeFile, mkdir } from "node:fs/promises";
-import { existsSync } from "node:fs";
-import { buildTrace, saveTrace, appendToQueryLog } from "./trace-builder.js";
-import { updateWiki } from "./wiki-updater.js";
-
-const PROCESSED_LOG = ".llm-kb/traces/.processed";
-
-async function loadProcessed(kbRoot: string): Promise<Set<string>> {
-  const path = join(kbRoot, PROCESSED_LOG);
-  if (!existsSync(path)) return new Set();
-  try {
-    const lines = (await readFile(path, "utf-8")).split("\n").filter(Boolean);
-    return new Set(lines);
-  } catch {
-    return new Set();
-  }
-}
-
-async function markProcessed(kbRoot: string, sessionId: string): Promise<void> {
-  const path = join(kbRoot, PROCESSED_LOG);
-  await mkdir(join(kbRoot, ".llm-kb", "traces"), { recursive: true });
-  await writeFile(path, sessionId + "\n", { flag: "a" });
-}
-
-/**
- * Watch .llm-kb/sessions/ for completed session files.
- * Processes them silently — saves traces, updates wiki, logs queries.
- * Persists processed IDs to .llm-kb/traces/.processed to survive restarts.
- */
-export async function startSessionWatcher(kbRoot: string): Promise<void> {
-  const sessionsDir = join(kbRoot, ".llm-kb", "sessions");
-  const sourcesDir  = join(kbRoot, ".llm-kb", "wiki", "sources");
-
-  const processed = await loadProcessed(kbRoot);
-  const timers = new Map<string, ReturnType<typeof setTimeout>>();
-
-  async function processSession(filePath: string): Promise<void> {
-    const sessionId = basename(filePath, ".jsonl").split("_")[1] ?? basename(filePath, ".jsonl");
-    if (processed.has(sessionId)) return;
-
-    try {
-      const trace = await buildTrace(filePath, sourcesDir);
-      if (!trace) return;
-
-      processed.add(trace.sessionId);
-      await markProcessed(kbRoot, trace.sessionId);
-
-      await saveTrace(kbRoot, trace);
-
-      if (trace.mode === "query") {
-        await appendToQueryLog(kbRoot, trace);
-        await updateWiki(kbRoot, trace);
-      }
-    } catch {
-      // Non-fatal — session may still be in progress
-    }
-  }
-
-  function scheduleProcess(filePath: string): void {
-    const existing = timers.get(filePath);
-    if (existing) clearTimeout(existing);
-    const timer = setTimeout(() => {
-      timers.delete(filePath);
-      processSession(filePath);
-    }, 1500);
-    timers.set(filePath, timer);
-  }
-
-  // Catch-up: process existing unprocessed sessions silently
-  if (existsSync(sessionsDir)) {
-    try {
-      const files = (await readdir(sessionsDir)).filter((f) => f.endsWith(".jsonl"));
-      for (const f of files) {
-        await processSession(join(sessionsDir, f));
-      }
-    } catch { /* ignore */ }
-  }
-
-  const watcher = watch(sessionsDir, {
-    ignoreInitial: true,
-    awaitWriteFinish: { stabilityThreshold: 500, pollInterval: 100 },
-    depth: 0,
-  });
-
-  watcher.on("add",    (p) => { if (p.endsWith(".jsonl")) scheduleProcess(p); });
-  watcher.on("change", (p) => { if (p.endsWith(".jsonl")) scheduleProcess(p); });
-}

package/src/trace-builder.ts

@@ -1,168 +0,0 @@
-import { readFile, writeFile, mkdir, readdir } from "node:fs/promises";
-import { existsSync } from "node:fs";
-import { join, basename as pathBasename } from "node:path";
-
-export interface KBTrace {
-  sessionId: string;
-  sessionFile: string;
-  timestamp: string;
-  mode: "query" | "index" | "unknown";
-  question?: string;
-  answer?: string;
-  filesRead: string[];
-  filesAvailable: string[];
-  filesSkipped: string[];
-  model?: string;
-  durationMs?: number;
-}
-
-/** Parse a session JSONL file and build a KBTrace. Returns null if session isn't complete yet. */
-export async function buildTrace(
-  sessionFile: string,
-  sourcesDir: string
-): Promise<KBTrace | null> {
-  const raw = await readFile(sessionFile, "utf-8");
-  const lines = raw.trim().split("\n").filter(Boolean);
-  if (lines.length < 2) return null;
-
-  const entries: any[] = [];
-  let header: any = null;
-  for (const line of lines) {
-    try {
-      const obj = JSON.parse(line);
-      if (obj.type === "session") header = obj;
-      else entries.push(obj);
-    } catch { /* skip malformed lines */ }
-  }
-
-  if (!header) return null;
-
-  const messages = entries.filter((e) => e.type === "message");
-  const lastAssistant = [...messages].reverse().find(
-    (e) => e.message?.role === "assistant" && e.message?.stopReason === "stop"
-  );
-  if (!lastAssistant) return null;
-
-  const modelChange = entries.find((e) => e.type === "model_change");
-  const model = modelChange?.modelId ?? lastAssistant.message?.model ?? undefined;
-
-  const firstUser = messages.find((e) => e.message?.role === "user");
-  const question = extractText(firstUser?.message?.content);
-
-  const sessionInfo = entries.find((e) => e.type === "session_info");
-  const sessionName: string = sessionInfo?.name ?? "";
-  const mode: KBTrace["mode"] = sessionName.startsWith("index:")
-    ? "index"
-    : sessionName.startsWith("query:") || question
-    ? "query"
-    : "unknown";
-
-  const answer = extractText(lastAssistant.message?.content);
-
-  const filesRead: string[] = [];
-  for (const entry of messages) {
-    if (entry.message?.role !== "assistant") continue;
-    for (const block of entry.message?.content ?? []) {
-      if (block.type === "toolCall" && block.name === "read") {
-        const p: string = block.arguments?.path ?? "";
-        if (p && !filesRead.includes(p)) filesRead.push(p);
-      }
-    }
-  }
-
-  let filesAvailable: string[] = [];
-  try {
-    const all = await readdir(sourcesDir);
-    filesAvailable = all.filter((f) => f.endsWith(".md"));
-  } catch { /* sources dir may not exist */ }
-
-  const filesSkipped = filesAvailable.filter(
-    (f) => !filesRead.some((r) => r.endsWith(f))
-  );
-
-  const firstMsg = messages[0];
-  const lastMsg = messages[messages.length - 1];
-  let durationMs: number | undefined;
-  if (firstMsg?.timestamp && lastMsg?.timestamp) {
-    durationMs = new Date(lastMsg.timestamp).getTime() - new Date(firstMsg.timestamp).getTime();
-  }
-
-  return {
-    sessionId: header.id,
-    sessionFile: pathBasename(sessionFile),
-    timestamp: header.timestamp,
-    mode,
-    question: question || undefined,
-    answer: answer || undefined,
-    filesRead,
-    filesAvailable,
-    filesSkipped,
-    model,
-    durationMs,
-  };
-}
-
-/** Write a KBTrace to .llm-kb/traces/<sessionId>.json */
-export async function saveTrace(kbRoot: string, trace: KBTrace): Promise<void> {
-  const tracesDir = join(kbRoot, ".llm-kb", "traces");
-  await mkdir(tracesDir, { recursive: true });
-  const outPath = join(tracesDir, `${trace.sessionId}.json`);
-  await writeFile(outPath, JSON.stringify(trace, null, 2) + "\n", "utf-8");
-}
-
-/** Append a query entry to .llm-kb/wiki/queries.md (newest first) */
-export async function appendToQueryLog(kbRoot: string, trace: KBTrace): Promise<void> {
-  if (trace.mode !== "query" || !trace.question) return;
-
-  const wikiDir = join(kbRoot, ".llm-kb", "wiki");
-  await mkdir(wikiDir, { recursive: true });
-  const logPath = join(wikiDir, "queries.md");
-
-  const date = new Date(trace.timestamp).toISOString().replace("T", " ").slice(0, 19);
-  const durationSec = trace.durationMs ? `${(trace.durationMs / 1000).toFixed(1)}s` : "?";
-  const filesLine = trace.filesRead.length > 0
-    ? trace.filesRead.map((f) => pathBasename(f)).join(", ")
-    : "_none_";
-
-  let header = "";
-  if (!existsSync(logPath)) {
-    header = `# Query Log\n\nAll queries run against this knowledge base.\n\n---\n\n`;
-  }
-
-  const entry = [
-    `## ${trace.question}`,
-    ``,
-    `- **Date:** ${date}`,
-    `- **Model:** ${trace.model ?? "unknown"}`,
-    `- **Duration:** ${durationSec}`,
-    `- **Files read:** ${filesLine}`,
-    trace.filesSkipped.length > 0
-      ? `- **Files skipped:** ${trace.filesSkipped.join(", ")}`
-      : null,
-    ``,
-    trace.answer ? `### Answer\n\n${trace.answer}` : null,
-    ``,
-    `---`,
-    ``,
-  ]
-    .filter((l) => l !== null)
-    .join("\n");
-
-  const existing = existsSync(logPath) ? await readFile(logPath, "utf-8") : "";
-  await writeFile(logPath, header + entry + existing, "utf-8");
-}
-
-// --- helpers ---
-
-function extractText(content: any): string {
-  if (!content) return "";
-  if (typeof content === "string") return content;
-  if (Array.isArray(content)) {
-    return content
-      .filter((b) => b.type === "text")
-      .map((b) => b.text ?? "")
-      .join("")
-      .trim();
-  }
-  return "";
-}

package/src/tui-display.ts

@@ -1,281 +0,0 @@
-import {
-  TUI, Container, Spacer, Text, Markdown, ProcessTerminal,
-  type MarkdownTheme, type Component, Input,
-} from "@mariozechner/pi-tui";
-import chalk from "chalk";
-
-// ── Markdown theme ──────────────────────────────────────────────────────────
-
-function createMarkdownTheme(): MarkdownTheme {
-  return {
-    heading:         (t) => chalk.bold(t),
-    link:            (t) => chalk.cyan(t),
-    linkUrl:         (t) => chalk.dim(t),
-    code:            (t) => chalk.cyan(t),
-    codeBlock:       (t) => chalk.dim(t),
-    codeBlockBorder: (t) => chalk.dim(t),
-    quote:           (t) => chalk.italic(t),
-    quoteBorder:     (t) => chalk.dim(t),
-    hr:              (t) => chalk.dim(t),
-    listBullet:      (t) => chalk.dim(t),
-    bold:            (t) => chalk.bold(t),
-    italic:          (t) => chalk.italic(t),
-    underline:       (t) => chalk.underline(t),
-    strikethrough:   (t) => chalk.strikethrough(t),
-  };
-}
-
-const mdTheme = createMarkdownTheme();
-
-// ── Helper components ───────────────────────────────────────────────────────
-
-function dimText(text: string, px = 1, py = 0): Text {
-  return new Text(chalk.dim(text), px, py);
-}
-
-class HRule implements Component {
-  private colorFn: (s: string) => string;
-  constructor(colorFn?: (s: string) => string) {
-    this.colorFn = colorFn ?? chalk.dim;
-  }
-  invalidate() {}
-  render(width: number): string[] {
-    return [this.colorFn("\u2500".repeat(width))];
-  }
-}
-
-// ── Chat display ────────────────────────────────────────────────────────────
-//
-// All components are appended sequentially to currentResponse in the order
-// events arrive. This naturally handles interleaved thinking/tools/text:
-//
-//   ⟡ model
-//   ▸ Thinking: reasoning...
-//   Let me read the file...              ← 1st Markdown block
-//   ▸ Reading  file.md                   ← tool call (between text blocks)
-//   ──────────────────────
-//   Based on the document...             ← 2nd Markdown block
-//   ▸ Thinking: let me check another...
-//   ▸ Reading  file2.md
-//   ──────────────────────
-//   The final answer is...               ← 3rd Markdown block
-//   ── 12.3s · 2 files read ──────
-
-export class ChatDisplay {
-  private tui: TUI;
-  private terminal: ProcessTerminal;
-  private messageArea: Container;
-  private inputArea: Container;
-  private input: Input;
-
-  // Current response (reset per prompt)
-  private currentResponse: Container | null = null;
-  private currentMd: Markdown | null = null;       // active text block
-  private currentThinking: Text | null = null;      // active thinking block
-  private hadSeparator = false;                     // has a ─── line been drawn?
-
-  private filesReadCount = 0;
-  private shownToolCalls = new Set<string>();
-  private startTime = Date.now();
-
-  onSubmit?: (text: string) => void;
-  onExit?: () => void;
-
-  constructor() {
-    this.terminal = new ProcessTerminal();
-    this.tui = new TUI(this.terminal);
-
-    this.messageArea = new Container();
-    this.tui.addChild(this.messageArea);
-
-    this.inputArea = new Container();
-    this.inputArea.addChild(new HRule((s) => chalk.hex("#c678dd")(s)));
-
-    this.input = new Input();
-    this.input.onSubmit = (text) => {
-      if (text.trim() && this.onSubmit) {
-        this.addUserMessage(text.trim());
-        this.onSubmit(text.trim());
-      }
-      this.input.setValue("");
-    };
-    this.inputArea.addChild(this.input);
-    this.inputArea.addChild(new HRule((s) => chalk.hex("#c678dd")(s)));
-
-    this.tui.addChild(this.inputArea);
-    this.tui.setFocus(this.input);
-  }
-
-  start(): void {
-    this.tui.start();
-
-    this.tui.addInputListener((data) => {
-      if (data === "\x03" || data === "\x04") {
-        this.stop();
-        if (this.onExit) this.onExit();
-        else process.exit(0);
-        return { consume: true };
-      }
-      return undefined;
-    });
-
-    this.tui.requestRender();
-  }
-
-  stop(): void {
-    this.tui.stop();
-  }
-
-  addUserMessage(text: string): void {
-    this.messageArea.addChild(new Spacer(1));
-    this.messageArea.addChild(new Text(chalk.bold(text), 1, 0));
-    this.tui.requestRender();
-  }
-
-  // ── Per-prompt lifecycle (events arrive in any order) ───────────────────
-
-  beginResponse(modelName: string): void {
-    this.filesReadCount = 0;
-    this.shownToolCalls = new Set();
-    this.startTime = Date.now();
-    this.currentMd = null;
-    this.currentThinking = null;
-    this.hadSeparator = false;
-
-    this.currentResponse = new Container();
-    this.currentResponse.addChild(new Spacer(1));
-    this.currentResponse.addChild(dimText(`\u27e1 ${modelName}`));
-    this.messageArea.addChild(this.currentResponse);
-    this.tui.requestRender();
-  }
-
-  /** Start or continue a thinking block. Closes any active text block. */
-  appendThinking(text: string): void {
-    if (!this.currentResponse) return;
-
-    // Close active text block — thinking interrupts it
-    this.currentMd = null;
-
-    if (!this.currentThinking) {
-      this.currentResponse.addChild(new Spacer(1));
-      this.currentResponse.addChild(dimText("\u25b8 Thinking"));
-      this.currentThinking = new Text(chalk.dim(chalk.italic(text)), 2, 0);
-      this.currentResponse.addChild(this.currentThinking);
-    } else {
-      const prev = (this.currentThinking as any).text ?? "";
-      this.currentThinking.setText(
-        chalk.dim(chalk.italic(prev.replace(/\x1b\[[0-9;]*m/g, "") + text))
-      );
-    }
-    this.tui.requestRender();
-  }
-
-  /** End the current thinking block */
-  endThinking(): void {
-    this.currentThinking = null;
-  }
-
-  /** Add a tool call line. Closes active text block. */
-  addToolCall(toolCallId: string, label: string, toolName: string): void {
-    if (!this.currentResponse || this.shownToolCalls.has(toolCallId)) return;
-    this.shownToolCalls.add(toolCallId);
-    if (toolName === "read") this.filesReadCount++;
-
-    // Close active text block — tool call interrupts it
-    this.currentMd = null;
-
-    this.currentResponse.addChild(dimText(`  \u25b8 ${label}`));
-    this.tui.requestRender();
-  }
-
-  /** Show code block (e.g. bash script the agent wrote) */
-  addCodeBlock(code: string): void {
-    if (!this.currentResponse) return;
-    this.currentMd = null;
-    // Use Markdown component to render the code block with proper formatting
-    this.currentResponse.addChild(new Markdown("```javascript\n" + code + "\n```", 2, 0, mdTheme));
-    this.tui.requestRender();
-  }
-
-  /** Show tool execution result (success/error) */
-  addToolResult(toolCallId: string, isError: boolean): void {
-    if (!this.currentResponse) return;
-    if (isError) {
-      this.currentResponse.addChild(new Text(chalk.red("    \u2717 failed"), 0, 0));
-      this.tui.requestRender();
-    }
-  }
-
-  /** Start a NEW text block with a separator (if not the first). */
-  beginAnswer(): void {
-    if (!this.currentResponse) return;
-
-    // Close previous thinking
-    this.currentThinking = null;
-
-    // Add separator line before answer text
-    this.currentResponse.addChild(new Spacer(1));
-    this.currentResponse.addChild(new HRule());
-    this.currentResponse.addChild(new Spacer(1));
-    this.hadSeparator = true;
-
-    // Create a new Markdown block for this text segment
-    this.currentMd = new Markdown("", 1, 0, mdTheme);
-    this.currentResponse.addChild(this.currentMd);
-    this.tui.requestRender();
-  }
-
-  /** Append text to the active Markdown block (creates one if needed) */
-  appendAnswer(text: string): void {
-    if (!this.currentResponse) return;
-
-    if (!this.currentMd) {
-      // No active text block — create one (with separator if this is the first)
-      if (!this.hadSeparator) {
-        this.currentResponse.addChild(new Spacer(1));
-        this.currentResponse.addChild(new HRule());
-        this.currentResponse.addChild(new Spacer(1));
-        this.hadSeparator = true;
-      }
-      this.currentMd = new Markdown("", 1, 0, mdTheme);
-      this.currentResponse.addChild(this.currentMd);
-    }
-
-    const prev = (this.currentMd as any).text ?? "";
-    this.currentMd.setText(prev + text);
-    this.tui.requestRender();
-  }
-
-  showCompletion(): void {
-    if (!this.currentResponse) return;
-    const elapsed = ((Date.now() - this.startTime) / 1000).toFixed(1);
-    const source = this.filesReadCount > 0
-      ? `${this.filesReadCount} file${this.filesReadCount !== 1 ? "s" : ""} read`
-      : "wiki";
-    const stats = `\u2500\u2500 ${elapsed}s \u00b7 ${source} `;
-
-    const completion: Component = {
-      invalidate() {},
-      render(width: number) {
-        const pad = Math.max(0, width - stats.length);
-        return [chalk.dim(stats + "\u2500".repeat(pad))];
-      },
-    };
-
-    this.currentResponse.addChild(new Spacer(1));
-    this.currentResponse.addChild(completion);
-    this.currentResponse = null;
-    this.currentMd = null;
-    this.currentThinking = null;
-    this.tui.requestRender();
-  }
-
-  enableInput(): void {
-    this.tui.setFocus(this.input);
-    this.tui.requestRender();
-  }
-
-  disableInput(): void {
-    this.tui.setFocus(null);
-  }
-}

package/src/utils.ts

@@ -1,17 +0,0 @@
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-/**
- * Find the node_modules directory for llm-kb's bundled libraries.
- * Walks up from the current file to locate the nearest node_modules.
- */
-export function getNodeModulesPath(): string {
-  let dir = __dirname;
-  for (let i = 0; i < 5; i++) {
-    const candidate = join(dir, "node_modules");
-    try { return candidate; } catch { dir = dirname(dir); }
-  }
-  return join(process.cwd(), "node_modules");
-}