llm-kb 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "llm-kb",
3
- "version": "0.2.0",
3
+ "version": "0.4.0",
4
4
  "description": "LLM-powered knowledge base. Drop documents, build a wiki, ask questions. Inspired by Karpathy.",
5
5
  "bin": {
6
6
  "llm-kb": "./bin/cli.js"
@@ -8,7 +8,9 @@
8
8
  "type": "module",
9
9
  "scripts": {
10
10
  "build": "tsup src/cli.ts --format esm --out-dir bin --clean",
11
- "dev": "tsup src/cli.ts --format esm --out-dir bin --watch"
11
+ "dev": "tsup src/cli.ts --format esm --out-dir bin --watch",
12
+ "test": "vitest run",
13
+ "test:watch": "vitest"
12
14
  },
13
15
  "keywords": [
14
16
  "llm",
@@ -28,16 +30,19 @@
28
30
  "dependencies": {
29
31
  "@llamaindex/liteparse": "^1.4.4",
30
32
  "@mariozechner/pi-coding-agent": "^0.65.0",
33
+ "adm-zip": "^0.5.17",
31
34
  "chalk": "^5.6.2",
32
35
  "chokidar": "^5.0.0",
33
36
  "commander": "^13.1.0",
34
37
  "exceljs": "^4.4.0",
35
- "mammoth": "^1.12.0",
36
- "officeparser": "^6.0.7",
37
- "ora": "^9.3.0"
38
+ "marked": "^15.0.12",
39
+ "marked-terminal": "^7.3.0",
40
+ "officeparser": "^6.0.7"
38
41
  },
39
42
  "devDependencies": {
43
+ "@types/marked-terminal": "^6.1.1",
40
44
  "tsup": "^8.4.0",
41
- "typescript": "^5.8.3"
45
+ "typescript": "^5.8.3",
46
+ "vitest": "^4.1.2"
42
47
  }
43
48
  }
package/plan.md CHANGED
@@ -50,6 +50,251 @@ Config file has no readers yet. Deferred to Phase 2/3. README updated instead.
50
50
  - `--save` flag — research mode, saves to `outputs/`, re-indexes
51
51
  - Query mode is read-only (read tool only). Research mode adds bash + write.
52
52
 
53
- **Deferred to Phase 4:**
54
- - Trace logging (JSON per query: question, filesRead, citations, tokens, duration)
55
- - Needed for eval, but no eval system yet to consume traces
53
+ ---
54
+
55
+ ## Phase 3: Auth Fix + Eval Loop + LLM Config
56
+
57
+ > Full spec: `PHASE3_SPEC.md`
58
+ > **Trigger:** 182 LinkedIn saves — people coming back to try `npx llm-kb run` this week.
59
+
60
+ ### Why now
61
+ - **Auth fix is urgent.** Users hit a wall if Pi SDK isn't installed. `ANTHROPIC_API_KEY` must work as fallback.
62
+ - **Eval loop is the differentiator.** Nobody else building llm-wiki scripts has traces + citation checking.
63
+ - **Model config cuts cost.** Haiku for indexing (10x cheaper than Sonnet). Users shouldn't pay Sonnet prices for one-line summaries.
64
+
65
+ ### Build order
66
+
67
+ ## Slice 1: Auth fix 🔴 DO FIRST
68
+
69
+ Check auth before creating any session. Support two paths:
70
+
71
+ ```
72
+ npx llm-kb run ./docs
73
+ ├─ ~/.pi/agent/auth.json exists? → Pi SDK auth. Done.
74
+ ├─ ANTHROPIC_API_KEY set? → Configure Pi SDK programmatically. Done.
75
+ └─ Neither? → Clear error:
76
+ No LLM authentication found.
77
+ Option 1: npm install -g @mariozechner/pi-coding-agent && pi
78
+ Option 2: export ANTHROPIC_API_KEY=sk-ant-...
79
+ ```
80
+
81
+ Definition of done:
82
+ - [ ] `ANTHROPIC_API_KEY=sk-... npx llm-kb run ./docs` works without Pi installed
83
+ - [ ] Pi SDK auth works as before (no regression)
84
+ - [ ] Clear error when neither is available
85
+ - [ ] README updated with both auth options
86
+
87
+ ## Slice 2: LLM config
88
+
89
+ Auto-generate `.llm-kb/config.json` on first run:
90
+
91
+ ```json
92
+ {
93
+ "indexModel": "claude-haiku-3-5",
94
+ "queryModel": "claude-sonnet-4-20250514",
95
+ "provider": "anthropic"
96
+ }
97
+ ```
98
+
99
+ Env var overrides: `LLM_KB_INDEX_MODEL`, `LLM_KB_QUERY_MODEL`
100
+ Priority: env var → config file → defaults
101
+
102
+ Definition of done:
103
+ - [ ] Config auto-generated on first run
104
+ - [ ] Haiku for indexing, Sonnet for query by default
105
+ - [ ] Env vars override config
106
+ - [ ] `llm-kb status` shows current model config
107
+
108
+ ## Slice 3: Trace logging
109
+
110
+ Every query logs JSON to `.llm-kb/traces/`:
111
+
112
+ ```json
113
+ {
114
+ "id": "2026-04-06T14-30-00-query",
115
+ "timestamp": "2026-04-06T14:30:00Z",
116
+ "question": "what are the reserve requirements?",
117
+ "mode": "query",
118
+ "filesRead": ["index.md", "reserve-policy.md"],
119
+ "filesSkipped": ["board-deck.md", "pipeline.md"],
120
+ "answer": "Reserve requirements are defined in...",
121
+ "citations": [
122
+ { "file": "reserve-policy.md", "location": "p.3", "claim": "Minimum reserve ratio of 12%" }
123
+ ],
124
+ "durationMs": 4200
125
+ }
126
+ ```
127
+
128
+ Capture via `session.subscribe()` — intercept tool calls to track `filesRead`.
129
+
130
+ Definition of done:
131
+ - [ ] Every query writes a trace JSON to `.llm-kb/traces/`
132
+ - [ ] Trace includes: question, mode, filesRead, filesSkipped, answer, citations, durationMs
133
+
134
+ ## Slice 4: `llm-kb status`
135
+
136
+ ```
137
+ Knowledge Base: ./research/.llm-kb/
138
+ Sources: 12 files (8 PDF, 2 XLSX, 1 DOCX, 1 TXT)
139
+ Index: 12 entries, last updated 2 min ago
140
+ Outputs: 3 saved research answers
141
+ Traces: 47 queries logged
142
+ Model: claude-sonnet-4 (query), claude-haiku-3-5 (index)
143
+ Auth: Pi SDK
144
+ ```
145
+
146
+ Definition of done:
147
+ - [ ] `llm-kb status` prints KB stats, auth method, model config, trace count
148
+
149
+ ## Slice 5: `llm-kb eval`
150
+
151
+ Pi SDK session (read-only) that:
152
+ 1. Reads trace files from `.llm-kb/traces/`
153
+ 2. For each trace checks: citation validity, missing sources, answer consistency
154
+ 3. Writes report to `.llm-kb/wiki/outputs/eval-report.md`
155
+ 4. Watcher detects report, re-indexes
156
+
157
+ ```bash
158
+ llm-kb eval --folder ./research
159
+ llm-kb eval --folder ./research --last 20
160
+ ```
161
+
162
+ Definition of done:
163
+ - [ ] `llm-kb eval` reads traces and writes eval-report.md
164
+ - [ ] Flags: invalid citations, skipped-but-relevant files, answer contradictions
165
+ - [ ] Report gets re-indexed by watcher
166
+
167
+ ## Slice 6: Blog Part 4
168
+
169
+ - Write AFTER `llm-kb eval` runs on real data
170
+ - Show actual eval-report.md output
171
+ - Title: "How llm-kb Knows When It Got It Wrong"
172
+
173
+ ---
174
+
175
+ ## Phase 3 Definition of Done
176
+
177
+ - [ ] `ANTHROPIC_API_KEY` works without Pi SDK installed
178
+ - [ ] Clear error when no auth found
179
+ - [ ] Config file with model selection (index vs query model)
180
+ - [ ] Every query logs a trace JSON to `.llm-kb/traces/`
181
+ - [ ] `llm-kb eval` checks citations and writes report
182
+ - [ ] `llm-kb status` shows KB stats + config + auth
183
+ - [ ] README updated with auth options + eval command
184
+ - [ ] Blog Part 4 written with real eval output
185
+
186
+ ---
187
+
188
+ ## Phase 4: Wiki Compilation (The Farzapedia Pattern)
189
+
190
+ > **Insight:** Farza's implementation is what Karpathy called the best version. The key isn't just indexing sources — it's compiling them into a **concept-organized wiki** with backlinked articles. The agent navigates concepts, not raw files.
191
+ >
192
+ > Current llm-kb: agent reads source summaries → picks source files → answers from raw docs.
193
+ > Wiki pattern: agent reads concept articles → drills into specific articles → answers from synthesized knowledge.
194
+
195
+ ### The Structural Shift
196
+
197
+ **Current (source-organized):**
198
+ ```
199
+ sources/lease-castlelake.md ─┐
200
+ sources/lease-genesis.md ─┤→ index.md (flat table, one row per source)
201
+ sources/maintenance-manual.md ─┘
202
+ ```
203
+
204
+ **Wiki (concept-organized):**
205
+ ```
206
+ sources/lease-castlelake.md ─┐
207
+ sources/lease-genesis.md ─┤→ articles/reserve-requirements.md
208
+ sources/maintenance-manual.md ─┘ articles/key-parties.md
209
+ articles/engine-types.md
210
+ articles/maintenance-obligations.md
211
+ index.md (catalog of ARTICLES with backlinks)
212
+ ```
213
+
214
+ Farza's quote: *"The structure of the wiki files and how it's all backlinked is very easily crawlable by any agent. Starting at index.md, the agent does a really good job at drilling into the specific pages it needs."*
215
+
216
+ Karpathy's quote: *"I thought I had to reach for fancy RAG, but the LLM has been pretty good about auto-maintaining index files and brief summaries of all the documents and it reads all the important related data fairly easily at this ~small scale."*
217
+
218
+ ### New Command
219
+
220
+ ```bash
221
+ llm-kb compile ./folder
222
+ ```
223
+
224
+ Or auto-triggered after indexing when `--wiki` flag is passed:
225
+
226
+ ```bash
227
+ llm-kb run ./folder --wiki # parse + index + compile wiki
228
+ ```
229
+
230
+ ### Wiki Directory Structure
231
+
232
+ ```
233
+ .llm-kb/
234
+ wiki/
235
+ sources/ # raw parsed files (unchanged)
236
+ articles/ # NEW: concept-organized articles
237
+ concepts/ # thematic articles (reserve-requirements.md, etc)
238
+ entities/ # people, companies, assets
239
+ timeline.md # chronological view of events
240
+ index.md # NOW points to articles, not sources
241
+ source-index.md # OLD flat source table (kept for reference)
242
+ ```
243
+
244
+ ### How Compile Works
245
+
246
+ 1. LLM reads ALL source files (via existing sources/ dir)
247
+ 2. LLM identifies key concepts, entities, and themes
248
+ 3. LLM writes one markdown article per concept — like a Wikipedia entry
249
+ 4. Each article has:
250
+ - Description (synthesized from multiple sources)
251
+ - Backlinks to related articles: `[[reserve-requirements]]`, `[[key-parties]]`
252
+ - Source citations: `(Source: lease-castlelake.md, p.3)`
253
+ 5. Writes `articles/index.md` — catalog of all articles with one-line descriptions
254
+
255
+ ### Incremental Updates (The Compounding Part)
256
+
257
+ When a new source is added (watcher detects it):
258
+ 1. Parse it → sources/
259
+ 2. LLM reads new source + `articles/index.md`
260
+ 3. LLM updates 2-3 most relevant existing articles where new content fits
261
+ 4. OR creates a new article if the topic is genuinely new
262
+ 5. Updates `articles/index.md` catalog
263
+
264
+ Farza: *"The most magical thing now is as I add new things, the system updates 2-3 different articles where it feels the context belongs, or just creates a new article. Like a super genius librarian."*
265
+
266
+ ### Query Change
267
+
268
+ With wiki mode active, query agent reads `articles/index.md` (concept catalog) instead of `source-index.md` (source table). Agent drills into concept articles, gets synthesized cross-referenced answers.
269
+
270
+ Fallback: if no articles compiled yet, falls back to source-index behavior.
271
+
272
+ ### Blog Post
273
+
274
+ Part 5 or 6: *"Why Your Knowledge Base Needs a Librarian, Not a Search Engine"*
275
+ - Show the before/after: flat source index vs concept wiki
276
+ - Show Farza's insight: built for agents, not humans
277
+ - Show the compounding: add one doc, 3 articles updated automatically
278
+ - Live demo: `llm-kb compile` on a real document set
279
+
280
+ ### Why This Matters for llm-kb
281
+
282
+ - Karpathy publicly called this the best implementation of his pattern
283
+ - Farza has **no public code** — llm-kb ships it as `npx llm-kb compile`
284
+ - Farzapedia got **920K views** on Farza's tweet, **Karpathy quote-tweeted** it at **920K views**
285
+ - First open-source implementation of the exact pattern Karpathy validated
286
+ - Blog post can reference both tweets legitimately ("inspired by Farza's Farzapedia")
287
+
288
+ ### Definition of Done (Phase 4)
289
+
290
+ - [ ] `llm-kb compile ./folder` reads sources and writes articles/ directory
291
+ - [ ] Each article is a proper markdown file with backlinks and source citations
292
+ - [ ] `articles/index.md` is a concept catalog (not a source table)
293
+ - [ ] Watcher triggers incremental article updates when new source added
294
+ - [ ] Query uses article index when available, falls back to source index
295
+ - [ ] `llm-kb status` shows: X sources, Y articles, last compiled
296
+ - [ ] Blog Part 5 written: concept wiki vs flat index, Farzapedia reference
297
+
298
+ ---
299
+
300
+ *Phase 4 added April 6, 2026 — inspired by Farzapedia (@FarzaTV), highlighted by Karpathy as best implementation of the LLM wiki pattern. 920K views on Farza's tweet, Karpathy quote-tweet at 920K views.*
package/src/auth.ts ADDED
@@ -0,0 +1,55 @@
1
+ import { existsSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { homedir } from "node:os";
4
+ import { AuthStorage } from "@mariozechner/pi-coding-agent";
5
+ import chalk from "chalk";
6
+
7
+ export type AuthMethod = "pi-sdk" | "api-key" | "none";
8
+
9
+ export interface AuthResult {
10
+ ok: true;
11
+ method: AuthMethod;
12
+ authStorage?: AuthStorage; // undefined = use Pi SDK default (file-based)
13
+ }
14
+
15
+ export interface AuthFailure {
16
+ ok: false;
17
+ }
18
+
19
+ /**
20
+ * Detect available auth and return an AuthStorage if needed.
21
+ *
22
+ * Priority:
23
+ * 1. Pi SDK auth.json → use default file-based storage (no override needed)
24
+ * 2. ANTHROPIC_API_KEY → create in-memory storage with the key
25
+ * 3. Neither → return failure
26
+ */
27
+ export function checkAuth(): AuthResult | AuthFailure {
28
+ const piAuthPath = join(homedir(), ".pi", "agent", "auth.json");
29
+
30
+ if (existsSync(piAuthPath)) {
31
+ return { ok: true, method: "pi-sdk" };
32
+ }
33
+
34
+ if (process.env.ANTHROPIC_API_KEY) {
35
+ const authStorage = AuthStorage.inMemory({
36
+ anthropic: { type: "api_key", key: process.env.ANTHROPIC_API_KEY },
37
+ });
38
+ return { ok: true, method: "api-key", authStorage };
39
+ }
40
+
41
+ return { ok: false };
42
+ }
43
+
44
+ /**
45
+ * Print a friendly error when no auth is configured and exit.
46
+ */
47
+ export function exitWithAuthError(): never {
48
+ console.error(chalk.red("\n No LLM authentication found.\n"));
49
+ console.error(` ${chalk.bold("Option 1:")} Install Pi SDK ${chalk.dim("(recommended)")}`);
50
+ console.error(chalk.dim(" npm install -g @mariozechner/pi-coding-agent"));
51
+ console.error(chalk.dim(" pi\n"));
52
+ console.error(` ${chalk.bold("Option 2:")} Set your Anthropic API key`);
53
+ console.error(chalk.dim(" export ANTHROPIC_API_KEY=sk-ant-...\n"));
54
+ process.exit(1);
55
+ }
package/src/cli.ts CHANGED
@@ -5,10 +5,15 @@ import { scan, summarize } from "./scan.js";
5
5
  import { parsePDF } from "./pdf.js";
6
6
  import { buildIndex } from "./indexer.js";
7
7
  import { startWatcher } from "./watcher.js";
8
- import { query } from "./query.js";
8
+ import { startSessionWatcher } from "./session-watcher.js";
9
+ import { query, createChat } from "./query.js";
10
+ import { runEval } from "./eval.js";
11
+ import { ChatDisplay } from "./tui-display.js";
9
12
  import { resolveKnowledgeBase } from "./resolve-kb.js";
13
+ import { checkAuth, exitWithAuthError } from "./auth.js";
14
+ import { ensureConfig, loadConfig } from "./config.js";
10
15
  import { existsSync } from "node:fs";
11
- import { mkdir } from "node:fs/promises";
16
+ import { mkdir, readdir, stat } from "node:fs/promises";
12
17
  import { resolve, join } from "node:path";
13
18
  import chalk from "chalk";
14
19
 
@@ -17,20 +22,26 @@ const program = new Command();
17
22
  program
18
23
  .name("llm-kb")
19
24
  .description("Drop files into a folder. Get a knowledge base you can query.")
20
- .version("0.2.0");
25
+ .version("0.4.0");
21
26
 
22
27
  program
23
28
  .command("run")
24
29
  .description("Scan, parse, index, and watch a folder")
25
30
  .argument("<folder>", "Path to your documents folder")
26
31
  .action(async (folder: string) => {
27
- console.log(`\n${chalk.bold("llm-kb")} v0.2.0\n`);
32
+ console.log(`\n${chalk.bold("llm-kb")} v0.4.0\n`);
33
+
34
+ const auth = checkAuth();
35
+ if (!auth.ok) exitWithAuthError();
28
36
 
29
37
  if (!existsSync(folder)) {
30
38
  console.error(chalk.red(`Error: Folder not found: ${folder}`));
31
39
  process.exit(1);
32
40
  }
33
41
 
42
+ const root = resolve(folder);
43
+ const config = await ensureConfig(root);
44
+
34
45
  console.log(`Scanning ${folder}...`);
35
46
 
36
47
  const files = await scan(folder);
@@ -44,12 +55,10 @@ program
44
55
  console.log(` Found ${chalk.bold(files.length.toString())} files (${summarize(files)})`);
45
56
  if (pdfs.length === 0) return;
46
57
 
47
- // Set up .llm-kb folder structure
48
- const root = resolve(folder);
49
58
  const sourcesDir = join(root, ".llm-kb", "wiki", "sources");
50
59
  await mkdir(sourcesDir, { recursive: true });
51
60
 
52
- // Parse PDFs with inline progress
61
+ // Parse PDFs
53
62
  let parsed = 0;
54
63
  let skipped = 0;
55
64
  let failed = 0;
@@ -57,76 +66,192 @@ program
57
66
 
58
67
  for (let i = 0; i < pdfs.length; i++) {
59
68
  const pdf = pdfs[i];
60
- const fullPath = join(root, pdf.path);
61
-
62
- // Inline progress — overwrite same line
63
- const progress = ` Parsing... ${i + 1}/${pdfs.length} — ${pdf.name}`;
64
- process.stdout.write(`\r${progress.padEnd(80)}`);
65
-
69
+ const progress = ` Parsing... ${i + 1}/${pdfs.length} \u2014 ${pdf.name}`;
70
+ process.stdout.write(`\r${progress.padEnd(process.stdout.columns || 80)}`);
66
71
  try {
67
- const result = await parsePDF(fullPath, sourcesDir);
68
- if (result.skipped) {
69
- skipped++;
70
- } else {
71
- parsed++;
72
- }
72
+ const result = await parsePDF(join(root, pdf.path), sourcesDir);
73
+ if (result.skipped) skipped++; else parsed++;
73
74
  } catch (err: any) {
74
75
  failed++;
75
76
  errors.push({ name: pdf.name, message: err.message });
76
77
  }
77
78
  }
78
79
 
79
- // Clear progress line
80
- process.stdout.write(`\r${"".padEnd(80)}\r`);
80
+ process.stdout.write(`\r${"".padEnd(process.stdout.columns || 80)}\r`);
81
81
 
82
- // Summary
83
82
  const parts: string[] = [];
84
83
  if (parsed > 0) parts.push(chalk.green(`${parsed} parsed`));
85
84
  if (skipped > 0) parts.push(chalk.dim(`${skipped} skipped (up to date)`));
86
85
  if (failed > 0) parts.push(chalk.red(`${failed} failed`));
87
86
  console.log(` ${parts.join(", ")}`);
87
+ for (const err of errors) console.log(chalk.red(` \u2717 ${err.name} \u2014 ${err.message}`));
88
88
 
89
- // Show errors
90
- for (const err of errors) {
91
- console.log(chalk.red(` ${err.name} ${err.message}`));
89
+ // Build index — skip if up to date
90
+ const indexFile = join(root, ".llm-kb", "wiki", "index.md");
91
+ let indexUpToDate = false;
92
+ if (parsed === 0 && existsSync(indexFile)) {
93
+ try {
94
+ const indexMtime = (await stat(indexFile)).mtimeMs;
95
+ const sourceFiles = await readdir(sourcesDir);
96
+ const mtimes = await Promise.all(sourceFiles.map((f) => stat(join(sourcesDir, f)).then((s) => s.mtimeMs)));
97
+ indexUpToDate = mtimes.every((mt) => indexMtime >= mt);
98
+ } catch {}
92
99
  }
93
100
 
94
- // Build index
95
- console.log(`\n Building index...`);
96
- try {
97
- await buildIndex(root, sourcesDir);
98
- console.log(chalk.green(` Index built: .llm-kb/wiki/index.md`));
99
- } catch (err: any) {
100
- console.error(chalk.red(` Index failed: ${err.message}`));
101
+ if (indexUpToDate) {
102
+ console.log(chalk.dim(`\n Index up to date.`));
103
+ } else {
104
+ console.log(`\n Building index... ${chalk.dim(`(${config.indexModel})`)}`);
105
+ try {
106
+ await buildIndex(root, sourcesDir, undefined, auth.authStorage, config.indexModel);
107
+ console.log(chalk.green(` Index built: .llm-kb/wiki/index.md`));
108
+ } catch (err: any) {
109
+ console.error(chalk.red(` Index failed: ${err.message}`));
110
+ }
101
111
  }
102
112
 
103
113
  console.log(`\n ${chalk.dim("Output:")} ${sourcesDir}`);
104
114
 
105
- // Start watching for new files
106
- console.log(chalk.dim(`\n Watching for new files... (Ctrl+C to stop)`));
107
- startWatcher({ folder: root, sourcesDir });
115
+ // Start watchers
116
+ startWatcher({ folder: root, sourcesDir, authStorage: auth.authStorage, indexModel: config.indexModel });
117
+ startSessionWatcher(root);
118
+
119
+ // TUI chat
120
+ const chatUI = new ChatDisplay();
121
+ const { session, display } = await createChat(root, {
122
+ authStorage: auth.authStorage,
123
+ modelId: config.queryModel,
124
+ tuiDisplay: chatUI,
125
+ });
126
+
127
+ chatUI.onSubmit = (text) => {
128
+ display.setQuestion(text);
129
+ // Fire-and-forget — don't await, TUI must stay responsive
130
+ session.prompt(text).catch(() => {});
131
+ };
132
+
133
+ chatUI.onExit = () => {
134
+ display.flush().then(() => {
135
+ session.dispose();
136
+ process.exit(0);
137
+ });
138
+ };
139
+
140
+ console.log(`\n${chalk.bold("Ready.")} Ask a question or drop files in to re-index.\n`);
141
+ chatUI.start();
108
142
  });
109
143
 
110
144
  program
111
145
  .command("query")
112
- .description("Ask a question across your knowledge base")
146
+ .description("Ask a single question (non-interactive, stdout)")
113
147
  .argument("<question>", "Your question")
114
148
  .option("--folder <path>", "Path to document folder (auto-detects if omitted)")
115
149
  .option("--save", "Save the answer to wiki/outputs/ (research mode)")
116
150
  .action(async (question: string, options: { folder?: string; save?: boolean }) => {
117
- const root = resolveKnowledgeBase(options.folder || process.cwd());
151
+ const auth = checkAuth();
152
+ if (!auth.ok) exitWithAuthError();
118
153
 
154
+ const root = resolveKnowledgeBase(options.folder || process.cwd());
119
155
  if (!root) {
120
156
  console.error(chalk.red("No knowledge base found. Run 'llm-kb run <folder>' first."));
121
157
  process.exit(1);
122
158
  }
123
159
 
160
+ const config = await loadConfig(root);
124
161
  try {
125
- await query(root, question, { save: options.save });
162
+ await query(root, question, {
163
+ save: options.save,
164
+ authStorage: auth.authStorage,
165
+ modelId: config.queryModel,
166
+ });
126
167
  } catch (err: any) {
127
168
  console.error(chalk.red(err.message));
128
169
  process.exit(1);
129
170
  }
130
171
  });
131
172
 
173
+ program
174
+ .command("eval")
175
+ .description("Analyze sessions for quality issues, wiki gaps, and performance")
176
+ .option("--folder <path>", "Path to document folder (auto-detects if omitted)")
177
+ .option("--last <n>", "Only check last N sessions", parseInt)
178
+ .action(async (options: { folder?: string; last?: number }) => {
179
+ const auth = checkAuth();
180
+ if (!auth.ok) exitWithAuthError();
181
+
182
+ const root = resolveKnowledgeBase(options.folder || process.cwd());
183
+ if (!root) {
184
+ console.error(chalk.red("No knowledge base found. Run 'llm-kb run <folder>' first."));
185
+ process.exit(1);
186
+ }
187
+
188
+ console.log(`\n${chalk.bold("llm-kb eval")}\n`);
189
+
190
+ const result = await runEval(root, {
191
+ authStorage: auth.authStorage,
192
+ last: options.last,
193
+ onProgress: (msg) => console.log(chalk.dim(` ${msg}`)),
194
+ });
195
+
196
+ const { metrics, issues, wikiGaps } = result;
197
+ const errors = issues.filter((i) => i.severity === "error").length;
198
+ const warnings = issues.filter((i) => i.severity === "warning").length;
199
+
200
+ console.log();
201
+ console.log(` ${chalk.bold("Results:")}`);
202
+ console.log(` Queries analyzed: ${metrics.totalQAs}`);
203
+ console.log(` Wiki hit rate: ${metrics.totalQAs > 0 ? Math.round(metrics.wikiHits / metrics.totalQAs * 100) : 0}%`);
204
+ console.log(` Wasted reads: ${metrics.wastedReads}`);
205
+ console.log(` Issues: ${errors > 0 ? chalk.red(`${errors} errors`) : chalk.green("0 errors")} ${warnings > 0 ? chalk.yellow(`${warnings} warnings`) : chalk.dim("0 warnings")}`);
206
+ console.log(` Wiki gaps: ${wikiGaps.length > 0 ? chalk.yellow(String(wikiGaps.length)) : chalk.green("0")}`);
207
+ console.log();
208
+ console.log(chalk.green(` Report: .llm-kb/wiki/outputs/eval-report.md`));
209
+ console.log();
210
+ });
211
+
212
+ program
213
+ .command("status")
214
+ .description("Show knowledge base stats and current config")
215
+ .option("--folder <path>", "Path to document folder (auto-detects if omitted)")
216
+ .action(async (options: { folder?: string }) => {
217
+ const root = resolveKnowledgeBase(options.folder || process.cwd());
218
+ if (!root) {
219
+ console.error(chalk.red("No knowledge base found. Run 'llm-kb run <folder>' first."));
220
+ process.exit(1);
221
+ }
222
+
223
+ const auth = checkAuth();
224
+ const config = await loadConfig(root);
225
+
226
+ const sourcesDir = join(root, ".llm-kb", "wiki", "sources");
227
+ const indexFile = join(root, ".llm-kb", "wiki", "index.md");
228
+ const articlesDir = join(root, ".llm-kb", "wiki", "articles");
229
+ const outputsDir = join(root, ".llm-kb", "wiki", "outputs");
230
+
231
+ let sourceCount = 0;
232
+ try { sourceCount = (await readdir(sourcesDir)).filter((f) => f.endsWith(".md")).length; } catch {}
233
+
234
+ let indexAge = "not built yet";
235
+ try {
236
+ const diffMin = Math.round((Date.now() - (await stat(indexFile)).mtimeMs) / 60000);
237
+ indexAge = diffMin < 1 ? "just now" : diffMin < 60 ? `${diffMin} min ago` : `${Math.round(diffMin / 60)} hr ago`;
238
+ } catch {}
239
+
240
+ let outputCount = 0;
241
+ try { outputCount = (await readdir(outputsDir)).filter((f) => f.endsWith(".md")).length; } catch {}
242
+
243
+ console.log(`\n${chalk.bold("Knowledge Base Status")}`);
244
+ console.log(` ${chalk.dim("Folder:")} ${root}`);
245
+ console.log(` ${chalk.dim("Sources:")} ${sourceCount > 0 ? `${sourceCount} parsed source${sourceCount !== 1 ? "s" : ""}` : chalk.yellow("none yet")}`);
246
+ console.log(` ${chalk.dim("Index:")} ${indexAge}`);
247
+ let articleCount = 0;
248
+ try { articleCount = (await readdir(articlesDir)).filter((f) => f.endsWith(".md") && f !== "index.md").length; } catch {}
249
+
250
+ if (articleCount > 0) console.log(` ${chalk.dim("Articles:")} ${articleCount} compiled`);
251
+ if (outputCount > 0) console.log(` ${chalk.dim("Outputs:")} ${outputCount} saved answer${outputCount !== 1 ? "s" : ""}`);
252
+ console.log(` ${chalk.dim("Models:")} ${chalk.cyan(config.queryModel)} ${chalk.dim("(query)")} ${chalk.cyan(config.indexModel)} ${chalk.dim("(index)")}`);
253
+ console.log(` ${chalk.dim("Auth:")} ${auth.ok ? (auth.method === "pi-sdk" ? "Pi SDK" : "ANTHROPIC_API_KEY") : chalk.red("not configured")}`);
254
+ console.log();
255
+ });
256
+
132
257
  program.parse();