llm-kb 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PHASE3_SPEC.md +245 -0
- package/PHASE4_SPEC.md +358 -0
- package/README.md +182 -61
- package/bin/cli.js +5473 -165
- package/package.json +11 -6
- package/plan.md +248 -3
- package/src/auth.ts +55 -0
- package/src/cli.ts +163 -38
- package/src/config.ts +61 -0
- package/src/eval.ts +548 -0
- package/src/indexer.ts +36 -32
- package/src/md-stream.ts +133 -0
- package/src/query.ts +408 -132
- package/src/session-store.ts +22 -0
- package/src/session-watcher.ts +89 -0
- package/src/trace-builder.ts +168 -0
- package/src/tui-display.ts +281 -0
- package/src/utils.ts +17 -0
- package/src/watcher.ts +5 -2
- package/src/wiki-updater.ts +136 -0
- package/test/auth.test.ts +65 -0
- package/test/config.test.ts +96 -0
- package/test/md-stream.test.ts +98 -0
- package/test/resolve-kb.test.ts +33 -0
- package/test/scan.test.ts +65 -0
- package/test/trace-builder.test.ts +215 -0
- package/vitest.config.ts +8 -0
- package/bin/chunk-MYQ36JJB.js +0 -118
- package/bin/indexer-LSYSZXZX.js +0 -6
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "llm-kb",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "LLM-powered knowledge base. Drop documents, build a wiki, ask questions. Inspired by Karpathy.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"llm-kb": "./bin/cli.js"
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
"type": "module",
|
|
9
9
|
"scripts": {
|
|
10
10
|
"build": "tsup src/cli.ts --format esm --out-dir bin --clean",
|
|
11
|
-
"dev": "tsup src/cli.ts --format esm --out-dir bin --watch"
|
|
11
|
+
"dev": "tsup src/cli.ts --format esm --out-dir bin --watch",
|
|
12
|
+
"test": "vitest run",
|
|
13
|
+
"test:watch": "vitest"
|
|
12
14
|
},
|
|
13
15
|
"keywords": [
|
|
14
16
|
"llm",
|
|
@@ -28,16 +30,19 @@
|
|
|
28
30
|
"dependencies": {
|
|
29
31
|
"@llamaindex/liteparse": "^1.4.4",
|
|
30
32
|
"@mariozechner/pi-coding-agent": "^0.65.0",
|
|
33
|
+
"adm-zip": "^0.5.17",
|
|
31
34
|
"chalk": "^5.6.2",
|
|
32
35
|
"chokidar": "^5.0.0",
|
|
33
36
|
"commander": "^13.1.0",
|
|
34
37
|
"exceljs": "^4.4.0",
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
+
"marked": "^15.0.12",
|
|
39
|
+
"marked-terminal": "^7.3.0",
|
|
40
|
+
"officeparser": "^6.0.7"
|
|
38
41
|
},
|
|
39
42
|
"devDependencies": {
|
|
43
|
+
"@types/marked-terminal": "^6.1.1",
|
|
40
44
|
"tsup": "^8.4.0",
|
|
41
|
-
"typescript": "^5.8.3"
|
|
45
|
+
"typescript": "^5.8.3",
|
|
46
|
+
"vitest": "^4.1.2"
|
|
42
47
|
}
|
|
43
48
|
}
|
package/plan.md
CHANGED
|
@@ -50,6 +50,251 @@ Config file has no readers yet. Deferred to Phase 2/3. README updated instead.
|
|
|
50
50
|
- `--save` flag — research mode, saves to `outputs/`, re-indexes
|
|
51
51
|
- Query mode is read-only (read tool only). Research mode adds bash + write.
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Phase 3: Auth Fix + Eval Loop + LLM Config
|
|
56
|
+
|
|
57
|
+
> Full spec: `PHASE3_SPEC.md`
|
|
58
|
+
> **Trigger:** 182 LinkedIn saves — people coming back to try `npx llm-kb run` this week.
|
|
59
|
+
|
|
60
|
+
### Why now
|
|
61
|
+
- **Auth fix is urgent.** Users hit a wall if Pi SDK isn't installed. `ANTHROPIC_API_KEY` must work as fallback.
|
|
62
|
+
- **Eval loop is the differentiator.** Nobody else building llm-wiki scripts has traces + citation checking.
|
|
63
|
+
- **Model config cuts cost.** Haiku for indexing (10x cheaper than Sonnet). Users shouldn't pay Sonnet prices for one-line summaries.
|
|
64
|
+
|
|
65
|
+
### Build order
|
|
66
|
+
|
|
67
|
+
## Slice 1: Auth fix 🔴 DO FIRST
|
|
68
|
+
|
|
69
|
+
Check auth before creating any session. Support two paths:
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
npx llm-kb run ./docs
|
|
73
|
+
├─ ~/.pi/agent/auth.json exists? → Pi SDK auth. Done.
|
|
74
|
+
├─ ANTHROPIC_API_KEY set? → Configure Pi SDK programmatically. Done.
|
|
75
|
+
└─ Neither? → Clear error:
|
|
76
|
+
No LLM authentication found.
|
|
77
|
+
Option 1: npm install -g @mariozechner/pi-coding-agent && pi
|
|
78
|
+
Option 2: export ANTHROPIC_API_KEY=sk-ant-...
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Definition of done:
|
|
82
|
+
- [ ] `ANTHROPIC_API_KEY=sk-... npx llm-kb run ./docs` works without Pi installed
|
|
83
|
+
- [ ] Pi SDK auth works as before (no regression)
|
|
84
|
+
- [ ] Clear error when neither is available
|
|
85
|
+
- [ ] README updated with both auth options
|
|
86
|
+
|
|
87
|
+
## Slice 2: LLM config
|
|
88
|
+
|
|
89
|
+
Auto-generate `.llm-kb/config.json` on first run:
|
|
90
|
+
|
|
91
|
+
```json
|
|
92
|
+
{
|
|
93
|
+
"indexModel": "claude-haiku-3-5",
|
|
94
|
+
"queryModel": "claude-sonnet-4-20250514",
|
|
95
|
+
"provider": "anthropic"
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Env var overrides: `LLM_KB_INDEX_MODEL`, `LLM_KB_QUERY_MODEL`
|
|
100
|
+
Priority: env var → config file → defaults
|
|
101
|
+
|
|
102
|
+
Definition of done:
|
|
103
|
+
- [ ] Config auto-generated on first run
|
|
104
|
+
- [ ] Haiku for indexing, Sonnet for query by default
|
|
105
|
+
- [ ] Env vars override config
|
|
106
|
+
- [ ] `llm-kb status` shows current model config
|
|
107
|
+
|
|
108
|
+
## Slice 3: Trace logging
|
|
109
|
+
|
|
110
|
+
Every query logs JSON to `.llm-kb/traces/`:
|
|
111
|
+
|
|
112
|
+
```json
|
|
113
|
+
{
|
|
114
|
+
"id": "2026-04-06T14-30-00-query",
|
|
115
|
+
"timestamp": "2026-04-06T14:30:00Z",
|
|
116
|
+
"question": "what are the reserve requirements?",
|
|
117
|
+
"mode": "query",
|
|
118
|
+
"filesRead": ["index.md", "reserve-policy.md"],
|
|
119
|
+
"filesSkipped": ["board-deck.md", "pipeline.md"],
|
|
120
|
+
"answer": "Reserve requirements are defined in...",
|
|
121
|
+
"citations": [
|
|
122
|
+
{ "file": "reserve-policy.md", "location": "p.3", "claim": "Minimum reserve ratio of 12%" }
|
|
123
|
+
],
|
|
124
|
+
"durationMs": 4200
|
|
125
|
+
}
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Capture via `session.subscribe()` — intercept tool calls to track `filesRead`.
|
|
129
|
+
|
|
130
|
+
Definition of done:
|
|
131
|
+
- [ ] Every query writes a trace JSON to `.llm-kb/traces/`
|
|
132
|
+
- [ ] Trace includes: question, mode, filesRead, filesSkipped, answer, citations, durationMs
|
|
133
|
+
|
|
134
|
+
## Slice 4: `llm-kb status`
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
Knowledge Base: ./research/.llm-kb/
|
|
138
|
+
Sources: 12 files (8 PDF, 2 XLSX, 1 DOCX, 1 TXT)
|
|
139
|
+
Index: 12 entries, last updated 2 min ago
|
|
140
|
+
Outputs: 3 saved research answers
|
|
141
|
+
Traces: 47 queries logged
|
|
142
|
+
Model: claude-sonnet-4 (query), claude-haiku-3-5 (index)
|
|
143
|
+
Auth: Pi SDK
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Definition of done:
|
|
147
|
+
- [ ] `llm-kb status` prints KB stats, auth method, model config, trace count
|
|
148
|
+
|
|
149
|
+
## Slice 5: `llm-kb eval`
|
|
150
|
+
|
|
151
|
+
Pi SDK session (read-only) that:
|
|
152
|
+
1. Reads trace files from `.llm-kb/traces/`
|
|
153
|
+
2. For each trace checks: citation validity, missing sources, answer consistency
|
|
154
|
+
3. Writes report to `.llm-kb/wiki/outputs/eval-report.md`
|
|
155
|
+
4. Watcher detects report, re-indexes
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
llm-kb eval --folder ./research
|
|
159
|
+
llm-kb eval --folder ./research --last 20
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Definition of done:
|
|
163
|
+
- [ ] `llm-kb eval` reads traces and writes eval-report.md
|
|
164
|
+
- [ ] Flags: invalid citations, skipped-but-relevant files, answer contradictions
|
|
165
|
+
- [ ] Report gets re-indexed by watcher
|
|
166
|
+
|
|
167
|
+
## Slice 6: Blog Part 4
|
|
168
|
+
|
|
169
|
+
- Write AFTER `llm-kb eval` runs on real data
|
|
170
|
+
- Show actual eval-report.md output
|
|
171
|
+
- Title: "How llm-kb Knows When It Got It Wrong"
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## Phase 3 Definition of Done
|
|
176
|
+
|
|
177
|
+
- [ ] `ANTHROPIC_API_KEY` works without Pi SDK installed
|
|
178
|
+
- [ ] Clear error when no auth found
|
|
179
|
+
- [ ] Config file with model selection (index vs query model)
|
|
180
|
+
- [ ] Every query logs a trace JSON to `.llm-kb/traces/`
|
|
181
|
+
- [ ] `llm-kb eval` checks citations and writes report
|
|
182
|
+
- [ ] `llm-kb status` shows KB stats + config + auth
|
|
183
|
+
- [ ] README updated with auth options + eval command
|
|
184
|
+
- [ ] Blog Part 4 written with real eval output
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## Phase 4: Wiki Compilation (The Farzapedia Pattern)
|
|
189
|
+
|
|
190
|
+
> **Insight:** Farza's implementation is what Karpathy called the best version. The key isn't just indexing sources — it's compiling them into a **concept-organized wiki** with backlinked articles. The agent navigates concepts, not raw files.
|
|
191
|
+
>
|
|
192
|
+
> Current llm-kb: agent reads source summaries → picks source files → answers from raw docs.
|
|
193
|
+
> Wiki pattern: agent reads concept articles → drills into specific articles → answers from synthesized knowledge.
|
|
194
|
+
|
|
195
|
+
### The Structural Shift
|
|
196
|
+
|
|
197
|
+
**Current (source-organized):**
|
|
198
|
+
```
|
|
199
|
+
sources/lease-castlelake.md ─┐
|
|
200
|
+
sources/lease-genesis.md ─┤→ index.md (flat table, one row per source)
|
|
201
|
+
sources/maintenance-manual.md ─┘
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
**Wiki (concept-organized):**
|
|
205
|
+
```
|
|
206
|
+
sources/lease-castlelake.md ─┐
|
|
207
|
+
sources/lease-genesis.md ─┤→ articles/reserve-requirements.md
|
|
208
|
+
sources/maintenance-manual.md ─┘ articles/key-parties.md
|
|
209
|
+
articles/engine-types.md
|
|
210
|
+
articles/maintenance-obligations.md
|
|
211
|
+
index.md (catalog of ARTICLES with backlinks)
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Farza's quote: *"The structure of the wiki files and how it's all backlinked is very easily crawlable by any agent. Starting at index.md, the agent does a really good job at drilling into the specific pages it needs."*
|
|
215
|
+
|
|
216
|
+
Karpathy's quote: *"I thought I had to reach for fancy RAG, but the LLM has been pretty good about auto-maintaining index files and brief summaries of all the documents and it reads all the important related data fairly easily at this ~small scale."*
|
|
217
|
+
|
|
218
|
+
### New Command
|
|
219
|
+
|
|
220
|
+
```bash
|
|
221
|
+
llm-kb compile ./folder
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Or auto-triggered after indexing when `--wiki` flag is passed:
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
llm-kb run ./folder --wiki # parse + index + compile wiki
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
### Wiki Directory Structure
|
|
231
|
+
|
|
232
|
+
```
|
|
233
|
+
.llm-kb/
|
|
234
|
+
wiki/
|
|
235
|
+
sources/ # raw parsed files (unchanged)
|
|
236
|
+
articles/ # NEW: concept-organized articles
|
|
237
|
+
concepts/ # thematic articles (reserve-requirements.md, etc)
|
|
238
|
+
entities/ # people, companies, assets
|
|
239
|
+
timeline.md # chronological view of events
|
|
240
|
+
index.md # NOW points to articles, not sources
|
|
241
|
+
source-index.md # OLD flat source table (kept for reference)
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
### How Compile Works
|
|
245
|
+
|
|
246
|
+
1. LLM reads ALL source files (via existing sources/ dir)
|
|
247
|
+
2. LLM identifies key concepts, entities, and themes
|
|
248
|
+
3. LLM writes one markdown article per concept — like a Wikipedia entry
|
|
249
|
+
4. Each article has:
|
|
250
|
+
- Description (synthesized from multiple sources)
|
|
251
|
+
- Backlinks to related articles: `[[reserve-requirements]]`, `[[key-parties]]`
|
|
252
|
+
- Source citations: `(Source: lease-castlelake.md, p.3)`
|
|
253
|
+
5. Writes `articles/index.md` — catalog of all articles with one-line descriptions
|
|
254
|
+
|
|
255
|
+
### Incremental Updates (The Compounding Part)
|
|
256
|
+
|
|
257
|
+
When a new source is added (watcher detects it):
|
|
258
|
+
1. Parse it → sources/
|
|
259
|
+
2. LLM reads new source + `articles/index.md`
|
|
260
|
+
3. LLM updates 2-3 most relevant existing articles where new content fits
|
|
261
|
+
4. OR creates a new article if the topic is genuinely new
|
|
262
|
+
5. Updates `articles/index.md` catalog
|
|
263
|
+
|
|
264
|
+
Farza: *"The most magical thing now is as I add new things, the system updates 2-3 different articles where it feels the context belongs, or just creates a new article. Like a super genius librarian."*
|
|
265
|
+
|
|
266
|
+
### Query Change
|
|
267
|
+
|
|
268
|
+
With wiki mode active, query agent reads `articles/index.md` (concept catalog) instead of `source-index.md` (source table). Agent drills into concept articles, gets synthesized cross-referenced answers.
|
|
269
|
+
|
|
270
|
+
Fallback: if no articles compiled yet, falls back to source-index behavior.
|
|
271
|
+
|
|
272
|
+
### Blog Post
|
|
273
|
+
|
|
274
|
+
Part 5 or 6: *"Why Your Knowledge Base Needs a Librarian, Not a Search Engine"*
|
|
275
|
+
- Show the before/after: flat source index vs concept wiki
|
|
276
|
+
- Show Farza's insight: built for agents, not humans
|
|
277
|
+
- Show the compounding: add one doc, 3 articles updated automatically
|
|
278
|
+
- Live demo: `llm-kb compile` on a real document set
|
|
279
|
+
|
|
280
|
+
### Why This Matters for llm-kb
|
|
281
|
+
|
|
282
|
+
- Karpathy publicly called this the best implementation of his pattern
|
|
283
|
+
- Farza has **no public code** — llm-kb ships it as `npx llm-kb compile`
|
|
284
|
+
- Farzapedia got **920K views** on Farza's tweet, **Karpathy quote-tweeted** it at **920K views**
|
|
285
|
+
- First open-source implementation of the exact pattern Karpathy validated
|
|
286
|
+
- Blog post can reference both tweets legitimately ("inspired by Farza's Farzapedia")
|
|
287
|
+
|
|
288
|
+
### Definition of Done (Phase 4)
|
|
289
|
+
|
|
290
|
+
- [ ] `llm-kb compile ./folder` reads sources and writes articles/ directory
|
|
291
|
+
- [ ] Each article is a proper markdown file with backlinks and source citations
|
|
292
|
+
- [ ] `articles/index.md` is a concept catalog (not a source table)
|
|
293
|
+
- [ ] Watcher triggers incremental article updates when new source added
|
|
294
|
+
- [ ] Query uses article index when available, falls back to source index
|
|
295
|
+
- [ ] `llm-kb status` shows: X sources, Y articles, last compiled
|
|
296
|
+
- [ ] Blog Part 5 written: concept wiki vs flat index, Farzapedia reference
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
*Phase 4 added April 6, 2026 — inspired by Farzapedia (@FarzaTV), highlighted by Karpathy as best implementation of the LLM wiki pattern. 920K views on Farza's tweet, Karpathy quote-tweet at 920K views.*
|
package/src/auth.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { AuthStorage } from "@mariozechner/pi-coding-agent";
|
|
5
|
+
import chalk from "chalk";
|
|
6
|
+
|
|
7
|
+
export type AuthMethod = "pi-sdk" | "api-key" | "none";
|
|
8
|
+
|
|
9
|
+
export interface AuthResult {
|
|
10
|
+
ok: true;
|
|
11
|
+
method: AuthMethod;
|
|
12
|
+
authStorage?: AuthStorage; // undefined = use Pi SDK default (file-based)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface AuthFailure {
|
|
16
|
+
ok: false;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Detect available auth and return an AuthStorage if needed.
|
|
21
|
+
*
|
|
22
|
+
* Priority:
|
|
23
|
+
* 1. Pi SDK auth.json → use default file-based storage (no override needed)
|
|
24
|
+
* 2. ANTHROPIC_API_KEY → create in-memory storage with the key
|
|
25
|
+
* 3. Neither → return failure
|
|
26
|
+
*/
|
|
27
|
+
export function checkAuth(): AuthResult | AuthFailure {
|
|
28
|
+
const piAuthPath = join(homedir(), ".pi", "agent", "auth.json");
|
|
29
|
+
|
|
30
|
+
if (existsSync(piAuthPath)) {
|
|
31
|
+
return { ok: true, method: "pi-sdk" };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (process.env.ANTHROPIC_API_KEY) {
|
|
35
|
+
const authStorage = AuthStorage.inMemory({
|
|
36
|
+
anthropic: { type: "api_key", key: process.env.ANTHROPIC_API_KEY },
|
|
37
|
+
});
|
|
38
|
+
return { ok: true, method: "api-key", authStorage };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return { ok: false };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Print a friendly error when no auth is configured and exit.
|
|
46
|
+
*/
|
|
47
|
+
export function exitWithAuthError(): never {
|
|
48
|
+
console.error(chalk.red("\n No LLM authentication found.\n"));
|
|
49
|
+
console.error(` ${chalk.bold("Option 1:")} Install Pi SDK ${chalk.dim("(recommended)")}`);
|
|
50
|
+
console.error(chalk.dim(" npm install -g @mariozechner/pi-coding-agent"));
|
|
51
|
+
console.error(chalk.dim(" pi\n"));
|
|
52
|
+
console.error(` ${chalk.bold("Option 2:")} Set your Anthropic API key`);
|
|
53
|
+
console.error(chalk.dim(" export ANTHROPIC_API_KEY=sk-ant-...\n"));
|
|
54
|
+
process.exit(1);
|
|
55
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -5,10 +5,15 @@ import { scan, summarize } from "./scan.js";
|
|
|
5
5
|
import { parsePDF } from "./pdf.js";
|
|
6
6
|
import { buildIndex } from "./indexer.js";
|
|
7
7
|
import { startWatcher } from "./watcher.js";
|
|
8
|
-
import {
|
|
8
|
+
import { startSessionWatcher } from "./session-watcher.js";
|
|
9
|
+
import { query, createChat } from "./query.js";
|
|
10
|
+
import { runEval } from "./eval.js";
|
|
11
|
+
import { ChatDisplay } from "./tui-display.js";
|
|
9
12
|
import { resolveKnowledgeBase } from "./resolve-kb.js";
|
|
13
|
+
import { checkAuth, exitWithAuthError } from "./auth.js";
|
|
14
|
+
import { ensureConfig, loadConfig } from "./config.js";
|
|
10
15
|
import { existsSync } from "node:fs";
|
|
11
|
-
import { mkdir } from "node:fs/promises";
|
|
16
|
+
import { mkdir, readdir, stat } from "node:fs/promises";
|
|
12
17
|
import { resolve, join } from "node:path";
|
|
13
18
|
import chalk from "chalk";
|
|
14
19
|
|
|
@@ -17,20 +22,26 @@ const program = new Command();
|
|
|
17
22
|
program
|
|
18
23
|
.name("llm-kb")
|
|
19
24
|
.description("Drop files into a folder. Get a knowledge base you can query.")
|
|
20
|
-
.version("0.
|
|
25
|
+
.version("0.4.0");
|
|
21
26
|
|
|
22
27
|
program
|
|
23
28
|
.command("run")
|
|
24
29
|
.description("Scan, parse, index, and watch a folder")
|
|
25
30
|
.argument("<folder>", "Path to your documents folder")
|
|
26
31
|
.action(async (folder: string) => {
|
|
27
|
-
console.log(`\n${chalk.bold("llm-kb")} v0.
|
|
32
|
+
console.log(`\n${chalk.bold("llm-kb")} v0.4.0\n`);
|
|
33
|
+
|
|
34
|
+
const auth = checkAuth();
|
|
35
|
+
if (!auth.ok) exitWithAuthError();
|
|
28
36
|
|
|
29
37
|
if (!existsSync(folder)) {
|
|
30
38
|
console.error(chalk.red(`Error: Folder not found: ${folder}`));
|
|
31
39
|
process.exit(1);
|
|
32
40
|
}
|
|
33
41
|
|
|
42
|
+
const root = resolve(folder);
|
|
43
|
+
const config = await ensureConfig(root);
|
|
44
|
+
|
|
34
45
|
console.log(`Scanning ${folder}...`);
|
|
35
46
|
|
|
36
47
|
const files = await scan(folder);
|
|
@@ -44,12 +55,10 @@ program
|
|
|
44
55
|
console.log(` Found ${chalk.bold(files.length.toString())} files (${summarize(files)})`);
|
|
45
56
|
if (pdfs.length === 0) return;
|
|
46
57
|
|
|
47
|
-
// Set up .llm-kb folder structure
|
|
48
|
-
const root = resolve(folder);
|
|
49
58
|
const sourcesDir = join(root, ".llm-kb", "wiki", "sources");
|
|
50
59
|
await mkdir(sourcesDir, { recursive: true });
|
|
51
60
|
|
|
52
|
-
// Parse PDFs
|
|
61
|
+
// Parse PDFs
|
|
53
62
|
let parsed = 0;
|
|
54
63
|
let skipped = 0;
|
|
55
64
|
let failed = 0;
|
|
@@ -57,76 +66,192 @@ program
|
|
|
57
66
|
|
|
58
67
|
for (let i = 0; i < pdfs.length; i++) {
|
|
59
68
|
const pdf = pdfs[i];
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
// Inline progress — overwrite same line
|
|
63
|
-
const progress = ` Parsing... ${i + 1}/${pdfs.length} — ${pdf.name}`;
|
|
64
|
-
process.stdout.write(`\r${progress.padEnd(80)}`);
|
|
65
|
-
|
|
69
|
+
const progress = ` Parsing... ${i + 1}/${pdfs.length} \u2014 ${pdf.name}`;
|
|
70
|
+
process.stdout.write(`\r${progress.padEnd(process.stdout.columns || 80)}`);
|
|
66
71
|
try {
|
|
67
|
-
const result = await parsePDF(
|
|
68
|
-
if (result.skipped)
|
|
69
|
-
skipped++;
|
|
70
|
-
} else {
|
|
71
|
-
parsed++;
|
|
72
|
-
}
|
|
72
|
+
const result = await parsePDF(join(root, pdf.path), sourcesDir);
|
|
73
|
+
if (result.skipped) skipped++; else parsed++;
|
|
73
74
|
} catch (err: any) {
|
|
74
75
|
failed++;
|
|
75
76
|
errors.push({ name: pdf.name, message: err.message });
|
|
76
77
|
}
|
|
77
78
|
}
|
|
78
79
|
|
|
79
|
-
|
|
80
|
-
process.stdout.write(`\r${"".padEnd(80)}\r`);
|
|
80
|
+
process.stdout.write(`\r${"".padEnd(process.stdout.columns || 80)}\r`);
|
|
81
81
|
|
|
82
|
-
// Summary
|
|
83
82
|
const parts: string[] = [];
|
|
84
83
|
if (parsed > 0) parts.push(chalk.green(`${parsed} parsed`));
|
|
85
84
|
if (skipped > 0) parts.push(chalk.dim(`${skipped} skipped (up to date)`));
|
|
86
85
|
if (failed > 0) parts.push(chalk.red(`${failed} failed`));
|
|
87
86
|
console.log(` ${parts.join(", ")}`);
|
|
87
|
+
for (const err of errors) console.log(chalk.red(` \u2717 ${err.name} \u2014 ${err.message}`));
|
|
88
88
|
|
|
89
|
-
//
|
|
90
|
-
|
|
91
|
-
|
|
89
|
+
// Build index — skip if up to date
|
|
90
|
+
const indexFile = join(root, ".llm-kb", "wiki", "index.md");
|
|
91
|
+
let indexUpToDate = false;
|
|
92
|
+
if (parsed === 0 && existsSync(indexFile)) {
|
|
93
|
+
try {
|
|
94
|
+
const indexMtime = (await stat(indexFile)).mtimeMs;
|
|
95
|
+
const sourceFiles = await readdir(sourcesDir);
|
|
96
|
+
const mtimes = await Promise.all(sourceFiles.map((f) => stat(join(sourcesDir, f)).then((s) => s.mtimeMs)));
|
|
97
|
+
indexUpToDate = mtimes.every((mt) => indexMtime >= mt);
|
|
98
|
+
} catch {}
|
|
92
99
|
}
|
|
93
100
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
if (indexUpToDate) {
|
|
102
|
+
console.log(chalk.dim(`\n Index up to date.`));
|
|
103
|
+
} else {
|
|
104
|
+
console.log(`\n Building index... ${chalk.dim(`(${config.indexModel})`)}`);
|
|
105
|
+
try {
|
|
106
|
+
await buildIndex(root, sourcesDir, undefined, auth.authStorage, config.indexModel);
|
|
107
|
+
console.log(chalk.green(` Index built: .llm-kb/wiki/index.md`));
|
|
108
|
+
} catch (err: any) {
|
|
109
|
+
console.error(chalk.red(` Index failed: ${err.message}`));
|
|
110
|
+
}
|
|
101
111
|
}
|
|
102
112
|
|
|
103
113
|
console.log(`\n ${chalk.dim("Output:")} ${sourcesDir}`);
|
|
104
114
|
|
|
105
|
-
// Start
|
|
106
|
-
|
|
107
|
-
|
|
115
|
+
// Start watchers
|
|
116
|
+
startWatcher({ folder: root, sourcesDir, authStorage: auth.authStorage, indexModel: config.indexModel });
|
|
117
|
+
startSessionWatcher(root);
|
|
118
|
+
|
|
119
|
+
// TUI chat
|
|
120
|
+
const chatUI = new ChatDisplay();
|
|
121
|
+
const { session, display } = await createChat(root, {
|
|
122
|
+
authStorage: auth.authStorage,
|
|
123
|
+
modelId: config.queryModel,
|
|
124
|
+
tuiDisplay: chatUI,
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
chatUI.onSubmit = (text) => {
|
|
128
|
+
display.setQuestion(text);
|
|
129
|
+
// Fire-and-forget — don't await, TUI must stay responsive
|
|
130
|
+
session.prompt(text).catch(() => {});
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
chatUI.onExit = () => {
|
|
134
|
+
display.flush().then(() => {
|
|
135
|
+
session.dispose();
|
|
136
|
+
process.exit(0);
|
|
137
|
+
});
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
console.log(`\n${chalk.bold("Ready.")} Ask a question or drop files in to re-index.\n`);
|
|
141
|
+
chatUI.start();
|
|
108
142
|
});
|
|
109
143
|
|
|
110
144
|
program
|
|
111
145
|
.command("query")
|
|
112
|
-
.description("Ask a question
|
|
146
|
+
.description("Ask a single question (non-interactive, stdout)")
|
|
113
147
|
.argument("<question>", "Your question")
|
|
114
148
|
.option("--folder <path>", "Path to document folder (auto-detects if omitted)")
|
|
115
149
|
.option("--save", "Save the answer to wiki/outputs/ (research mode)")
|
|
116
150
|
.action(async (question: string, options: { folder?: string; save?: boolean }) => {
|
|
117
|
-
const
|
|
151
|
+
const auth = checkAuth();
|
|
152
|
+
if (!auth.ok) exitWithAuthError();
|
|
118
153
|
|
|
154
|
+
const root = resolveKnowledgeBase(options.folder || process.cwd());
|
|
119
155
|
if (!root) {
|
|
120
156
|
console.error(chalk.red("No knowledge base found. Run 'llm-kb run <folder>' first."));
|
|
121
157
|
process.exit(1);
|
|
122
158
|
}
|
|
123
159
|
|
|
160
|
+
const config = await loadConfig(root);
|
|
124
161
|
try {
|
|
125
|
-
await query(root, question, {
|
|
162
|
+
await query(root, question, {
|
|
163
|
+
save: options.save,
|
|
164
|
+
authStorage: auth.authStorage,
|
|
165
|
+
modelId: config.queryModel,
|
|
166
|
+
});
|
|
126
167
|
} catch (err: any) {
|
|
127
168
|
console.error(chalk.red(err.message));
|
|
128
169
|
process.exit(1);
|
|
129
170
|
}
|
|
130
171
|
});
|
|
131
172
|
|
|
173
|
+
program
|
|
174
|
+
.command("eval")
|
|
175
|
+
.description("Analyze sessions for quality issues, wiki gaps, and performance")
|
|
176
|
+
.option("--folder <path>", "Path to document folder (auto-detects if omitted)")
|
|
177
|
+
.option("--last <n>", "Only check last N sessions", parseInt)
|
|
178
|
+
.action(async (options: { folder?: string; last?: number }) => {
|
|
179
|
+
const auth = checkAuth();
|
|
180
|
+
if (!auth.ok) exitWithAuthError();
|
|
181
|
+
|
|
182
|
+
const root = resolveKnowledgeBase(options.folder || process.cwd());
|
|
183
|
+
if (!root) {
|
|
184
|
+
console.error(chalk.red("No knowledge base found. Run 'llm-kb run <folder>' first."));
|
|
185
|
+
process.exit(1);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
console.log(`\n${chalk.bold("llm-kb eval")}\n`);
|
|
189
|
+
|
|
190
|
+
const result = await runEval(root, {
|
|
191
|
+
authStorage: auth.authStorage,
|
|
192
|
+
last: options.last,
|
|
193
|
+
onProgress: (msg) => console.log(chalk.dim(` ${msg}`)),
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
const { metrics, issues, wikiGaps } = result;
|
|
197
|
+
const errors = issues.filter((i) => i.severity === "error").length;
|
|
198
|
+
const warnings = issues.filter((i) => i.severity === "warning").length;
|
|
199
|
+
|
|
200
|
+
console.log();
|
|
201
|
+
console.log(` ${chalk.bold("Results:")}`);
|
|
202
|
+
console.log(` Queries analyzed: ${metrics.totalQAs}`);
|
|
203
|
+
console.log(` Wiki hit rate: ${metrics.totalQAs > 0 ? Math.round(metrics.wikiHits / metrics.totalQAs * 100) : 0}%`);
|
|
204
|
+
console.log(` Wasted reads: ${metrics.wastedReads}`);
|
|
205
|
+
console.log(` Issues: ${errors > 0 ? chalk.red(`${errors} errors`) : chalk.green("0 errors")} ${warnings > 0 ? chalk.yellow(`${warnings} warnings`) : chalk.dim("0 warnings")}`);
|
|
206
|
+
console.log(` Wiki gaps: ${wikiGaps.length > 0 ? chalk.yellow(String(wikiGaps.length)) : chalk.green("0")}`);
|
|
207
|
+
console.log();
|
|
208
|
+
console.log(chalk.green(` Report: .llm-kb/wiki/outputs/eval-report.md`));
|
|
209
|
+
console.log();
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
program
|
|
213
|
+
.command("status")
|
|
214
|
+
.description("Show knowledge base stats and current config")
|
|
215
|
+
.option("--folder <path>", "Path to document folder (auto-detects if omitted)")
|
|
216
|
+
.action(async (options: { folder?: string }) => {
|
|
217
|
+
const root = resolveKnowledgeBase(options.folder || process.cwd());
|
|
218
|
+
if (!root) {
|
|
219
|
+
console.error(chalk.red("No knowledge base found. Run 'llm-kb run <folder>' first."));
|
|
220
|
+
process.exit(1);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const auth = checkAuth();
|
|
224
|
+
const config = await loadConfig(root);
|
|
225
|
+
|
|
226
|
+
const sourcesDir = join(root, ".llm-kb", "wiki", "sources");
|
|
227
|
+
const indexFile = join(root, ".llm-kb", "wiki", "index.md");
|
|
228
|
+
const articlesDir = join(root, ".llm-kb", "wiki", "articles");
|
|
229
|
+
const outputsDir = join(root, ".llm-kb", "wiki", "outputs");
|
|
230
|
+
|
|
231
|
+
let sourceCount = 0;
|
|
232
|
+
try { sourceCount = (await readdir(sourcesDir)).filter((f) => f.endsWith(".md")).length; } catch {}
|
|
233
|
+
|
|
234
|
+
let indexAge = "not built yet";
|
|
235
|
+
try {
|
|
236
|
+
const diffMin = Math.round((Date.now() - (await stat(indexFile)).mtimeMs) / 60000);
|
|
237
|
+
indexAge = diffMin < 1 ? "just now" : diffMin < 60 ? `${diffMin} min ago` : `${Math.round(diffMin / 60)} hr ago`;
|
|
238
|
+
} catch {}
|
|
239
|
+
|
|
240
|
+
let outputCount = 0;
|
|
241
|
+
try { outputCount = (await readdir(outputsDir)).filter((f) => f.endsWith(".md")).length; } catch {}
|
|
242
|
+
|
|
243
|
+
console.log(`\n${chalk.bold("Knowledge Base Status")}`);
|
|
244
|
+
console.log(` ${chalk.dim("Folder:")} ${root}`);
|
|
245
|
+
console.log(` ${chalk.dim("Sources:")} ${sourceCount > 0 ? `${sourceCount} parsed source${sourceCount !== 1 ? "s" : ""}` : chalk.yellow("none yet")}`);
|
|
246
|
+
console.log(` ${chalk.dim("Index:")} ${indexAge}`);
|
|
247
|
+
let articleCount = 0;
|
|
248
|
+
try { articleCount = (await readdir(articlesDir)).filter((f) => f.endsWith(".md") && f !== "index.md").length; } catch {}
|
|
249
|
+
|
|
250
|
+
if (articleCount > 0) console.log(` ${chalk.dim("Articles:")} ${articleCount} compiled`);
|
|
251
|
+
if (outputCount > 0) console.log(` ${chalk.dim("Outputs:")} ${outputCount} saved answer${outputCount !== 1 ? "s" : ""}`);
|
|
252
|
+
console.log(` ${chalk.dim("Models:")} ${chalk.cyan(config.queryModel)} ${chalk.dim("(query)")} ${chalk.cyan(config.indexModel)} ${chalk.dim("(index)")}`);
|
|
253
|
+
console.log(` ${chalk.dim("Auth:")} ${auth.ok ? (auth.method === "pi-sdk" ? "Pi SDK" : "ANTHROPIC_API_KEY") : chalk.red("not configured")}`);
|
|
254
|
+
console.log();
|
|
255
|
+
});
|
|
256
|
+
|
|
132
257
|
program.parse();
|