@zosmaai/pi-llm-wiki 0.7.4 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +1 -0
- package/docs/api.md +301 -53
- package/extensions/llm-wiki/index.ts +17 -0
- package/extensions/llm-wiki/lib/source-extractors.ts +74 -11
- package/extensions/llm-wiki/lib/source-packet.ts +13 -3
- package/extensions/llm-wiki/lib/utils.ts +73 -3
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
### Fixed
|
|
6
|
+
- **Personal wiki created at doubled path `~/.llm-wiki/.llm-wiki/…`**: `getPersonalWikiRoot()` returned the dot-dir itself (`~/.llm-wiki`) while `getVaultPaths()` then appended another `.llm-wiki/` segment, so the personal vault was written to `~/.llm-wiki/.llm-wiki/wiki/…`. Fixed by aligning `getPersonalWikiRoot()` with the same "root = parent of `.llm-wiki/`" contract used by project vaults. `WIKI_HOME` continues to override the parent.
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- **`migrateDoubledPersonalVault()`** helper (`extensions/llm-wiki/lib/utils.ts`): Idempotent, in-place flatten of any vault that was already written to the broken doubled layout. Moves entries from `<root>/.llm-wiki/.llm-wiki/*` up to `<root>/.llm-wiki/*`, preserves outer entries on collision, removes the inner dir only when fully drained. Returns `null` when the layout is already correct, so it is safe to call on every session start.
|
|
10
|
+
- **Auto-migration on `session_start`**: The extension now runs `migrateDoubledPersonalVault()` on the personal wiki at every session start. Existing broken vaults are flattened the next time the user opens or reloads pi — no manual step required. A one-line status message is shown when a flatten actually happens; otherwise the check is silent.
|
|
11
|
+
- **`scripts/migrate-llm-wiki.js --fix-doubled`** flag: Manual recovery for arbitrary roots (`--fix-doubled ~/`, `--fix-doubled /some/project`). Supports `--dry-run` and `--force`.
|
|
12
|
+
- **9 regression tests** (`test/personal-wiki-paths.test.ts`): pin `getPersonalWikiRoot()` to the parent-of-dotdir contract, exercise `WIKI_HOME`, and verify the migration helper across no-op, idempotent, and collision paths.
|
|
13
|
+
|
|
5
14
|
## [0.7.0] - 2026-05-13
|
|
6
15
|
|
|
7
16
|
### Added
|
package/README.md
CHANGED
|
@@ -120,6 +120,7 @@ The result is a wiki that **compounds** as you capture sources, ask questions, a
|
|
|
120
120
|
| `/wiki-status` | Show a concise operational summary |
|
|
121
121
|
| `/wiki-digest [--period daily\|weekly]` | Generate a digest of recent activity |
|
|
122
122
|
| `/wiki-retro` | Save atomic insights from completed tasks |
|
|
123
|
+
| `/wiki-req <concept>` | Decompose a concept into atomic, traceable requirement pages |
|
|
123
124
|
|
|
124
125
|
---
|
|
125
126
|
|
package/docs/api.md
CHANGED
|
@@ -1,105 +1,353 @@
|
|
|
1
1
|
# API Reference
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
All 13 tools registered by the extension. Parameters marked `?` are optional.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
---
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
## wiki_bootstrap
|
|
8
|
+
|
|
9
|
+
Initialize a new LLM Wiki vault with the 4-layer architecture. Creates config, templates, schema,
|
|
10
|
+
and metadata scaffolding.
|
|
11
|
+
|
|
12
|
+
**Parameters**
|
|
13
|
+
|
|
14
|
+
| Name | Type | Required | Description |
|
|
15
|
+
|------|------|----------|-------------|
|
|
16
|
+
| `topic` | `string` | ✅ | Main topic of the wiki |
|
|
17
|
+
| `mode` | `string` | — | `"personal"` or `"company"` (default: `"personal"`) |
|
|
18
|
+
| `root` | `string` | — | Root directory to bootstrap in (default: current working directory) |
|
|
19
|
+
|
|
20
|
+
**Returns**
|
|
8
21
|
|
|
9
22
|
```
|
|
10
|
-
|
|
23
|
+
details: { root: string, mode: string, topic: string }
|
|
11
24
|
```
|
|
12
25
|
|
|
13
|
-
|
|
26
|
+
Confirmation text includes the vault path, directory layout, and a prompt to capture the first source.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## wiki_capture_source
|
|
14
31
|
|
|
15
|
-
Capture a URL, file, or text into an immutable source packet.
|
|
32
|
+
Capture a URL, local file, or pasted text into an immutable source packet and skeleton source page.
|
|
33
|
+
Provide exactly one of `url`, `file_path`, or `text`.
|
|
34
|
+
|
|
35
|
+
**Parameters**
|
|
36
|
+
|
|
37
|
+
| Name | Type | Required | Description |
|
|
38
|
+
|------|------|----------|-------------|
|
|
39
|
+
| `url` | `string` | — | URL to fetch and capture |
|
|
40
|
+
| `file_path` | `string` | — | Absolute or relative path to a local file (PDF, md, txt, html, XML, JSON) |
|
|
41
|
+
| `text` | `string` | — | Pasted text content to capture directly |
|
|
42
|
+
| `title` | `string` | — | Title override (used for `text` captures; inferred from URL/file otherwise) |
|
|
43
|
+
|
|
44
|
+
**Returns**
|
|
16
45
|
|
|
17
46
|
```
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
createSourcePage?: boolean
|
|
25
|
-
)
|
|
47
|
+
details: {
|
|
48
|
+
sourceId: string, // e.g. "SRC-2026-06-03-001"
|
|
49
|
+
packetPath: string, // path to raw/sources/SRC-.../
|
|
50
|
+
sourcePagePath: string, // path to wiki/sources/SRC-....md (skeleton)
|
|
51
|
+
extractedPreview: string // first 300 chars of extracted content
|
|
52
|
+
}
|
|
26
53
|
```
|
|
27
54
|
|
|
28
|
-
|
|
55
|
+
Errors with `isError: true` if no vault exists or no source input is provided.
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## wiki_ingest
|
|
60
|
+
|
|
61
|
+
Return a batch of uningested source packets for the LLM to synthesize. Does not write anything
|
|
62
|
+
itself — the model reads the returned extracted content, fills in the skeleton source page,
|
|
63
|
+
and creates entity/concept pages.
|
|
64
|
+
|
|
65
|
+
**Parameters**
|
|
66
|
+
|
|
67
|
+
| Name | Type | Required | Description |
|
|
68
|
+
|------|------|----------|-------------|
|
|
69
|
+
| `source_id` | `string` | — | Process a specific source ID only; leave empty to get the next unprocessed batch |
|
|
70
|
+
| `batch_size` | `number` | — | Max sources to return (default: `3`, max: `5`) |
|
|
71
|
+
|
|
72
|
+
**Returns**
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
details: {
|
|
76
|
+
batch: string[], // source IDs in this batch, e.g. ["SRC-2026-06-03-001"]
|
|
77
|
+
remaining: number // sources still waiting after this batch
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Each batch entry includes the source title, char count, and the path to read (`raw/sources/{id}/extracted.md`).
|
|
82
|
+
Returns a "all sources ingested" message with `{ ingested, total }` when nothing is pending.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## wiki_ensure_page
|
|
87
|
+
|
|
88
|
+
Resolve or safely create a canonical wiki page. Returns immediately if the page already exists
|
|
89
|
+
(no overwrite). Uses a built-in template when `content` is not provided.
|
|
90
|
+
|
|
91
|
+
**Parameters**
|
|
92
|
+
|
|
93
|
+
| Name | Type | Required | Description |
|
|
94
|
+
|------|------|----------|-------------|
|
|
95
|
+
| `type` | `string` | ✅ | Page type: `"entity"`, `"concept"`, `"synthesis"`, `"analysis"`, or `"requirement"` |
|
|
96
|
+
| `title` | `string` | ✅ | Human-readable page title; auto-slugified to a kebab-case filename |
|
|
97
|
+
| `content` | `string` | — | Full markdown content for the page; if omitted, the type-appropriate template is used |
|
|
98
|
+
|
|
99
|
+
**Returns**
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
details: { path: string, created: boolean }
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
`created: false` means the page already existed and was not modified.
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## wiki_recall
|
|
110
|
+
|
|
111
|
+
Search both the personal (`~/.llm-wiki/`) and project (`.llm-wiki/`) vaults for pages relevant to
|
|
112
|
+
a query. Uses chunk-level scoring, weighted field matching, and pseudo-relevance feedback. Also
|
|
113
|
+
called automatically before every agent turn.
|
|
114
|
+
|
|
115
|
+
**Parameters**
|
|
116
|
+
|
|
117
|
+
| Name | Type | Required | Description |
|
|
118
|
+
|------|------|----------|-------------|
|
|
119
|
+
| `query` | `string` | ✅ | Search query — use the user's full request or key terms |
|
|
120
|
+
| `max_results` | `number` | — | Maximum pages to return (default: `5`, max: `10`) |
|
|
121
|
+
|
|
122
|
+
**Returns**
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
details: {
|
|
126
|
+
query: string,
|
|
127
|
+
matches: Array<{
|
|
128
|
+
id: string, // folder-qualified page ID, e.g. "concepts/rag"
|
|
129
|
+
title: string,
|
|
130
|
+
type: string, // "source" | "entity" | "concept" | "synthesis" | "analysis"
|
|
131
|
+
preview: string, // best-matching chunk or page intro (~200 chars)
|
|
132
|
+
path: string, // absolute filesystem path to the .md file
|
|
133
|
+
score: number, // relevance score (higher = better)
|
|
134
|
+
vaultLabel?: string // "📓 personal" when result is from the personal vault
|
|
135
|
+
}>
|
|
136
|
+
}
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Returns empty `matches: []` with a hint to use `wiki_retro` when the wiki has no matching pages.
|
|
140
|
+
|
|
141
|
+
---
|
|
142
|
+
|
|
143
|
+
## wiki_search
|
|
29
144
|
|
|
30
|
-
|
|
145
|
+
Exact keyword search across the generated registry. Faster and simpler than `wiki_recall` — no
|
|
146
|
+
scoring, no PRF, no vault layering. Use for lookups when you already know what you're looking for.
|
|
147
|
+
|
|
148
|
+
**Parameters**
|
|
149
|
+
|
|
150
|
+
| Name | Type | Required | Description |
|
|
151
|
+
|------|------|----------|-------------|
|
|
152
|
+
| `query` | `string` | ✅ | Search term matched against page IDs, titles, and types |
|
|
153
|
+
| `type` | `string` | — | Filter results to a specific page type (e.g. `"concept"`, `"entity"`) |
|
|
154
|
+
|
|
155
|
+
**Returns**
|
|
31
156
|
|
|
32
157
|
```
|
|
33
|
-
|
|
158
|
+
details: {
|
|
159
|
+
query: string,
|
|
160
|
+
matches: Array<{ id: string, title: string, type: string }>
|
|
161
|
+
}
|
|
34
162
|
```
|
|
35
163
|
|
|
36
|
-
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## wiki_retro
|
|
167
|
+
|
|
168
|
+
Save an atomic insight from a completed task as a single lightweight markdown file in
|
|
169
|
+
`wiki/sources/`. Does not create a full source packet. Rebuilds metadata immediately so the
|
|
170
|
+
insight is searchable in the same session.
|
|
171
|
+
|
|
172
|
+
**Parameters**
|
|
37
173
|
|
|
38
|
-
|
|
174
|
+
| Name | Type | Required | Description |
|
|
175
|
+
|------|------|----------|-------------|
|
|
176
|
+
| `slug` | `string` | ✅ | Unique kebab-case identifier (e.g. `"jwt-revocation-pattern"`). Used as the filename and for lookups. |
|
|
177
|
+
| `title` | `string` | ✅ | Short descriptive title, 60 chars max. Noun phrase, not a sentence. |
|
|
178
|
+
| `body` | `string` | ✅ | Markdown content explaining what was learned. Include `[[wikilinks]]` to related pages. |
|
|
179
|
+
| `category` | `string` | — | Optional grouping label (e.g. `"frontend"`, `"architecture"`, `"devops"`, `"bugfix"`) |
|
|
180
|
+
|
|
181
|
+
**Returns**
|
|
39
182
|
|
|
40
183
|
```
|
|
41
|
-
|
|
42
|
-
type: "concept" | "entity" | "synthesis" | "analysis",
|
|
43
|
-
title: string,
|
|
44
|
-
aliases?: string[],
|
|
45
|
-
tags?: string[],
|
|
46
|
-
summary?: string,
|
|
47
|
-
createIfMissing?: boolean
|
|
48
|
-
)
|
|
184
|
+
details: { slug: string, title: string, category: string | null }
|
|
49
185
|
```
|
|
50
186
|
|
|
51
|
-
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## wiki_observe
|
|
190
|
+
|
|
191
|
+
Record a timestamped, relevance-rated observation during a session. Saved to `wiki/sources/` with
|
|
192
|
+
`status: observation`. Immediately searchable via `wiki_recall`. Intended for mid-session capture;
|
|
193
|
+
use `wiki_retro` for end-of-task summaries.
|
|
194
|
+
|
|
195
|
+
**Parameters**
|
|
196
|
+
|
|
197
|
+
| Name | Type | Required | Description |
|
|
198
|
+
|------|------|----------|-------------|
|
|
199
|
+
| `title` | `string` | ✅ | Short descriptive title, ≤80 chars. Noun phrase, not a sentence. |
|
|
200
|
+
| `content` | `string` | ✅ | Plain prose: what happened, was decided, or was learned. Preserve specifics (file paths, function names, error messages, numbers). |
|
|
201
|
+
| `relevance` | `"low" \| "medium" \| "high" \| "critical"` | ✅ | Retention priority. `low` = routine; `medium` = task context; `high` = non-trivial decisions; `critical` = persistent identity/preference or completed work that must not be redone. |
|
|
202
|
+
| `tags` | `string` | — | Space-separated tags for categorisation (e.g. `"auth backend migration"`) |
|
|
203
|
+
| `source_context` | `string` | — | What was being worked on (e.g. `"Adding authentication module"`) |
|
|
52
204
|
|
|
53
|
-
|
|
205
|
+
**Returns**
|
|
54
206
|
|
|
55
207
|
```
|
|
56
|
-
|
|
208
|
+
details: { slug: string, title: string, relevance: string, tags: string | null }
|
|
57
209
|
```
|
|
58
210
|
|
|
59
|
-
|
|
211
|
+
The slug is auto-generated as `obs-YYYY-MM-DD-{title-slug}`.
|
|
60
212
|
|
|
61
|
-
|
|
213
|
+
---
|
|
214
|
+
|
|
215
|
+
## wiki_lint
|
|
216
|
+
|
|
217
|
+
Deterministic health check of the wiki. Scans for orphan pages (no inbound links), missing pages
|
|
218
|
+
(linked but not created), and contradiction markers. Optionally auto-creates stub pages for
|
|
219
|
+
knowledge gaps cited in two or more pages.
|
|
220
|
+
|
|
221
|
+
**Parameters**
|
|
222
|
+
|
|
223
|
+
| Name | Type | Required | Description |
|
|
224
|
+
|------|------|----------|-------------|
|
|
225
|
+
| `auto_fix` | `boolean` | — | When `true`, auto-creates stub concept pages for gaps mentioned in ≥2 pages (default: `false`) |
|
|
226
|
+
|
|
227
|
+
**Returns**
|
|
62
228
|
|
|
63
229
|
```
|
|
64
|
-
|
|
230
|
+
details: {
|
|
231
|
+
pages: number,
|
|
232
|
+
orphans: number,
|
|
233
|
+
missingPages: number,
|
|
234
|
+
contradictions: number,
|
|
235
|
+
reportPath: string, // path to the generated lint report .md file
|
|
236
|
+
gaps: number // knowledge gaps tracked in .discoveries/gaps.json
|
|
237
|
+
}
|
|
65
238
|
```
|
|
66
239
|
|
|
67
|
-
|
|
240
|
+
The lint report is written to `.llm-wiki/outputs/lint-YYYY-MM-DD.md`.
|
|
241
|
+
Contradictions are flagged by the presence of `⚠️ **Contradiction` markers in page content and
|
|
242
|
+
always require human review.
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## wiki_status
|
|
247
|
+
|
|
248
|
+
Report wiki health and statistics from the generated registry. Reads pre-built metadata — does not
|
|
249
|
+
scan files directly.
|
|
250
|
+
|
|
251
|
+
**Parameters**
|
|
252
|
+
|
|
253
|
+
None.
|
|
68
254
|
|
|
69
|
-
|
|
255
|
+
**Returns**
|
|
70
256
|
|
|
71
257
|
```
|
|
72
|
-
|
|
258
|
+
details: {
|
|
259
|
+
topic: string,
|
|
260
|
+
mode: string, // "personal" or "company"
|
|
261
|
+
totalPages: number,
|
|
262
|
+
byType: Record<string, number>, // e.g. { concept: 4, entity: 2, source: 7 }
|
|
263
|
+
orphans: number,
|
|
264
|
+
gaps: number,
|
|
265
|
+
health: "✅ Good" | "⚠️ Warning" | "🔴 Empty"
|
|
266
|
+
}
|
|
73
267
|
```
|
|
74
268
|
|
|
75
|
-
|
|
269
|
+
Health is `"⚠️ Warning"` when orphan count exceeds 5, `"🔴 Empty"` when the registry has no pages.
|
|
76
270
|
|
|
77
|
-
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## wiki_rebuild_meta
|
|
274
|
+
|
|
275
|
+
Force a full synchronous rebuild of all generated metadata: `registry.json`, `backlinks.json`,
|
|
276
|
+
`index.md`, `log.md`. Use when metadata appears out of sync with actual wiki files.
|
|
277
|
+
|
|
278
|
+
**Parameters**
|
|
279
|
+
|
|
280
|
+
None.
|
|
281
|
+
|
|
282
|
+
**Returns**
|
|
78
283
|
|
|
79
284
|
```
|
|
80
|
-
|
|
285
|
+
details: { pageCount: number }
|
|
81
286
|
```
|
|
82
287
|
|
|
83
|
-
|
|
288
|
+
---
|
|
84
289
|
|
|
85
|
-
|
|
290
|
+
## wiki_log_event
|
|
291
|
+
|
|
292
|
+
Append a structured event to `meta/events.jsonl` and regenerate `meta/log.md`. Every event is
|
|
293
|
+
timestamped automatically.
|
|
294
|
+
|
|
295
|
+
**Parameters**
|
|
296
|
+
|
|
297
|
+
| Name | Type | Required | Description |
|
|
298
|
+
|------|------|----------|-------------|
|
|
299
|
+
| `kind` | `string` | ✅ | Event kind label (e.g. `"ingest"`, `"query"`, `"decision"`, `"integrate"`) |
|
|
300
|
+
| `details` | `object` | — | Arbitrary additional fields to store alongside the event |
|
|
301
|
+
|
|
302
|
+
**Returns**
|
|
86
303
|
|
|
87
304
|
```
|
|
88
|
-
|
|
89
|
-
kind: string,
|
|
90
|
-
title: string,
|
|
91
|
-
summary?: string,
|
|
92
|
-
sourceIds?: string[],
|
|
93
|
-
pagePaths?: string[],
|
|
94
|
-
notes?: string[],
|
|
95
|
-
actor?: "agent" | "user" | "extension"
|
|
96
|
-
)
|
|
305
|
+
details: { kind: string }
|
|
97
306
|
```
|
|
98
307
|
|
|
99
|
-
|
|
308
|
+
---
|
|
309
|
+
|
|
310
|
+
## wiki_watch
|
|
100
311
|
|
|
101
|
-
|
|
312
|
+
Output the shell command needed to schedule automatic wiki updates (discover → ingest → lint) via
|
|
313
|
+
pi's `schedule_prompt` cron system. Does not schedule anything directly — it returns the command
|
|
314
|
+
for the user to run.
|
|
315
|
+
|
|
316
|
+
**Parameters**
|
|
317
|
+
|
|
318
|
+
| Name | Type | Required | Description |
|
|
319
|
+
|------|------|----------|-------------|
|
|
320
|
+
| `interval` | `string` | ✅ | `"daily"` (8:00 AM), `"weekly"` (Monday 9:00 AM), `"hourly"`, or `"stop"` (prints removal instructions) |
|
|
321
|
+
|
|
322
|
+
**Returns**
|
|
102
323
|
|
|
103
324
|
```
|
|
104
|
-
|
|
325
|
+
details: {
|
|
326
|
+
interval: string,
|
|
327
|
+
cronSchedule: string, // e.g. "0 0 8 * * *"
|
|
328
|
+
label: string // e.g. "Daily at 8:00 AM"
|
|
329
|
+
}
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
When `interval` is `"stop"`, returns `details: { action: "stop_instructions" }` with instructions
|
|
333
|
+
for removing existing jobs via `schedule_prompt action=remove`.
|
|
334
|
+
|
|
335
|
+
---
|
|
336
|
+
|
|
337
|
+
## Error Shape
|
|
338
|
+
|
|
339
|
+
All tools return `isError: true` in their result when a hard error occurs (no vault found, missing
|
|
340
|
+
required input). The `text` content will contain a human-readable explanation. Check for `isError`
|
|
341
|
+
before using `details`.
|
|
342
|
+
|
|
343
|
+
```ts
|
|
344
|
+
{
|
|
345
|
+
content: [{ type: "text", text: string }],
|
|
346
|
+
details: { error: string },
|
|
347
|
+
isError: true
|
|
348
|
+
}
|
|
105
349
|
```
|
|
350
|
+
|
|
351
|
+
The most common error is **"No wiki found — run wiki_bootstrap first"**, returned by every tool
|
|
352
|
+
except `wiki_bootstrap` itself when `.llm-wiki/config.json` does not exist in the resolved vault
|
|
353
|
+
root.
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
ensureVaultStructure,
|
|
26
26
|
fmtDate,
|
|
27
27
|
getVaultPaths,
|
|
28
|
+
migrateDoubledPersonalVault,
|
|
28
29
|
resolveVaultPaths,
|
|
29
30
|
writeJson,
|
|
30
31
|
} from "./lib/utils.js";
|
|
@@ -70,6 +71,22 @@ export default function (pi: ExtensionAPI) {
|
|
|
70
71
|
let needsTopicInference = false;
|
|
71
72
|
|
|
72
73
|
pi.on("session_start", async (_event, ctx) => {
|
|
74
|
+
// One-shot recovery for vaults created with the broken personal-root
|
|
75
|
+
// (~/.llm-wiki/.llm-wiki/… doubled layout). Runs on every session start
|
|
76
|
+
// because it is a cheap existence-check no-op when the layout is correct.
|
|
77
|
+
try {
|
|
78
|
+
const migration = migrateDoubledPersonalVault();
|
|
79
|
+
if (migration && migration.moved.length > 0) {
|
|
80
|
+
ctx.ui.setStatus(
|
|
81
|
+
"llm-wiki",
|
|
82
|
+
`🧠 Personal wiki layout fixed: flattened ${migration.moved.length} entries out of ${migration.from} (see CHANGELOG)`,
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
} catch (err) {
|
|
86
|
+
// Never let migration crash session start.
|
|
87
|
+
console.warn(`[llm-wiki] doubled-dotdir migration skipped: ${(err as Error).message}`);
|
|
88
|
+
}
|
|
89
|
+
|
|
73
90
|
const paths = resolveVaultPaths(process.cwd());
|
|
74
91
|
if (!existsSync(join(paths.dotWiki, "config.json"))) {
|
|
75
92
|
// Silently create the wiki vault — no UI prompts
|
|
@@ -1,14 +1,21 @@
|
|
|
1
1
|
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
2
2
|
import { exec } from "./utils.js";
|
|
3
3
|
|
|
4
|
+
export type ExtractionStatus = "success" | "failed" | "unsupported";
|
|
5
|
+
|
|
4
6
|
export interface ExtractedContent {
|
|
5
7
|
extracted: string;
|
|
6
8
|
title?: string;
|
|
9
|
+
extractor?: string;
|
|
10
|
+
extraction_status?: ExtractionStatus;
|
|
11
|
+
content_type?: string;
|
|
7
12
|
}
|
|
8
13
|
|
|
9
14
|
export interface FileExtractor {
|
|
10
15
|
format: string;
|
|
11
16
|
shouldReadText: boolean;
|
|
17
|
+
extractorName?: string;
|
|
18
|
+
content_type?: string;
|
|
12
19
|
matches(filePath: string): boolean;
|
|
13
20
|
extract(args: FileExtractArgs): Promise<string> | string;
|
|
14
21
|
}
|
|
@@ -38,25 +45,38 @@ const FILE_EXTRACTORS: FileExtractor[] = [
|
|
|
38
45
|
{
|
|
39
46
|
format: "pdf",
|
|
40
47
|
shouldReadText: false,
|
|
48
|
+
extractorName: "markitdown",
|
|
49
|
+
content_type: "application/pdf",
|
|
41
50
|
matches: hasExtension(".pdf"),
|
|
42
51
|
extract: ({ pi, filePath, signal }) => extractPdf(pi, filePath, signal),
|
|
43
52
|
},
|
|
44
|
-
textFileExtractor("markdown", [".md"]),
|
|
45
|
-
textFileExtractor("text", [".txt"]),
|
|
46
|
-
textFileExtractor("html", [".html", ".htm"]),
|
|
53
|
+
textFileExtractor("markdown", [".md"], "text/markdown"),
|
|
54
|
+
textFileExtractor("text", [".txt"], "text/plain"),
|
|
55
|
+
textFileExtractor("html", [".html", ".htm"], "text/html"),
|
|
47
56
|
{
|
|
48
57
|
format: "xml",
|
|
49
58
|
shouldReadText: true,
|
|
59
|
+
extractorName: "xmlToMarkdown",
|
|
60
|
+
content_type: "application/xml",
|
|
50
61
|
matches: hasExtension(".xml"),
|
|
51
62
|
extract: ({ content }) => xmlToMarkdown(content),
|
|
52
63
|
},
|
|
53
64
|
{
|
|
54
65
|
format: "json",
|
|
55
66
|
shouldReadText: true,
|
|
67
|
+
extractorName: "jsonToMarkdown",
|
|
68
|
+
content_type: "application/json",
|
|
56
69
|
matches: hasExtension(".json"),
|
|
57
70
|
extract: ({ content }) => jsonToMarkdown(content),
|
|
58
71
|
},
|
|
59
|
-
|
|
72
|
+
{
|
|
73
|
+
format: "docx",
|
|
74
|
+
shouldReadText: false,
|
|
75
|
+
extractorName: "markitdown",
|
|
76
|
+
content_type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
77
|
+
matches: hasExtension(".docx"),
|
|
78
|
+
extract: ({ pi, filePath, signal }) => extractDocx(pi, filePath, signal),
|
|
79
|
+
},
|
|
60
80
|
textFileExtractor("file", []),
|
|
61
81
|
];
|
|
62
82
|
|
|
@@ -91,10 +111,16 @@ export function pdfExtractionFailureMessage(source: string): string {
|
|
|
91
111
|
return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
|
|
92
112
|
}
|
|
93
113
|
|
|
94
|
-
function textFileExtractor(
|
|
114
|
+
function textFileExtractor(
|
|
115
|
+
format: string,
|
|
116
|
+
extensions: string[],
|
|
117
|
+
contentType?: string,
|
|
118
|
+
): FileExtractor {
|
|
95
119
|
return {
|
|
96
120
|
format,
|
|
97
121
|
shouldReadText: true,
|
|
122
|
+
extractorName: "passthrough",
|
|
123
|
+
content_type: contentType,
|
|
98
124
|
matches: extensions.length ? hasAnyExtension(extensions) : () => true,
|
|
99
125
|
extract: ({ content }) => content,
|
|
100
126
|
};
|
|
@@ -113,13 +139,33 @@ async function extractPdf(pi: ExtensionAPI, source: string, signal?: AbortSignal
|
|
|
113
139
|
return extracted || pdfExtractionFailureMessage(source);
|
|
114
140
|
}
|
|
115
141
|
|
|
142
|
+
export function docxExtractionFailureMessage(source: string): string {
|
|
143
|
+
return `_DOCX content could not be converted to markdown from ${source}. Ensure uvx and markitdown are installed._\n`;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async function extractDocx(
|
|
147
|
+
pi: ExtensionAPI,
|
|
148
|
+
source: string,
|
|
149
|
+
signal?: AbortSignal,
|
|
150
|
+
): Promise<string> {
|
|
151
|
+
const extracted = await extractWithMarkItDown(pi, source, signal);
|
|
152
|
+
return extracted || docxExtractionFailureMessage(source);
|
|
153
|
+
}
|
|
154
|
+
|
|
116
155
|
async function extractPdfUrl(
|
|
117
156
|
pi: ExtensionAPI,
|
|
118
157
|
url: string,
|
|
119
158
|
signal?: AbortSignal,
|
|
120
159
|
): Promise<ExtractedContent> {
|
|
121
160
|
const extracted = await extractPdf(pi, url, signal);
|
|
122
|
-
|
|
161
|
+
const failed = extracted.includes("could not be converted");
|
|
162
|
+
return {
|
|
163
|
+
extracted,
|
|
164
|
+
title: titleFromMarkdown(extracted),
|
|
165
|
+
extractor: "markitdown",
|
|
166
|
+
extraction_status: failed ? "failed" : "success",
|
|
167
|
+
content_type: "application/pdf",
|
|
168
|
+
};
|
|
123
169
|
}
|
|
124
170
|
|
|
125
171
|
async function extractTextUrl(
|
|
@@ -129,13 +175,30 @@ async function extractTextUrl(
|
|
|
129
175
|
): Promise<ExtractedContent> {
|
|
130
176
|
const markitdownExtracted = await extractWithMarkItDown(pi, url, signal);
|
|
131
177
|
if (markitdownExtracted) {
|
|
132
|
-
return {
|
|
178
|
+
return {
|
|
179
|
+
extracted: markitdownExtracted,
|
|
180
|
+
title: titleFromMarkdown(markitdownExtracted),
|
|
181
|
+
extractor: "markitdown",
|
|
182
|
+
extraction_status: "success",
|
|
183
|
+
};
|
|
133
184
|
}
|
|
134
185
|
|
|
135
186
|
const curlExtracted = await fetchTextUrl(pi, url, signal);
|
|
136
|
-
if (!curlExtracted) return { extracted: "" };
|
|
137
|
-
if (looksLikePdf(curlExtracted))
|
|
138
|
-
|
|
187
|
+
if (!curlExtracted) return { extracted: "", extractor: "none", extraction_status: "failed" };
|
|
188
|
+
if (looksLikePdf(curlExtracted)) {
|
|
189
|
+
return {
|
|
190
|
+
extracted: pdfExtractionFailureMessage(url),
|
|
191
|
+
extractor: "curl",
|
|
192
|
+
extraction_status: "failed",
|
|
193
|
+
content_type: "application/pdf",
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
extracted: curlExtracted,
|
|
198
|
+
title: titleFromHtml(curlExtracted),
|
|
199
|
+
extractor: "curl",
|
|
200
|
+
extraction_status: "success",
|
|
201
|
+
};
|
|
139
202
|
}
|
|
140
203
|
|
|
141
204
|
async function extractWithMarkItDown(
|
|
@@ -149,7 +212,7 @@ async function extractWithMarkItDown(
|
|
|
149
212
|
const mdResult = await exec(
|
|
150
213
|
pi,
|
|
151
214
|
"sh",
|
|
152
|
-
["-c", `uvx --from 'markitdown[pdf]' markitdown "${source}" 2>/dev/null || echo ""`],
|
|
215
|
+
["-c", `uvx --from 'markitdown[docx,pdf]' markitdown "${source}" 2>/dev/null || echo ""`],
|
|
153
216
|
{ signal, timeout: markitdownTimeoutMs() },
|
|
154
217
|
);
|
|
155
218
|
return mdResult.stdout.trim() ? mdResult.stdout : "";
|
|
@@ -106,9 +106,16 @@ function fileCaptureSource(
|
|
|
106
106
|
fallbackText: "",
|
|
107
107
|
preserveOriginal: (packetPath) =>
|
|
108
108
|
preserveFileOriginal(pi, packetPath, filePath, fileName, content, signal),
|
|
109
|
-
extract: async () =>
|
|
110
|
-
|
|
111
|
-
|
|
109
|
+
extract: async () => {
|
|
110
|
+
const extractedStr = await extractor.extract({ pi, filePath, content, signal });
|
|
111
|
+
const failed = extractedStr.includes("could not be converted");
|
|
112
|
+
return {
|
|
113
|
+
extracted: extractedStr,
|
|
114
|
+
extractor: extractor.extractorName ?? "passthrough",
|
|
115
|
+
extraction_status: (failed ? "failed" : "success") as "failed" | "success",
|
|
116
|
+
...(extractor.content_type ? { content_type: extractor.content_type } : {}),
|
|
117
|
+
};
|
|
118
|
+
},
|
|
112
119
|
manifest: () => ({
|
|
113
120
|
title: fileName,
|
|
114
121
|
file_path: filePath,
|
|
@@ -152,6 +159,9 @@ function finalizeCapture(
|
|
|
152
159
|
captured: fmtDate(),
|
|
153
160
|
packet_version: "1.0",
|
|
154
161
|
...source.manifest({ ...content, extracted }),
|
|
162
|
+
extractor: content.extractor ?? "passthrough",
|
|
163
|
+
extraction_status: content.extraction_status ?? "success",
|
|
164
|
+
...(content.content_type ? { content_type: content.content_type } : {}),
|
|
155
165
|
};
|
|
156
166
|
|
|
157
167
|
writeFileSync(join(packet.packetPath, "extracted.md"), extracted, "utf-8");
|
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
existsSync,
|
|
3
|
+
mkdirSync,
|
|
4
|
+
readFileSync,
|
|
5
|
+
readdirSync,
|
|
6
|
+
renameSync,
|
|
7
|
+
rmdirSync,
|
|
8
|
+
statSync,
|
|
9
|
+
writeFileSync,
|
|
10
|
+
} from "node:fs";
|
|
2
11
|
import { homedir } from "node:os";
|
|
3
12
|
import { dirname, join, resolve } from "node:path";
|
|
4
13
|
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
@@ -33,11 +42,24 @@ export function detectVaultFormat(dir: string): VaultFormat {
|
|
|
33
42
|
return "none";
|
|
34
43
|
}
|
|
35
44
|
|
|
36
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* Get the personal wiki root directory.
|
|
47
|
+
*
|
|
48
|
+
* The "root" follows the same contract as project wikis: it is the directory
|
|
49
|
+
* that *contains* the `.llm-wiki/` dot-dir, NOT the dot-dir itself.
|
|
50
|
+
* So the personal vault lives at `<root>/.llm-wiki/`.
|
|
51
|
+
*
|
|
52
|
+
* Default root: `homedir()` → personal vault at `~/.llm-wiki/`.
|
|
53
|
+
* Override: `WIKI_HOME` env var → personal vault at `$WIKI_HOME/.llm-wiki/`.
|
|
54
|
+
*
|
|
55
|
+
* NOTE: Previously this returned `~/.llm-wiki` (the dot-dir itself), which
|
|
56
|
+
* caused `getVaultPaths()` to compose paths like `~/.llm-wiki/.llm-wiki/raw`.
|
|
57
|
+
* See `migrateDoubledPersonalVault()` for the one-shot recovery.
|
|
58
|
+
*/
|
|
37
59
|
export function getPersonalWikiRoot(): string {
|
|
38
60
|
const envWiki = process.env.WIKI_HOME;
|
|
39
61
|
if (envWiki) return envWiki;
|
|
40
|
-
return
|
|
62
|
+
return homedir();
|
|
41
63
|
}
|
|
42
64
|
|
|
43
65
|
/** Get VaultPaths for the personal wiki. */
|
|
@@ -45,6 +67,54 @@ export function getPersonalWikiPaths(): VaultPaths {
|
|
|
45
67
|
return getVaultPaths(getPersonalWikiRoot());
|
|
46
68
|
}
|
|
47
69
|
|
|
70
|
+
/**
|
|
71
|
+
* One-shot, idempotent migration for vaults that were created with the broken
|
|
72
|
+
* `getPersonalWikiRoot()` (returned the dot-dir itself, so `getVaultPaths()`
|
|
73
|
+
* composed `<root>/.llm-wiki/.llm-wiki/...`).
|
|
74
|
+
*
|
|
75
|
+
* Detects a doubled layout at `<root>/.llm-wiki/.llm-wiki/config.json` and
|
|
76
|
+
* flattens it up by one level. Safe to call on every session start: if the
|
|
77
|
+
* doubled sentinel is absent, this is a no-op.
|
|
78
|
+
*
|
|
79
|
+
* Returns a description of the action taken (or `null` if no migration was
|
|
80
|
+
* needed) so callers can surface a one-line status message.
|
|
81
|
+
*/
|
|
82
|
+
export function migrateDoubledPersonalVault(
|
|
83
|
+
parentRoot: string = getPersonalWikiRoot(),
|
|
84
|
+
): { moved: string[]; from: string; to: string; skipped: string[] } | null {
|
|
85
|
+
const outerDotWiki = join(parentRoot, ".llm-wiki");
|
|
86
|
+
const innerDotWiki = join(outerDotWiki, ".llm-wiki");
|
|
87
|
+
const innerSentinel = join(innerDotWiki, "config.json");
|
|
88
|
+
|
|
89
|
+
if (!existsSync(innerSentinel)) return null;
|
|
90
|
+
|
|
91
|
+
const moved: string[] = [];
|
|
92
|
+
const skipped: string[] = [];
|
|
93
|
+
|
|
94
|
+
for (const entry of readdirSync(innerDotWiki)) {
|
|
95
|
+
const src = join(innerDotWiki, entry);
|
|
96
|
+
const dest = join(outerDotWiki, entry);
|
|
97
|
+
if (existsSync(dest)) {
|
|
98
|
+
// Collision — leave the inner copy in place rather than clobber.
|
|
99
|
+
skipped.push(entry);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
renameSync(src, dest);
|
|
103
|
+
moved.push(entry);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Only remove the inner dir if it is fully drained.
|
|
107
|
+
if (skipped.length === 0) {
|
|
108
|
+
try {
|
|
109
|
+
rmdirSync(innerDotWiki);
|
|
110
|
+
} catch {
|
|
111
|
+
// Leave behind if something raced us; harmless.
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return { moved, from: innerDotWiki, to: outerDotWiki, skipped };
|
|
116
|
+
}
|
|
117
|
+
|
|
48
118
|
/**
|
|
49
119
|
* Check if a vault is the personal wiki location.
|
|
50
120
|
* Used in layered recall to avoid double-counting.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zosmaai/pi-llm-wiki",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.1",
|
|
4
4
|
"description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi",
|