llm-wiki-compiler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +150 -0
- package/dist/cli.js +1415 -0
- package/dist/cli.js.map +1 -0
- package/package.json +61 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,1415 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/cli.ts
|
|
4
|
+
import "dotenv/config";
|
|
5
|
+
import { createRequire } from "module";
|
|
6
|
+
import { Command } from "commander";
|
|
7
|
+
|
|
8
|
+
// src/commands/ingest.ts
|
|
9
|
+
import path3 from "path";
|
|
10
|
+
import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
|
|
11
|
+
|
|
12
|
+
// src/utils/markdown.ts
|
|
13
|
+
import { writeFile, rename, readFile, mkdir } from "fs/promises";
|
|
14
|
+
import path from "path";
|
|
15
|
+
import yaml from "js-yaml";
|
|
16
|
+
function slugify(title) {
|
|
17
|
+
return title.toLowerCase().replace(/['']/g, "").replace(/[^\w\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
18
|
+
}
|
|
19
|
+
function buildFrontmatter(fields) {
|
|
20
|
+
const dumped = yaml.dump(fields, { lineWidth: -1, quotingType: '"' }).trimEnd();
|
|
21
|
+
return `---
|
|
22
|
+
${dumped}
|
|
23
|
+
---`;
|
|
24
|
+
}
|
|
25
|
+
function parseFrontmatter(content) {
|
|
26
|
+
const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
|
27
|
+
if (!match) {
|
|
28
|
+
return { meta: {}, body: content };
|
|
29
|
+
}
|
|
30
|
+
let meta = {};
|
|
31
|
+
try {
|
|
32
|
+
const parsed = yaml.load(match[1]);
|
|
33
|
+
if (parsed && typeof parsed === "object") {
|
|
34
|
+
meta = parsed;
|
|
35
|
+
}
|
|
36
|
+
} catch {
|
|
37
|
+
}
|
|
38
|
+
return { meta, body: match[2] };
|
|
39
|
+
}
|
|
40
|
+
async function atomicWrite(filePath, content) {
|
|
41
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
42
|
+
const tmpPath = filePath + ".tmp";
|
|
43
|
+
await writeFile(tmpPath, content, "utf-8");
|
|
44
|
+
await rename(tmpPath, filePath);
|
|
45
|
+
}
|
|
46
|
+
async function safeReadFile(filePath) {
|
|
47
|
+
try {
|
|
48
|
+
return await readFile(filePath, "utf-8");
|
|
49
|
+
} catch {
|
|
50
|
+
return "";
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function validateWikiPage(content) {
|
|
54
|
+
if (!content || content.trim().length === 0) return false;
|
|
55
|
+
const { meta, body } = parseFrontmatter(content);
|
|
56
|
+
if (!meta.title) return false;
|
|
57
|
+
if (body.trim().length === 0) return false;
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// src/utils/constants.ts
|
|
62
|
+
var MAX_SOURCE_CHARS = 1e5;
|
|
63
|
+
var MIN_SOURCE_CHARS = 50;
|
|
64
|
+
var QUERY_PAGE_LIMIT = 5;
|
|
65
|
+
var COMPILE_CONCURRENCY = 5;
|
|
66
|
+
var RETRY_COUNT = 3;
|
|
67
|
+
var RETRY_BASE_MS = 1e3;
|
|
68
|
+
var RETRY_MULTIPLIER = 4;
|
|
69
|
+
var MODEL = "claude-sonnet-4-20250514";
|
|
70
|
+
var SOURCES_DIR = "sources";
|
|
71
|
+
var CONCEPTS_DIR = "wiki/concepts";
|
|
72
|
+
var QUERIES_DIR = "wiki/queries";
|
|
73
|
+
var LLMWIKI_DIR = ".llmwiki";
|
|
74
|
+
var STATE_FILE = ".llmwiki/state.json";
|
|
75
|
+
var LOCK_FILE = ".llmwiki/lock";
|
|
76
|
+
var INDEX_FILE = "wiki/index.md";
|
|
77
|
+
|
|
78
|
+
// src/utils/output.ts
|
|
79
|
+
var RESET = "\x1B[0m";
|
|
80
|
+
var BOLD = "\x1B[1m";
|
|
81
|
+
var DIM = "\x1B[2m";
|
|
82
|
+
var GREEN = "\x1B[32m";
|
|
83
|
+
var YELLOW = "\x1B[33m";
|
|
84
|
+
var BLUE = "\x1B[34m";
|
|
85
|
+
var MAGENTA = "\x1B[35m";
|
|
86
|
+
var CYAN = "\x1B[36m";
|
|
87
|
+
var RED = "\x1B[31m";
|
|
88
|
+
function bold(text) {
|
|
89
|
+
return `${BOLD}${text}${RESET}`;
|
|
90
|
+
}
|
|
91
|
+
function dim(text) {
|
|
92
|
+
return `${DIM}${text}${RESET}`;
|
|
93
|
+
}
|
|
94
|
+
function success(text) {
|
|
95
|
+
return `${GREEN}${text}${RESET}`;
|
|
96
|
+
}
|
|
97
|
+
function warn(text) {
|
|
98
|
+
return `${YELLOW}${text}${RESET}`;
|
|
99
|
+
}
|
|
100
|
+
function info(text) {
|
|
101
|
+
return `${BLUE}${text}${RESET}`;
|
|
102
|
+
}
|
|
103
|
+
function error(text) {
|
|
104
|
+
return `${RED}${text}${RESET}`;
|
|
105
|
+
}
|
|
106
|
+
function concept(text) {
|
|
107
|
+
return `${MAGENTA}${BOLD}${text}${RESET}`;
|
|
108
|
+
}
|
|
109
|
+
function source(text) {
|
|
110
|
+
return `${CYAN}${text}${RESET}`;
|
|
111
|
+
}
|
|
112
|
+
function status(icon, message) {
|
|
113
|
+
console.log(`${icon} ${message}`);
|
|
114
|
+
}
|
|
115
|
+
function header(title) {
|
|
116
|
+
console.log(`
|
|
117
|
+
${BOLD}${title}${RESET}`);
|
|
118
|
+
console.log(dim("\u2500".repeat(Math.min(title.length + 4, 60))));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// src/ingest/web.ts
|
|
122
|
+
import { JSDOM } from "jsdom";
|
|
123
|
+
import { Readability } from "@mozilla/readability";
|
|
124
|
+
import TurndownService from "turndown";
|
|
125
|
+
async function fetchAndParse(url) {
|
|
126
|
+
const response = await fetch(url);
|
|
127
|
+
if (!response.ok) {
|
|
128
|
+
throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
|
|
129
|
+
}
|
|
130
|
+
return response;
|
|
131
|
+
}
|
|
132
|
+
function extractReadableContent(html, url) {
|
|
133
|
+
const dom = new JSDOM(html, { url });
|
|
134
|
+
const reader = new Readability(dom.window.document);
|
|
135
|
+
const article = reader.parse();
|
|
136
|
+
if (!article || !article.content) {
|
|
137
|
+
throw new Error(`Could not extract readable content from ${url}`);
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
title: article.title || "Untitled",
|
|
141
|
+
htmlContent: article.content
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
function convertToMarkdown(html) {
|
|
145
|
+
const turndown = new TurndownService({ headingStyle: "atx" });
|
|
146
|
+
return turndown.turndown(html);
|
|
147
|
+
}
|
|
148
|
+
async function ingestWeb(url) {
|
|
149
|
+
const response = await fetchAndParse(url);
|
|
150
|
+
const html = await response.text();
|
|
151
|
+
const { title, htmlContent } = extractReadableContent(html, url);
|
|
152
|
+
const content = convertToMarkdown(htmlContent);
|
|
153
|
+
return { title, content };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// src/ingest/file.ts
|
|
157
|
+
import { readFile as readFile2 } from "fs/promises";
|
|
158
|
+
import path2 from "path";
|
|
159
|
+
var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".txt"]);
|
|
160
|
+
function titleFromFilename(filePath) {
|
|
161
|
+
const basename = path2.basename(filePath, path2.extname(filePath));
|
|
162
|
+
return basename.replace(/[-_]+/g, " ").trim();
|
|
163
|
+
}
|
|
164
|
+
function wrapPlainText(text) {
|
|
165
|
+
return `\`\`\`
|
|
166
|
+
${text}
|
|
167
|
+
\`\`\``;
|
|
168
|
+
}
|
|
169
|
+
async function ingestFile(filePath) {
|
|
170
|
+
const ext = path2.extname(filePath).toLowerCase();
|
|
171
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
172
|
+
throw new Error(
|
|
173
|
+
`Unsupported file type "${ext}". Only .md and .txt files are supported.`
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
const raw = await readFile2(filePath, "utf-8");
|
|
177
|
+
const title = titleFromFilename(filePath);
|
|
178
|
+
const content = ext === ".md" ? raw : wrapPlainText(raw);
|
|
179
|
+
return { title, content };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// src/commands/ingest.ts
|
|
183
|
+
function isUrl(source2) {
|
|
184
|
+
return source2.startsWith("http://") || source2.startsWith("https://");
|
|
185
|
+
}
|
|
186
|
+
function enforceCharLimit(content) {
|
|
187
|
+
if (content.length <= MAX_SOURCE_CHARS) {
|
|
188
|
+
return { content, truncated: false, originalChars: content.length };
|
|
189
|
+
}
|
|
190
|
+
status(
|
|
191
|
+
"!",
|
|
192
|
+
warn(
|
|
193
|
+
`Content truncated from ${content.length.toLocaleString()} to ${MAX_SOURCE_CHARS.toLocaleString()} characters.`
|
|
194
|
+
)
|
|
195
|
+
);
|
|
196
|
+
return {
|
|
197
|
+
content: content.slice(0, MAX_SOURCE_CHARS),
|
|
198
|
+
truncated: true,
|
|
199
|
+
originalChars: content.length
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
function enforceMinContent(content) {
|
|
203
|
+
const length = content.trim().length;
|
|
204
|
+
if (length === 0) {
|
|
205
|
+
throw new Error(
|
|
206
|
+
"No readable content could be extracted from the source."
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
if (length < MIN_SOURCE_CHARS) {
|
|
210
|
+
status(
|
|
211
|
+
"!",
|
|
212
|
+
warn(
|
|
213
|
+
`Content seems very short (${length} chars, minimum recommended is ${MIN_SOURCE_CHARS}).`
|
|
214
|
+
)
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
function buildDocument(title, source2, result) {
|
|
219
|
+
const meta = {
|
|
220
|
+
title,
|
|
221
|
+
source: source2,
|
|
222
|
+
ingestedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
223
|
+
};
|
|
224
|
+
if (result.truncated) {
|
|
225
|
+
meta.truncated = true;
|
|
226
|
+
meta.originalChars = result.originalChars;
|
|
227
|
+
}
|
|
228
|
+
const frontmatter = buildFrontmatter(meta);
|
|
229
|
+
return `${frontmatter}
|
|
230
|
+
|
|
231
|
+
${result.content}
|
|
232
|
+
`;
|
|
233
|
+
}
|
|
234
|
+
async function saveSource(title, document) {
|
|
235
|
+
const filename = `${slugify(title)}.md`;
|
|
236
|
+
const destPath = path3.join(SOURCES_DIR, filename);
|
|
237
|
+
await mkdir2(SOURCES_DIR, { recursive: true });
|
|
238
|
+
await writeFile2(destPath, document, "utf-8");
|
|
239
|
+
return destPath;
|
|
240
|
+
}
|
|
241
|
+
async function ingest(source2) {
|
|
242
|
+
status("*", info(`Ingesting: ${source2}`));
|
|
243
|
+
const { title, content } = isUrl(source2) ? await ingestWeb(source2) : await ingestFile(source2);
|
|
244
|
+
const result = enforceCharLimit(content);
|
|
245
|
+
enforceMinContent(result.content);
|
|
246
|
+
const document = buildDocument(title, source2, result);
|
|
247
|
+
const savedPath = await saveSource(title, document);
|
|
248
|
+
status(
|
|
249
|
+
"+",
|
|
250
|
+
success(`Saved ${bold(title)} \u2192 ${source(savedPath)}`)
|
|
251
|
+
);
|
|
252
|
+
status("\u2192", dim("Next: llmwiki compile"));
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// src/commands/compile.ts
|
|
256
|
+
import { existsSync as existsSync3 } from "fs";
|
|
257
|
+
|
|
258
|
+
// src/compiler/index.ts
|
|
259
|
+
import { readFile as readFile7, readdir as readdir4 } from "fs/promises";
|
|
260
|
+
import path10 from "path";
|
|
261
|
+
|
|
262
|
+
// src/utils/state.ts
|
|
263
|
+
import { readFile as readFile3, writeFile as writeFile3, rename as rename2, mkdir as mkdir3, copyFile } from "fs/promises";
|
|
264
|
+
import { existsSync } from "fs";
|
|
265
|
+
import path4 from "path";
|
|
266
|
+
function emptyState() {
|
|
267
|
+
return { version: 1, indexHash: "", sources: {} };
|
|
268
|
+
}
|
|
269
|
+
async function readState(root) {
|
|
270
|
+
const filePath = path4.join(root, STATE_FILE);
|
|
271
|
+
if (!existsSync(filePath)) {
|
|
272
|
+
return emptyState();
|
|
273
|
+
}
|
|
274
|
+
try {
|
|
275
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
276
|
+
return JSON.parse(raw);
|
|
277
|
+
} catch {
|
|
278
|
+
const bakPath = filePath + ".bak";
|
|
279
|
+
console.warn(`\u26A0 Corrupt state.json \u2014 backed up to ${bakPath}, starting fresh.`);
|
|
280
|
+
await copyFile(filePath, bakPath);
|
|
281
|
+
return emptyState();
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
async function writeState(root, state) {
|
|
285
|
+
const dir = path4.join(root, LLMWIKI_DIR);
|
|
286
|
+
await mkdir3(dir, { recursive: true });
|
|
287
|
+
const filePath = path4.join(root, STATE_FILE);
|
|
288
|
+
const tmpPath = filePath + ".tmp";
|
|
289
|
+
await writeFile3(tmpPath, JSON.stringify(state, null, 2), "utf-8");
|
|
290
|
+
await rename2(tmpPath, filePath);
|
|
291
|
+
}
|
|
292
|
+
async function updateSourceState(root, sourceFile, entry) {
|
|
293
|
+
const state = await readState(root);
|
|
294
|
+
state.sources[sourceFile] = entry;
|
|
295
|
+
await writeState(root, state);
|
|
296
|
+
}
|
|
297
|
+
async function removeSourceState(root, sourceFile) {
|
|
298
|
+
const state = await readState(root);
|
|
299
|
+
delete state.sources[sourceFile];
|
|
300
|
+
await writeState(root, state);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// src/utils/llm.ts
|
|
304
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
305
|
+
var client = null;
|
|
306
|
+
function getClient() {
|
|
307
|
+
if (!client) {
|
|
308
|
+
client = new Anthropic();
|
|
309
|
+
}
|
|
310
|
+
return client;
|
|
311
|
+
}
|
|
312
|
+
function sleep(ms) {
|
|
313
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
314
|
+
}
|
|
315
|
+
async function callClaude(options) {
|
|
316
|
+
const { system, messages, tools, maxTokens = 4096, stream = false, onToken } = options;
|
|
317
|
+
const anthropic = getClient();
|
|
318
|
+
for (let attempt = 0; attempt <= RETRY_COUNT; attempt++) {
|
|
319
|
+
try {
|
|
320
|
+
if (stream) {
|
|
321
|
+
return await callClaudeStreaming(anthropic, system, messages, maxTokens, onToken);
|
|
322
|
+
}
|
|
323
|
+
if (tools && tools.length > 0) {
|
|
324
|
+
return await callClaudeToolUse(anthropic, system, messages, tools, maxTokens);
|
|
325
|
+
}
|
|
326
|
+
return await callClaudeBasic(anthropic, system, messages, maxTokens);
|
|
327
|
+
} catch (error2) {
|
|
328
|
+
if (attempt === RETRY_COUNT) throw error2;
|
|
329
|
+
const delayMs = RETRY_BASE_MS * Math.pow(RETRY_MULTIPLIER, attempt);
|
|
330
|
+
const errMsg = error2 instanceof Error ? error2.message : String(error2);
|
|
331
|
+
console.warn(`\u26A0 API call failed (attempt ${attempt + 1}/${RETRY_COUNT + 1}): ${errMsg}`);
|
|
332
|
+
console.warn(` Retrying in ${delayMs / 1e3}s...`);
|
|
333
|
+
await sleep(delayMs);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
throw new Error("Unreachable");
|
|
337
|
+
}
|
|
338
|
+
async function callClaudeStreaming(anthropic, system, messages, maxTokens, onToken) {
|
|
339
|
+
const stream = anthropic.messages.stream({
|
|
340
|
+
model: MODEL,
|
|
341
|
+
max_tokens: maxTokens,
|
|
342
|
+
system,
|
|
343
|
+
messages
|
|
344
|
+
});
|
|
345
|
+
let fullText = "";
|
|
346
|
+
for await (const event of stream) {
|
|
347
|
+
if (event.type === "content_block_delta" && event.delta.type === "text_delta") {
|
|
348
|
+
fullText += event.delta.text;
|
|
349
|
+
onToken?.(event.delta.text);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return fullText;
|
|
353
|
+
}
|
|
354
|
+
async function callClaudeToolUse(anthropic, system, messages, tools, maxTokens) {
|
|
355
|
+
const response = await anthropic.messages.create({
|
|
356
|
+
model: MODEL,
|
|
357
|
+
max_tokens: maxTokens,
|
|
358
|
+
system,
|
|
359
|
+
messages,
|
|
360
|
+
tools
|
|
361
|
+
});
|
|
362
|
+
const toolBlock = response.content.find((block) => block.type === "tool_use");
|
|
363
|
+
if (toolBlock && toolBlock.type === "tool_use") {
|
|
364
|
+
return JSON.stringify(toolBlock.input);
|
|
365
|
+
}
|
|
366
|
+
const textBlock = response.content.find((block) => block.type === "text");
|
|
367
|
+
if (textBlock && textBlock.type === "text") {
|
|
368
|
+
return textBlock.text;
|
|
369
|
+
}
|
|
370
|
+
return "";
|
|
371
|
+
}
|
|
372
|
+
async function callClaudeBasic(anthropic, system, messages, maxTokens) {
|
|
373
|
+
const response = await anthropic.messages.create({
|
|
374
|
+
model: MODEL,
|
|
375
|
+
max_tokens: maxTokens,
|
|
376
|
+
system,
|
|
377
|
+
messages
|
|
378
|
+
});
|
|
379
|
+
const textBlock = response.content.find((block) => block.type === "text");
|
|
380
|
+
if (textBlock && textBlock.type === "text") {
|
|
381
|
+
return textBlock.text;
|
|
382
|
+
}
|
|
383
|
+
return "";
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// src/utils/lock.ts
|
|
387
|
+
import { open, readFile as readFile4, unlink, mkdir as mkdir4 } from "fs/promises";
|
|
388
|
+
import path5 from "path";
|
|
389
|
+
var RECLAIM_SUFFIX = ".reclaim";
|
|
390
|
+
var MAX_ACQUIRE_ATTEMPTS = 2;
|
|
391
|
+
function isProcessAlive(pid) {
|
|
392
|
+
try {
|
|
393
|
+
process.kill(pid, 0);
|
|
394
|
+
return true;
|
|
395
|
+
} catch {
|
|
396
|
+
return false;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
async function acquireLock(root) {
|
|
400
|
+
const lockPath = path5.join(root, LOCK_FILE);
|
|
401
|
+
await mkdir4(path5.join(root, LLMWIKI_DIR), { recursive: true });
|
|
402
|
+
for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt++) {
|
|
403
|
+
const created = await tryCreateLock(lockPath);
|
|
404
|
+
if (created) return true;
|
|
405
|
+
const stale = await isLockStale(lockPath);
|
|
406
|
+
if (!stale) {
|
|
407
|
+
status("!", warn("Another compilation is running."));
|
|
408
|
+
return false;
|
|
409
|
+
}
|
|
410
|
+
const reclaimed = await reclaimStaleLock(root, lockPath);
|
|
411
|
+
if (reclaimed) return true;
|
|
412
|
+
}
|
|
413
|
+
status("!", warn("Could not acquire lock after retrying."));
|
|
414
|
+
return false;
|
|
415
|
+
}
|
|
416
|
+
async function reclaimStaleLock(root, lockPath) {
|
|
417
|
+
const reclaimPath = lockPath + RECLAIM_SUFFIX;
|
|
418
|
+
const gotReclaimLock = await acquireReclaimLock(reclaimPath);
|
|
419
|
+
if (!gotReclaimLock) return false;
|
|
420
|
+
try {
|
|
421
|
+
if (!await isLockStale(lockPath)) {
|
|
422
|
+
return false;
|
|
423
|
+
}
|
|
424
|
+
try {
|
|
425
|
+
await unlink(lockPath);
|
|
426
|
+
} catch {
|
|
427
|
+
}
|
|
428
|
+
const acquired = await tryCreateLock(lockPath);
|
|
429
|
+
if (acquired) {
|
|
430
|
+
status("i", dim("Reclaimed stale lock from dead process."));
|
|
431
|
+
}
|
|
432
|
+
return acquired;
|
|
433
|
+
} finally {
|
|
434
|
+
try {
|
|
435
|
+
await unlink(reclaimPath);
|
|
436
|
+
} catch {
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
async function acquireReclaimLock(reclaimPath) {
|
|
441
|
+
if (await tryCreateLock(reclaimPath)) return true;
|
|
442
|
+
if (!await isLockStale(reclaimPath)) return false;
|
|
443
|
+
try {
|
|
444
|
+
await unlink(reclaimPath);
|
|
445
|
+
} catch {
|
|
446
|
+
}
|
|
447
|
+
return false;
|
|
448
|
+
}
|
|
449
|
+
async function tryCreateLock(lockPath) {
|
|
450
|
+
try {
|
|
451
|
+
const fd = await open(lockPath, "wx");
|
|
452
|
+
await fd.writeFile(String(process.pid), "utf-8");
|
|
453
|
+
await fd.close();
|
|
454
|
+
return true;
|
|
455
|
+
} catch (err) {
|
|
456
|
+
if (err instanceof Error && "code" in err && err.code === "EEXIST") {
|
|
457
|
+
return false;
|
|
458
|
+
}
|
|
459
|
+
throw err;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
async function isLockStale(lockPath) {
|
|
463
|
+
try {
|
|
464
|
+
const content = await readFile4(lockPath, "utf-8");
|
|
465
|
+
const pid = parseInt(content.trim(), 10);
|
|
466
|
+
if (isNaN(pid)) return true;
|
|
467
|
+
return !isProcessAlive(pid);
|
|
468
|
+
} catch {
|
|
469
|
+
return true;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
async function releaseLock(root) {
|
|
473
|
+
const lockPath = path5.join(root, LOCK_FILE);
|
|
474
|
+
try {
|
|
475
|
+
await unlink(lockPath);
|
|
476
|
+
} catch {
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// src/compiler/prompts.ts
|
|
481
|
+
var CONCEPT_EXTRACTION_TOOL = {
|
|
482
|
+
name: "extract_concepts",
|
|
483
|
+
description: "Extract knowledge concepts from a source document",
|
|
484
|
+
input_schema: {
|
|
485
|
+
type: "object",
|
|
486
|
+
properties: {
|
|
487
|
+
concepts: {
|
|
488
|
+
type: "array",
|
|
489
|
+
items: {
|
|
490
|
+
type: "object",
|
|
491
|
+
properties: {
|
|
492
|
+
concept: {
|
|
493
|
+
type: "string",
|
|
494
|
+
description: "Human-readable concept title"
|
|
495
|
+
},
|
|
496
|
+
summary: {
|
|
497
|
+
type: "string",
|
|
498
|
+
description: "One-line description"
|
|
499
|
+
},
|
|
500
|
+
is_new: {
|
|
501
|
+
type: "boolean",
|
|
502
|
+
description: "True if this is a new concept not in existing wiki"
|
|
503
|
+
}
|
|
504
|
+
},
|
|
505
|
+
required: ["concept", "summary", "is_new"]
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
},
|
|
509
|
+
required: ["concepts"]
|
|
510
|
+
}
|
|
511
|
+
};
|
|
512
|
+
function buildExtractionPrompt(sourceContent, existingIndex) {
|
|
513
|
+
const indexSection = existingIndex ? `
|
|
514
|
+
|
|
515
|
+
Here is the existing wiki index \u2014 avoid duplicating concepts already covered:
|
|
516
|
+
|
|
517
|
+
${existingIndex}` : "\n\nNo existing wiki pages yet.";
|
|
518
|
+
return [
|
|
519
|
+
"You are a knowledge extraction engine. Analyze the following source document",
|
|
520
|
+
"and identify 3-8 distinct, meaningful concepts worth documenting as wiki pages.",
|
|
521
|
+
"Each concept should be a standalone topic that someone might look up.",
|
|
522
|
+
"Focus on key ideas, techniques, patterns, or entities \u2014 not trivial details.",
|
|
523
|
+
"Use the extract_concepts tool to return your findings.",
|
|
524
|
+
indexSection,
|
|
525
|
+
"\n\n--- SOURCE DOCUMENT ---\n\n",
|
|
526
|
+
sourceContent
|
|
527
|
+
].join("\n");
|
|
528
|
+
}
|
|
529
|
+
function buildPagePrompt(concept2, sourceContent, existingPage, relatedPages) {
|
|
530
|
+
const existingSection = existingPage ? `
|
|
531
|
+
|
|
532
|
+
Existing page to update:
|
|
533
|
+
|
|
534
|
+
${existingPage}` : "";
|
|
535
|
+
const relatedSection = relatedPages ? `
|
|
536
|
+
|
|
537
|
+
Related wiki pages for cross-referencing:
|
|
538
|
+
|
|
539
|
+
${relatedPages}` : "";
|
|
540
|
+
return [
|
|
541
|
+
`You are a wiki author. Write a clear, well-structured markdown page about "${concept2}".`,
|
|
542
|
+
"Draw facts only from the provided source material.",
|
|
543
|
+
"Include a ## Sources section at the end listing the source document.",
|
|
544
|
+
"Suggest [[wikilinks]] to related concepts where appropriate.",
|
|
545
|
+
"Write in a neutral, informative tone. Be concise but thorough.",
|
|
546
|
+
existingSection,
|
|
547
|
+
relatedSection,
|
|
548
|
+
"\n\n--- SOURCE MATERIAL ---\n\n",
|
|
549
|
+
sourceContent
|
|
550
|
+
].join("\n");
|
|
551
|
+
}
|
|
552
|
+
function parseConcepts(toolOutput) {
|
|
553
|
+
try {
|
|
554
|
+
const parsed = JSON.parse(toolOutput);
|
|
555
|
+
const concepts = parsed.concepts ?? [];
|
|
556
|
+
return concepts.filter(
|
|
557
|
+
(c) => typeof c.concept === "string" && typeof c.summary === "string" && typeof c.is_new === "boolean"
|
|
558
|
+
);
|
|
559
|
+
} catch {
|
|
560
|
+
return [];
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// src/compiler/hasher.ts
|
|
565
|
+
import { createHash } from "crypto";
|
|
566
|
+
import { readFile as readFile5, readdir } from "fs/promises";
|
|
567
|
+
import path6 from "path";
|
|
568
|
+
async function hashFile(filePath) {
|
|
569
|
+
const content = await readFile5(filePath, "utf-8");
|
|
570
|
+
return createHash("sha256").update(content).digest("hex");
|
|
571
|
+
}
|
|
572
|
+
async function detectChanges(root, prevState) {
|
|
573
|
+
const sourcesPath = path6.join(root, SOURCES_DIR);
|
|
574
|
+
const currentFiles = await listSourceFiles(sourcesPath);
|
|
575
|
+
const changes = [];
|
|
576
|
+
for (const file of currentFiles) {
|
|
577
|
+
const status2 = await classifyFile(root, file, prevState);
|
|
578
|
+
changes.push({ file, status: status2 });
|
|
579
|
+
}
|
|
580
|
+
const deletedChanges = findDeletedFiles(currentFiles, prevState);
|
|
581
|
+
changes.push(...deletedChanges);
|
|
582
|
+
return changes;
|
|
583
|
+
}
|
|
584
|
+
async function listSourceFiles(sourcesPath) {
|
|
585
|
+
try {
|
|
586
|
+
const entries = await readdir(sourcesPath);
|
|
587
|
+
return entries.filter((f) => f.endsWith(".md"));
|
|
588
|
+
} catch {
|
|
589
|
+
return [];
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
async function classifyFile(root, file, prevState) {
|
|
593
|
+
const filePath = path6.join(root, SOURCES_DIR, file);
|
|
594
|
+
const hash = await hashFile(filePath);
|
|
595
|
+
const prev = prevState.sources[file];
|
|
596
|
+
if (!prev) return "new";
|
|
597
|
+
if (prev.hash !== hash) return "changed";
|
|
598
|
+
return "unchanged";
|
|
599
|
+
}
|
|
600
|
+
function findDeletedFiles(currentFiles, prevState) {
|
|
601
|
+
const currentSet = new Set(currentFiles);
|
|
602
|
+
return Object.keys(prevState.sources).filter((file) => !currentSet.has(file)).map((file) => ({ file, status: "deleted" }));
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// src/compiler/deps.ts
|
|
606
|
+
function buildConceptToSourcesMap(sources) {
|
|
607
|
+
const conceptMap = /* @__PURE__ */ new Map();
|
|
608
|
+
for (const [sourceFile, entry] of Object.entries(sources)) {
|
|
609
|
+
for (const slug of entry.concepts) {
|
|
610
|
+
const existing = conceptMap.get(slug);
|
|
611
|
+
if (existing) {
|
|
612
|
+
existing.push(sourceFile);
|
|
613
|
+
} else {
|
|
614
|
+
conceptMap.set(slug, [sourceFile]);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
return conceptMap;
|
|
619
|
+
}
|
|
620
|
+
function findAffectedSources(state, directChanges) {
|
|
621
|
+
const changedFiles = new Set(
|
|
622
|
+
directChanges.filter((c) => c.status === "new" || c.status === "changed").map((c) => c.file)
|
|
623
|
+
);
|
|
624
|
+
const deletedFiles = new Set(
|
|
625
|
+
directChanges.filter((c) => c.status === "deleted").map((c) => c.file)
|
|
626
|
+
);
|
|
627
|
+
const conceptMap = buildConceptToSourcesMap(state.sources);
|
|
628
|
+
const affected = /* @__PURE__ */ new Set();
|
|
629
|
+
for (const changedFile of changedFiles) {
|
|
630
|
+
const sourceEntry = state.sources[changedFile];
|
|
631
|
+
if (!sourceEntry) continue;
|
|
632
|
+
for (const slug of sourceEntry.concepts) {
|
|
633
|
+
const contributors = conceptMap.get(slug);
|
|
634
|
+
if (!contributors || contributors.length < 2) continue;
|
|
635
|
+
for (const contributor of contributors) {
|
|
636
|
+
const skip = changedFiles.has(contributor) || deletedFiles.has(contributor) || affected.has(contributor);
|
|
637
|
+
if (!skip) {
|
|
638
|
+
affected.add(contributor);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
return Array.from(affected);
|
|
644
|
+
}
|
|
645
|
+
function findFrozenSlugs(state, changes) {
|
|
646
|
+
const frozen = new Set(state.frozenSlugs ?? []);
|
|
647
|
+
const deletedFiles = changes.filter((c) => c.status === "deleted").map((c) => c.file);
|
|
648
|
+
const conceptMap = buildConceptToSourcesMap(state.sources);
|
|
649
|
+
for (const file of deletedFiles) {
|
|
650
|
+
const entry = state.sources[file];
|
|
651
|
+
if (!entry) continue;
|
|
652
|
+
for (const slug of entry.concepts) {
|
|
653
|
+
const contributors = conceptMap.get(slug);
|
|
654
|
+
if (contributors && contributors.length > 1) {
|
|
655
|
+
frozen.add(slug);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
return frozen;
|
|
660
|
+
}
|
|
661
|
+
async function persistFrozenSlugs(root, frozenSlugs, successfulExtractions) {
|
|
662
|
+
const currentState = await readState(root);
|
|
663
|
+
const conceptMap = buildConceptToSourcesMap(currentState.sources);
|
|
664
|
+
const extractedBy = /* @__PURE__ */ new Set();
|
|
665
|
+
for (const result of successfulExtractions) {
|
|
666
|
+
if (result.concepts.length === 0) continue;
|
|
667
|
+
for (const c of result.concepts) {
|
|
668
|
+
extractedBy.add(slugify(c.concept));
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
const compiledFiles = new Set(
|
|
672
|
+
successfulExtractions.filter((r) => r.concepts.length > 0).map((r) => r.sourceFile)
|
|
673
|
+
);
|
|
674
|
+
const remaining = /* @__PURE__ */ new Set();
|
|
675
|
+
for (const slug of frozenSlugs) {
|
|
676
|
+
const owners = conceptMap.get(slug) ?? [];
|
|
677
|
+
const allOwnersCompiled = owners.length > 0 && owners.every((f) => compiledFiles.has(f)) && extractedBy.has(slug);
|
|
678
|
+
if (!allOwnersCompiled) remaining.add(slug);
|
|
679
|
+
}
|
|
680
|
+
const stateToSave = { ...currentState, frozenSlugs: Array.from(remaining) };
|
|
681
|
+
await writeState(root, stateToSave);
|
|
682
|
+
}
|
|
683
|
+
function findLateAffectedSources(extractions, state, allChanges) {
|
|
684
|
+
const compilingFiles = new Set(
|
|
685
|
+
allChanges.filter((c) => c.status === "new" || c.status === "changed").map((c) => c.file)
|
|
686
|
+
);
|
|
687
|
+
const deletedFiles = new Set(
|
|
688
|
+
allChanges.filter((c) => c.status === "deleted").map((c) => c.file)
|
|
689
|
+
);
|
|
690
|
+
const conceptMap = buildConceptToSourcesMap(state.sources);
|
|
691
|
+
const freshSlugs = /* @__PURE__ */ new Set();
|
|
692
|
+
for (const result of extractions) {
|
|
693
|
+
const oldConcepts = new Set(state.sources[result.sourceFile]?.concepts ?? []);
|
|
694
|
+
for (const c of result.concepts) {
|
|
695
|
+
const slug = slugify(c.concept);
|
|
696
|
+
if (!oldConcepts.has(slug)) {
|
|
697
|
+
freshSlugs.add(slug);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
const affected = /* @__PURE__ */ new Set();
|
|
702
|
+
for (const slug of freshSlugs) {
|
|
703
|
+
const owners = conceptMap.get(slug);
|
|
704
|
+
if (!owners) continue;
|
|
705
|
+
for (const owner of owners) {
|
|
706
|
+
if (!compilingFiles.has(owner) && !deletedFiles.has(owner)) {
|
|
707
|
+
affected.add(owner);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
return Array.from(affected);
|
|
712
|
+
}
|
|
713
|
+
function findSharedConcepts(sourceFile, state) {
|
|
714
|
+
const shared = /* @__PURE__ */ new Set();
|
|
715
|
+
const sourceEntry = state.sources[sourceFile];
|
|
716
|
+
if (!sourceEntry) return shared;
|
|
717
|
+
const conceptMap = buildConceptToSourcesMap(state.sources);
|
|
718
|
+
for (const slug of sourceEntry.concepts) {
|
|
719
|
+
const contributors = conceptMap.get(slug);
|
|
720
|
+
if (contributors && contributors.length > 1) {
|
|
721
|
+
shared.add(slug);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
return shared;
|
|
725
|
+
}
|
|
726
|
+
async function freezeFailedExtractions(root, results, frozenSlugs) {
|
|
727
|
+
for (const result of results) {
|
|
728
|
+
if (result.concepts.length > 0) continue;
|
|
729
|
+
status("!", warn(`${result.sourceFile}: no concepts \u2014 will retry.`));
|
|
730
|
+
const currentState = await readState(root);
|
|
731
|
+
const oldConcepts = currentState.sources[result.sourceFile]?.concepts ?? [];
|
|
732
|
+
for (const slug of oldConcepts) frozenSlugs.add(slug);
|
|
733
|
+
await updateSourceState(root, result.sourceFile, {
|
|
734
|
+
hash: "",
|
|
735
|
+
concepts: oldConcepts,
|
|
736
|
+
compiledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
737
|
+
});
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// src/compiler/orphan.ts
|
|
742
|
+
import path7 from "path";
|
|
743
|
+
async function markOrphaned(root, sourceFile, state) {
|
|
744
|
+
const sourceEntry = state.sources[sourceFile];
|
|
745
|
+
if (!sourceEntry) return;
|
|
746
|
+
const sharedSlugs = findSharedConcepts(sourceFile, state);
|
|
747
|
+
for (const slug of sourceEntry.concepts) {
|
|
748
|
+
if (sharedSlugs.has(slug)) {
|
|
749
|
+
status("i", dim(`Kept: ${slug}.md (shared with other sources)`));
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
752
|
+
await orphanPage(root, slug, "source deleted");
|
|
753
|
+
}
|
|
754
|
+
await removeSourceState(root, sourceFile);
|
|
755
|
+
}
|
|
756
|
+
async function orphanUnownedFrozenPages(root, frozenSlugs) {
|
|
757
|
+
const currentState = await readState(root);
|
|
758
|
+
const ownedSlugs = /* @__PURE__ */ new Set();
|
|
759
|
+
for (const entry of Object.values(currentState.sources)) {
|
|
760
|
+
for (const slug of entry.concepts) ownedSlugs.add(slug);
|
|
761
|
+
}
|
|
762
|
+
for (const slug of frozenSlugs) {
|
|
763
|
+
if (ownedSlugs.has(slug)) continue;
|
|
764
|
+
await orphanPage(root, slug, "no remaining sources");
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
async function orphanPage(root, slug, reason) {
|
|
768
|
+
const pagePath = path7.join(root, CONCEPTS_DIR, `${slug}.md`);
|
|
769
|
+
const content = await safeReadFile(pagePath);
|
|
770
|
+
if (!content) return;
|
|
771
|
+
const { meta } = parseFrontmatter(content);
|
|
772
|
+
if (meta.orphaned === true) return;
|
|
773
|
+
const updated = content.replace("---\n", "---\norphaned: true\n");
|
|
774
|
+
await atomicWrite(pagePath, updated);
|
|
775
|
+
status("\u26A0", warn(`Orphaned: ${slug}.md (${reason})`));
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
// src/compiler/resolver.ts
|
|
779
|
+
import { readdir as readdir2, readFile as readFile6 } from "fs/promises";
|
|
780
|
+
import path8 from "path";
|
|
781
|
+
import { existsSync as existsSync2 } from "fs";
|
|
782
|
+
async function buildTitleIndex(root) {
|
|
783
|
+
const conceptsDir = path8.join(root, CONCEPTS_DIR);
|
|
784
|
+
if (!existsSync2(conceptsDir)) return [];
|
|
785
|
+
const files = await readdir2(conceptsDir);
|
|
786
|
+
const pages = [];
|
|
787
|
+
for (const file of files) {
|
|
788
|
+
if (!file.endsWith(".md")) continue;
|
|
789
|
+
const filePath = path8.join(conceptsDir, file);
|
|
790
|
+
const content = await readFile6(filePath, "utf-8");
|
|
791
|
+
const { meta } = parseFrontmatter(content);
|
|
792
|
+
if (meta.title && typeof meta.title === "string" && !meta.orphaned) {
|
|
793
|
+
pages.push({
|
|
794
|
+
slug: file.replace(/\.md$/, ""),
|
|
795
|
+
title: meta.title,
|
|
796
|
+
filePath
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
return pages;
|
|
801
|
+
}
|
|
802
|
+
function isInsideWikilink(text, position) {
|
|
803
|
+
const before = text.lastIndexOf("[[", position);
|
|
804
|
+
const after = text.indexOf("]]", position);
|
|
805
|
+
if (before === -1 || after === -1) return false;
|
|
806
|
+
const closeBefore = text.indexOf("]]", before);
|
|
807
|
+
return closeBefore >= position;
|
|
808
|
+
}
|
|
809
|
+
function isWordBoundary(text, start, end) {
|
|
810
|
+
const before = start === 0 || /[\s,.:;!?()\[\]{}/"']/.test(text[start - 1]);
|
|
811
|
+
const after = end >= text.length || /[\s,.:;!?()\[\]{}/"']/.test(text[end]);
|
|
812
|
+
return before && after;
|
|
813
|
+
}
|
|
814
|
+
function addWikilinks(body, titles, selfTitle) {
|
|
815
|
+
let result = body;
|
|
816
|
+
for (const page of titles) {
|
|
817
|
+
if (page.title.toLowerCase() === selfTitle.toLowerCase()) continue;
|
|
818
|
+
const escaped = page.title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
819
|
+
const regex = new RegExp(escaped, "gi");
|
|
820
|
+
let match;
|
|
821
|
+
const matches = [];
|
|
822
|
+
while ((match = regex.exec(result)) !== null) {
|
|
823
|
+
matches.push({ start: match.index, end: match.index + match[0].length });
|
|
824
|
+
}
|
|
825
|
+
for (const m of matches.reverse()) {
|
|
826
|
+
if (isInsideWikilink(result, m.start)) continue;
|
|
827
|
+
if (!isWordBoundary(result, m.start, m.end)) continue;
|
|
828
|
+
result = result.slice(0, m.start) + `[[${page.title}]]` + result.slice(m.end);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
return result;
|
|
832
|
+
}
|
|
833
|
+
async function resolveLinks(root, changedSlugs, newSlugs) {
|
|
834
|
+
const titleIndex = await buildTitleIndex(root);
|
|
835
|
+
if (titleIndex.length === 0) return 0;
|
|
836
|
+
let linkCount = 0;
|
|
837
|
+
linkCount += await resolveOutboundLinks(titleIndex, changedSlugs);
|
|
838
|
+
linkCount += await resolveInboundLinks(titleIndex, newSlugs);
|
|
839
|
+
if (linkCount > 0) {
|
|
840
|
+
status("\u{1F517}", dim(`Resolved links in ${linkCount} page(s)`));
|
|
841
|
+
}
|
|
842
|
+
return linkCount;
|
|
843
|
+
}
|
|
844
|
+
async function resolveOutboundLinks(titleIndex, changedSlugs) {
|
|
845
|
+
let count = 0;
|
|
846
|
+
for (const page of titleIndex) {
|
|
847
|
+
if (!changedSlugs.includes(page.slug)) continue;
|
|
848
|
+
const didLink = await linkPage(page, titleIndex);
|
|
849
|
+
if (didLink) count++;
|
|
850
|
+
}
|
|
851
|
+
return count;
|
|
852
|
+
}
|
|
853
|
+
async function resolveInboundLinks(titleIndex, newSlugs) {
|
|
854
|
+
if (newSlugs.length === 0) return 0;
|
|
855
|
+
const newTitles = titleIndex.filter((p) => newSlugs.includes(p.slug));
|
|
856
|
+
if (newTitles.length === 0) return 0;
|
|
857
|
+
let count = 0;
|
|
858
|
+
for (const page of titleIndex) {
|
|
859
|
+
if (newSlugs.includes(page.slug)) continue;
|
|
860
|
+
const content = await readFile6(page.filePath, "utf-8");
|
|
861
|
+
const { body } = parseFrontmatter(content);
|
|
862
|
+
const linked = addWikilinks(body, newTitles, page.title);
|
|
863
|
+
if (linked !== body) {
|
|
864
|
+
const newContent = content.replace(body, linked);
|
|
865
|
+
await atomicWrite(page.filePath, newContent);
|
|
866
|
+
count++;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
return count;
|
|
870
|
+
}
|
|
871
|
+
async function linkPage(page, titleIndex) {
|
|
872
|
+
const content = await readFile6(page.filePath, "utf-8");
|
|
873
|
+
const { body } = parseFrontmatter(content);
|
|
874
|
+
const linked = addWikilinks(body, titleIndex, page.title);
|
|
875
|
+
if (linked === body) return false;
|
|
876
|
+
const newContent = content.replace(body, linked);
|
|
877
|
+
await atomicWrite(page.filePath, newContent);
|
|
878
|
+
return true;
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// src/compiler/indexgen.ts
|
|
882
|
+
import { readdir as readdir3 } from "fs/promises";
|
|
883
|
+
import path9 from "path";
|
|
884
|
+
async function generateIndex(root) {
|
|
885
|
+
status("*", info("Generating index..."));
|
|
886
|
+
const conceptsPath = path9.join(root, CONCEPTS_DIR);
|
|
887
|
+
const queriesPath = path9.join(root, QUERIES_DIR);
|
|
888
|
+
const concepts = await collectPageSummaries(conceptsPath);
|
|
889
|
+
const queries = await collectPageSummaries(queriesPath);
|
|
890
|
+
concepts.sort((a, b) => a.title.localeCompare(b.title));
|
|
891
|
+
queries.sort((a, b) => a.title.localeCompare(b.title));
|
|
892
|
+
const indexContent = buildIndexContent(concepts, queries);
|
|
893
|
+
const indexPath = path9.join(root, INDEX_FILE);
|
|
894
|
+
await atomicWrite(indexPath, indexContent);
|
|
895
|
+
const total = concepts.length + queries.length;
|
|
896
|
+
status("+", success(`Index updated with ${total} pages.`));
|
|
897
|
+
}
|
|
898
|
+
async function collectPageSummaries(conceptsPath) {
|
|
899
|
+
let files;
|
|
900
|
+
try {
|
|
901
|
+
files = await readdir3(conceptsPath);
|
|
902
|
+
} catch {
|
|
903
|
+
return [];
|
|
904
|
+
}
|
|
905
|
+
const pages = [];
|
|
906
|
+
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
907
|
+
const content = await safeReadFile(path9.join(conceptsPath, file));
|
|
908
|
+
const { meta } = parseFrontmatter(content);
|
|
909
|
+
if (meta.title && typeof meta.title === "string" && !meta.orphaned) {
|
|
910
|
+
pages.push({
|
|
911
|
+
title: meta.title,
|
|
912
|
+
slug: file.replace(/\.md$/, ""),
|
|
913
|
+
summary: typeof meta.summary === "string" ? meta.summary : ""
|
|
914
|
+
});
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
return pages;
|
|
918
|
+
}
|
|
919
|
+
function stripWikilinks(text) {
|
|
920
|
+
return text.replace(/\[\[([^\]]+)\]\]/g, "$1");
|
|
921
|
+
}
|
|
922
|
+
function buildIndexContent(concepts, queries) {
|
|
923
|
+
const lines = ["# Knowledge Wiki", "", "## Concepts", ""];
|
|
924
|
+
for (const page of concepts) {
|
|
925
|
+
lines.push(`- **[[${page.title}]]** \u2014 ${stripWikilinks(page.summary)}`);
|
|
926
|
+
}
|
|
927
|
+
if (queries.length > 0) {
|
|
928
|
+
lines.push("", "## Saved Queries", "");
|
|
929
|
+
for (const page of queries) {
|
|
930
|
+
lines.push(`- **[[${page.title}]]** \u2014 ${stripWikilinks(page.summary)}`);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
const total = concepts.length + queries.length;
|
|
934
|
+
lines.push("");
|
|
935
|
+
lines.push(`_${total} pages | Generated ${(/* @__PURE__ */ new Date()).toISOString()}_`);
|
|
936
|
+
lines.push("");
|
|
937
|
+
return lines.join("\n");
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
// src/compiler/index.ts
|
|
941
|
+
import pLimit from "p-limit";
|
|
942
|
+
async function compile(root) {
|
|
943
|
+
header("llmwiki compile");
|
|
944
|
+
const locked = await acquireLock(root);
|
|
945
|
+
if (!locked) {
|
|
946
|
+
status("!", error("Could not acquire lock. Try again later."));
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
try {
|
|
950
|
+
await runCompilePipeline(root);
|
|
951
|
+
} finally {
|
|
952
|
+
await releaseLock(root);
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
async function runCompilePipeline(root) {
|
|
956
|
+
const state = await readState(root);
|
|
957
|
+
const changes = await detectChanges(root, state);
|
|
958
|
+
const affectedFiles = findAffectedSources(state, changes);
|
|
959
|
+
for (const file of affectedFiles) {
|
|
960
|
+
status("~", info(`${file} [affected by shared concept]`));
|
|
961
|
+
changes.push({ file, status: "changed" });
|
|
962
|
+
}
|
|
963
|
+
const toCompile = changes.filter((c) => c.status === "new" || c.status === "changed");
|
|
964
|
+
const deleted = changes.filter((c) => c.status === "deleted");
|
|
965
|
+
const unchanged = changes.filter((c) => c.status === "unchanged");
|
|
966
|
+
if (toCompile.length === 0 && deleted.length === 0) {
|
|
967
|
+
status("\u2713", success("Nothing to compile \u2014 all sources up to date."));
|
|
968
|
+
return;
|
|
969
|
+
}
|
|
970
|
+
printChangesSummary(changes);
|
|
971
|
+
for (const del of deleted) {
|
|
972
|
+
await markOrphaned(root, del.file, state);
|
|
973
|
+
}
|
|
974
|
+
const frozenSlugs = findFrozenSlugs(state, changes);
|
|
975
|
+
for (const slug of frozenSlugs) {
|
|
976
|
+
status("i", dim(`Frozen: ${slug} (shared with deleted source)`));
|
|
977
|
+
}
|
|
978
|
+
const extractions = [];
|
|
979
|
+
for (const change of toCompile) {
|
|
980
|
+
extractions.push(await extractForSource(root, change.file));
|
|
981
|
+
}
|
|
982
|
+
const lateAffected = findLateAffectedSources(extractions, state, changes);
|
|
983
|
+
for (const file of lateAffected) {
|
|
984
|
+
status("~", info(`${file} [shares concept with new source]`));
|
|
985
|
+
extractions.push(await extractForSource(root, file));
|
|
986
|
+
}
|
|
987
|
+
await freezeFailedExtractions(root, extractions, frozenSlugs);
|
|
988
|
+
const merged = mergeExtractions(extractions, frozenSlugs);
|
|
989
|
+
const limit = pLimit(COMPILE_CONCURRENCY);
|
|
990
|
+
const pageResults = await Promise.all(
|
|
991
|
+
merged.map((entry) => limit(async () => {
|
|
992
|
+
await generateMergedPage(root, entry);
|
|
993
|
+
return entry;
|
|
994
|
+
}))
|
|
995
|
+
);
|
|
996
|
+
const allChangedSlugs = pageResults.map((e) => e.slug);
|
|
997
|
+
const allNewSlugs = pageResults.filter((e) => e.concept.is_new).map((e) => e.slug);
|
|
998
|
+
for (const result of extractions) {
|
|
999
|
+
if (result.concepts.length === 0) continue;
|
|
1000
|
+
await persistSourceState(root, result.sourcePath, result.sourceFile, result.concepts);
|
|
1001
|
+
}
|
|
1002
|
+
if (frozenSlugs.size > 0) {
|
|
1003
|
+
await orphanUnownedFrozenPages(root, frozenSlugs);
|
|
1004
|
+
}
|
|
1005
|
+
await persistFrozenSlugs(root, frozenSlugs, extractions);
|
|
1006
|
+
if (allChangedSlugs.length > 0) {
|
|
1007
|
+
status("\u{1F517}", info("Resolving interlinks..."));
|
|
1008
|
+
await resolveLinks(root, allChangedSlugs, allNewSlugs);
|
|
1009
|
+
}
|
|
1010
|
+
await generateIndex(root);
|
|
1011
|
+
header("Compilation complete");
|
|
1012
|
+
status("\u2713", success(
|
|
1013
|
+
`${toCompile.length} compiled, ${unchanged.length} skipped, ${deleted.length} deleted`
|
|
1014
|
+
));
|
|
1015
|
+
if (toCompile.length > 0) {
|
|
1016
|
+
status("\u2192", dim('Next: llmwiki query "your question here"'));
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
function printChangesSummary(changes) {
|
|
1020
|
+
const iconMap = {
|
|
1021
|
+
new: "+",
|
|
1022
|
+
changed: "~",
|
|
1023
|
+
unchanged: ".",
|
|
1024
|
+
deleted: "-"
|
|
1025
|
+
};
|
|
1026
|
+
const fmtMap = {
|
|
1027
|
+
new: success,
|
|
1028
|
+
changed: warn,
|
|
1029
|
+
unchanged: dim,
|
|
1030
|
+
deleted: error
|
|
1031
|
+
};
|
|
1032
|
+
for (const c of changes) {
|
|
1033
|
+
const icon = iconMap[c.status] ?? "?";
|
|
1034
|
+
const fmt = fmtMap[c.status] ?? dim;
|
|
1035
|
+
status(icon, fmt(`${c.file} [${c.status}]`));
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
async function extractForSource(root, sourceFile) {
|
|
1039
|
+
header(`Extracting: ${sourceFile}`);
|
|
1040
|
+
const sourcePath = path10.join(root, SOURCES_DIR, sourceFile);
|
|
1041
|
+
const sourceContent = await readFile7(sourcePath, "utf-8");
|
|
1042
|
+
const existingIndex = await safeReadFile(path10.join(root, INDEX_FILE));
|
|
1043
|
+
const concepts = await extractConcepts(sourceContent, existingIndex);
|
|
1044
|
+
if (concepts.length > 0) logExtractedConcepts(concepts);
|
|
1045
|
+
return { sourceFile, sourcePath, sourceContent, concepts };
|
|
1046
|
+
}
|
|
1047
|
+
function mergeExtractions(extractions, frozenSlugs) {
|
|
1048
|
+
const bySlug = /* @__PURE__ */ new Map();
|
|
1049
|
+
for (const result of extractions) {
|
|
1050
|
+
if (result.concepts.length === 0) continue;
|
|
1051
|
+
for (const concept2 of result.concepts) {
|
|
1052
|
+
const slug = slugify(concept2.concept);
|
|
1053
|
+
if (frozenSlugs.has(slug)) continue;
|
|
1054
|
+
const existing = bySlug.get(slug);
|
|
1055
|
+
if (existing) {
|
|
1056
|
+
existing.sourceFiles.push(result.sourceFile);
|
|
1057
|
+
existing.combinedContent += `
|
|
1058
|
+
|
|
1059
|
+
--- SOURCE: ${result.sourceFile} ---
|
|
1060
|
+
|
|
1061
|
+
${result.sourceContent}`;
|
|
1062
|
+
} else {
|
|
1063
|
+
bySlug.set(slug, {
|
|
1064
|
+
slug,
|
|
1065
|
+
concept: concept2,
|
|
1066
|
+
sourceFiles: [result.sourceFile],
|
|
1067
|
+
combinedContent: `--- SOURCE: ${result.sourceFile} ---
|
|
1068
|
+
|
|
1069
|
+
${result.sourceContent}`
|
|
1070
|
+
});
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
return Array.from(bySlug.values());
|
|
1075
|
+
}
|
|
1076
|
+
async function generateMergedPage(root, entry) {
|
|
1077
|
+
const pagePath = path10.join(root, CONCEPTS_DIR, `${entry.slug}.md`);
|
|
1078
|
+
const existingPage = await safeReadFile(pagePath);
|
|
1079
|
+
const relatedPages = await loadRelatedPages(root, entry.slug);
|
|
1080
|
+
status(">", info(`Generating: ${entry.concept.concept}`));
|
|
1081
|
+
const system = buildPagePrompt(
|
|
1082
|
+
entry.concept.concept,
|
|
1083
|
+
entry.combinedContent,
|
|
1084
|
+
existingPage,
|
|
1085
|
+
relatedPages
|
|
1086
|
+
);
|
|
1087
|
+
const pageBody = await callClaude({
|
|
1088
|
+
system,
|
|
1089
|
+
messages: [
|
|
1090
|
+
{ role: "user", content: `Write the wiki page for "${entry.concept.concept}".` }
|
|
1091
|
+
],
|
|
1092
|
+
stream: true,
|
|
1093
|
+
onToken: (token) => process.stdout.write(dim(token))
|
|
1094
|
+
});
|
|
1095
|
+
process.stdout.write("\n");
|
|
1096
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1097
|
+
const existing = existingPage ? parseFrontmatter(existingPage) : null;
|
|
1098
|
+
const createdAt = existing?.meta.createdAt && typeof existing.meta.createdAt === "string" ? existing.meta.createdAt : now;
|
|
1099
|
+
const frontmatter = buildFrontmatter({
|
|
1100
|
+
title: entry.concept.concept,
|
|
1101
|
+
summary: entry.concept.summary,
|
|
1102
|
+
sources: entry.sourceFiles,
|
|
1103
|
+
createdAt,
|
|
1104
|
+
updatedAt: now
|
|
1105
|
+
});
|
|
1106
|
+
const fullPage = `${frontmatter}
|
|
1107
|
+
|
|
1108
|
+
${pageBody}
|
|
1109
|
+
`;
|
|
1110
|
+
await writePageIfValid(pagePath, fullPage, entry.concept.concept);
|
|
1111
|
+
}
|
|
1112
|
+
async function extractConcepts(sourceContent, existingIndex) {
|
|
1113
|
+
status("*", info("Extracting concepts..."));
|
|
1114
|
+
const system = buildExtractionPrompt(sourceContent, existingIndex);
|
|
1115
|
+
const rawOutput = await callClaude({
|
|
1116
|
+
system,
|
|
1117
|
+
messages: [{ role: "user", content: "Extract the key concepts from this source." }],
|
|
1118
|
+
tools: [CONCEPT_EXTRACTION_TOOL]
|
|
1119
|
+
});
|
|
1120
|
+
return parseConcepts(rawOutput);
|
|
1121
|
+
}
|
|
1122
|
+
function logExtractedConcepts(concepts) {
|
|
1123
|
+
for (const c of concepts) {
|
|
1124
|
+
const tag = c.is_new ? success("NEW") : dim("update");
|
|
1125
|
+
status("*", `${concept(c.concept)} [${tag}] \u2014 ${c.summary}`);
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
async function loadRelatedPages(root, excludeSlug) {
|
|
1129
|
+
const conceptsPath = path10.join(root, CONCEPTS_DIR);
|
|
1130
|
+
let files;
|
|
1131
|
+
try {
|
|
1132
|
+
files = await readdir4(conceptsPath);
|
|
1133
|
+
} catch {
|
|
1134
|
+
return "";
|
|
1135
|
+
}
|
|
1136
|
+
const related = files.filter((f) => f.endsWith(".md") && f !== `${excludeSlug}.md`).slice(0, 5);
|
|
1137
|
+
const contents = [];
|
|
1138
|
+
for (const f of related) {
|
|
1139
|
+
const content = await safeReadFile(path10.join(conceptsPath, f));
|
|
1140
|
+
if (!content) continue;
|
|
1141
|
+
const { meta } = parseFrontmatter(content);
|
|
1142
|
+
if (meta.orphaned) continue;
|
|
1143
|
+
contents.push(content);
|
|
1144
|
+
}
|
|
1145
|
+
return contents.join("\n\n---\n\n");
|
|
1146
|
+
}
|
|
1147
|
+
async function writePageIfValid(pagePath, content, conceptTitle) {
|
|
1148
|
+
if (!validateWikiPage(content)) {
|
|
1149
|
+
status("!", warn(`Invalid page for "${conceptTitle}" \u2014 skipped.`));
|
|
1150
|
+
return;
|
|
1151
|
+
}
|
|
1152
|
+
await atomicWrite(pagePath, content);
|
|
1153
|
+
status("+", success(`Wrote: ${conceptTitle}`));
|
|
1154
|
+
}
|
|
1155
|
+
async function persistSourceState(root, sourcePath, sourceFile, concepts) {
|
|
1156
|
+
const hash = await hashFile(sourcePath);
|
|
1157
|
+
const entry = {
|
|
1158
|
+
hash,
|
|
1159
|
+
concepts: concepts.map((c) => slugify(c.concept)),
|
|
1160
|
+
compiledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1161
|
+
};
|
|
1162
|
+
await updateSourceState(root, sourceFile, entry);
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
// src/commands/compile.ts
|
|
1166
|
+
async function compileCommand() {
|
|
1167
|
+
if (!existsSync3(SOURCES_DIR)) {
|
|
1168
|
+
status(
|
|
1169
|
+
"!",
|
|
1170
|
+
warn("No sources found. Run `llmwiki ingest <url>` first.")
|
|
1171
|
+
);
|
|
1172
|
+
return;
|
|
1173
|
+
}
|
|
1174
|
+
await compile(process.cwd());
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
// src/commands/query.ts
|
|
1178
|
+
import { existsSync as existsSync4 } from "fs";
|
|
1179
|
+
import path11 from "path";
|
|
1180
|
+
var PAGE_DIRS = [CONCEPTS_DIR, QUERIES_DIR];
|
|
1181
|
+
var PAGE_SELECTION_TOOL = {
|
|
1182
|
+
name: "select_pages",
|
|
1183
|
+
description: "Select the most relevant wiki pages to answer a question",
|
|
1184
|
+
input_schema: {
|
|
1185
|
+
type: "object",
|
|
1186
|
+
properties: {
|
|
1187
|
+
pages: {
|
|
1188
|
+
type: "array",
|
|
1189
|
+
items: {
|
|
1190
|
+
type: "string",
|
|
1191
|
+
description: "Slug of a relevant wiki page (e.g. 'llm-knowledge-bases')"
|
|
1192
|
+
},
|
|
1193
|
+
maxItems: QUERY_PAGE_LIMIT
|
|
1194
|
+
},
|
|
1195
|
+
reasoning: {
|
|
1196
|
+
type: "string",
|
|
1197
|
+
description: "Brief explanation of why these pages were selected"
|
|
1198
|
+
}
|
|
1199
|
+
},
|
|
1200
|
+
required: ["pages", "reasoning"]
|
|
1201
|
+
}
|
|
1202
|
+
};
|
|
1203
|
+
async function selectPages(question, indexContent) {
|
|
1204
|
+
const systemPrompt = "You are a knowledge base assistant. Given a question and a wiki index, select the most relevant pages.";
|
|
1205
|
+
const userMessage = `Question: ${question}
|
|
1206
|
+
|
|
1207
|
+
Wiki Index:
|
|
1208
|
+
${indexContent}`;
|
|
1209
|
+
const rawResult = await callClaude({
|
|
1210
|
+
system: systemPrompt,
|
|
1211
|
+
messages: [{ role: "user", content: userMessage }],
|
|
1212
|
+
tools: [PAGE_SELECTION_TOOL]
|
|
1213
|
+
});
|
|
1214
|
+
try {
|
|
1215
|
+
const parsed = JSON.parse(rawResult);
|
|
1216
|
+
return {
|
|
1217
|
+
pages: Array.isArray(parsed.pages) ? parsed.pages.filter((p) => typeof p === "string") : [],
|
|
1218
|
+
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "No reasoning provided"
|
|
1219
|
+
};
|
|
1220
|
+
} catch {
|
|
1221
|
+
return { pages: [], reasoning: "Failed to parse page selection response" };
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
async function loadSelectedPages(root, slugs) {
|
|
1225
|
+
const sections = [];
|
|
1226
|
+
for (const slug of slugs) {
|
|
1227
|
+
let content = "";
|
|
1228
|
+
for (const dir of PAGE_DIRS) {
|
|
1229
|
+
const candidate = await safeReadFile(path11.join(root, dir, `${slug}.md`));
|
|
1230
|
+
if (!candidate) continue;
|
|
1231
|
+
const { meta } = parseFrontmatter(candidate);
|
|
1232
|
+
if (meta.orphaned) continue;
|
|
1233
|
+
content = candidate;
|
|
1234
|
+
break;
|
|
1235
|
+
}
|
|
1236
|
+
if (!content) {
|
|
1237
|
+
status("?", warn(`Page not found: ${slug}.md \u2014 skipping`));
|
|
1238
|
+
continue;
|
|
1239
|
+
}
|
|
1240
|
+
sections.push(`--- Page: ${slug} ---
|
|
1241
|
+
${content}`);
|
|
1242
|
+
}
|
|
1243
|
+
return sections.join("\n\n");
|
|
1244
|
+
}
|
|
1245
|
+
async function streamAnswer(question, pagesContent) {
|
|
1246
|
+
const systemPrompt = "You are a knowledge assistant. Answer the question using ONLY the wiki content provided. Cite specific pages using [[Page Title]] wikilinks. If the wiki doesn't contain enough information, say so.";
|
|
1247
|
+
const userMessage = `Question: ${question}
|
|
1248
|
+
|
|
1249
|
+
Relevant wiki pages:
|
|
1250
|
+
${pagesContent}`;
|
|
1251
|
+
const answer = await callClaude({
|
|
1252
|
+
system: systemPrompt,
|
|
1253
|
+
messages: [{ role: "user", content: userMessage }],
|
|
1254
|
+
stream: true,
|
|
1255
|
+
onToken: (text) => process.stdout.write(text)
|
|
1256
|
+
});
|
|
1257
|
+
process.stdout.write("\n");
|
|
1258
|
+
return answer;
|
|
1259
|
+
}
|
|
1260
|
+
function summarizeAnswer(answer) {
|
|
1261
|
+
const firstLine = answer.trim().split(/\n/)[0] ?? "";
|
|
1262
|
+
const firstSentence = firstLine.split(/(?<=[.!?])\s/)[0] ?? firstLine;
|
|
1263
|
+
return firstSentence.slice(0, 120);
|
|
1264
|
+
}
|
|
1265
|
+
async function saveQueryPage(root, question, answer) {
|
|
1266
|
+
const slug = slugify(question);
|
|
1267
|
+
const filePath = path11.join(root, QUERIES_DIR, `${slug}.md`);
|
|
1268
|
+
const frontmatter = buildFrontmatter({
|
|
1269
|
+
title: question,
|
|
1270
|
+
summary: summarizeAnswer(answer),
|
|
1271
|
+
type: "query",
|
|
1272
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1273
|
+
});
|
|
1274
|
+
const document = `${frontmatter}
|
|
1275
|
+
|
|
1276
|
+
${answer}
|
|
1277
|
+
`;
|
|
1278
|
+
await atomicWrite(filePath, document);
|
|
1279
|
+
status(
|
|
1280
|
+
"+",
|
|
1281
|
+
success(`Saved query \u2192 ${source(filePath)}`)
|
|
1282
|
+
);
|
|
1283
|
+
await generateIndex(root);
|
|
1284
|
+
}
|
|
1285
|
+
async function queryCommand(root, question, options) {
|
|
1286
|
+
if (!existsSync4(path11.join(root, INDEX_FILE))) {
|
|
1287
|
+
status("!", error("Wiki index not found. Run `llmwiki compile` first."));
|
|
1288
|
+
return;
|
|
1289
|
+
}
|
|
1290
|
+
header("Selecting relevant pages");
|
|
1291
|
+
const indexContent = await safeReadFile(path11.join(root, INDEX_FILE));
|
|
1292
|
+
const { pages: rawPages, reasoning } = await selectPages(question, indexContent);
|
|
1293
|
+
const pages = rawPages.map((p) => slugify(p));
|
|
1294
|
+
status("i", dim(`Reasoning: ${reasoning}`));
|
|
1295
|
+
status("*", info(`Selected ${pages.length} page(s): ${rawPages.join(", ")}`));
|
|
1296
|
+
header("Generating answer");
|
|
1297
|
+
const pagesContent = await loadSelectedPages(root, pages);
|
|
1298
|
+
if (!pagesContent) {
|
|
1299
|
+
status("!", error("No matching pages found. Try refining your question."));
|
|
1300
|
+
return;
|
|
1301
|
+
}
|
|
1302
|
+
const answer = await streamAnswer(question, pagesContent);
|
|
1303
|
+
if (options.save) {
|
|
1304
|
+
await saveQueryPage(root, question, answer);
|
|
1305
|
+
status("\u2192", dim("Saved. Future queries will use this answer as context."));
|
|
1306
|
+
} else {
|
|
1307
|
+
status("\u2192", dim("Tip: use --save to add this answer to your wiki"));
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
// src/commands/watch.ts
|
|
1312
|
+
import { watch as chokidarWatch } from "chokidar";
|
|
1313
|
+
import { existsSync as existsSync5 } from "fs";
|
|
1314
|
+
import path12 from "path";
|
|
1315
|
+
var DEBOUNCE_MS = 500;
|
|
1316
|
+
async function watchCommand() {
|
|
1317
|
+
const sourcesPath = path12.resolve(SOURCES_DIR);
|
|
1318
|
+
if (!existsSync5(sourcesPath)) {
|
|
1319
|
+
status(
|
|
1320
|
+
"!",
|
|
1321
|
+
warn("No sources/ directory found. Run `llmwiki ingest <url>` first.")
|
|
1322
|
+
);
|
|
1323
|
+
return;
|
|
1324
|
+
}
|
|
1325
|
+
header("llmwiki watch");
|
|
1326
|
+
status("\u{1F441}", info(`Watching ${sourcesPath} for changes...`));
|
|
1327
|
+
status("i", dim("Press Ctrl+C to stop.\n"));
|
|
1328
|
+
let compiling = false;
|
|
1329
|
+
let pendingRecompile = false;
|
|
1330
|
+
let debounceTimer = null;
|
|
1331
|
+
const triggerCompile = async () => {
|
|
1332
|
+
if (compiling) {
|
|
1333
|
+
pendingRecompile = true;
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
compiling = true;
|
|
1337
|
+
try {
|
|
1338
|
+
await compile(process.cwd());
|
|
1339
|
+
} catch (err) {
|
|
1340
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1341
|
+
status("!", error(`Compile failed: ${msg}`));
|
|
1342
|
+
}
|
|
1343
|
+
compiling = false;
|
|
1344
|
+
if (pendingRecompile) {
|
|
1345
|
+
pendingRecompile = false;
|
|
1346
|
+
await triggerCompile();
|
|
1347
|
+
}
|
|
1348
|
+
};
|
|
1349
|
+
const scheduleCompile = (eventPath, event) => {
|
|
1350
|
+
status(
|
|
1351
|
+
"~",
|
|
1352
|
+
dim(`${event}: ${path12.basename(eventPath)}`)
|
|
1353
|
+
);
|
|
1354
|
+
if (debounceTimer) clearTimeout(debounceTimer);
|
|
1355
|
+
debounceTimer = setTimeout(triggerCompile, DEBOUNCE_MS);
|
|
1356
|
+
};
|
|
1357
|
+
const watcher = chokidarWatch(sourcesPath, {
|
|
1358
|
+
ignoreInitial: true,
|
|
1359
|
+
awaitWriteFinish: { stabilityThreshold: 200 }
|
|
1360
|
+
});
|
|
1361
|
+
watcher.on("add", (p) => scheduleCompile(p, "added")).on("change", (p) => scheduleCompile(p, "changed")).on("unlink", (p) => scheduleCompile(p, "deleted"));
|
|
1362
|
+
await new Promise(() => {
|
|
1363
|
+
});
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
// src/cli.ts
|
|
1367
|
+
var require2 = createRequire(import.meta.url);
|
|
1368
|
+
var { version } = require2("../package.json");
|
|
1369
|
+
var program = new Command();
|
|
1370
|
+
program.name("llmwiki").description("The knowledge compiler \u2014 raw sources in, interlinked wiki out").version(version);
|
|
1371
|
+
program.command("ingest <source>").description("Ingest a URL or local file into sources/").action(async (source2) => {
|
|
1372
|
+
try {
|
|
1373
|
+
await ingest(source2);
|
|
1374
|
+
} catch (err) {
|
|
1375
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
1376
|
+
process.exit(1);
|
|
1377
|
+
}
|
|
1378
|
+
});
|
|
1379
|
+
program.command("compile").description("Compile sources/ into an interlinked wiki").action(async () => {
|
|
1380
|
+
requireApiKey();
|
|
1381
|
+
try {
|
|
1382
|
+
await compileCommand();
|
|
1383
|
+
} catch (err) {
|
|
1384
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
1385
|
+
process.exit(1);
|
|
1386
|
+
}
|
|
1387
|
+
});
|
|
1388
|
+
program.command("query <question>").description("Ask a question against the wiki").option("--save", "Save the answer as a wiki page").action(async (question, options) => {
|
|
1389
|
+
requireApiKey();
|
|
1390
|
+
try {
|
|
1391
|
+
await queryCommand(process.cwd(), question, options);
|
|
1392
|
+
} catch (err) {
|
|
1393
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
1394
|
+
process.exit(1);
|
|
1395
|
+
}
|
|
1396
|
+
});
|
|
1397
|
+
program.command("watch").description("Watch sources/ and auto-recompile on changes").action(async () => {
|
|
1398
|
+
requireApiKey();
|
|
1399
|
+
try {
|
|
1400
|
+
await watchCommand();
|
|
1401
|
+
} catch (err) {
|
|
1402
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
1403
|
+
process.exit(1);
|
|
1404
|
+
}
|
|
1405
|
+
});
|
|
1406
|
+
program.parse();
|
|
1407
|
+
function requireApiKey() {
|
|
1408
|
+
if (!process.env.ANTHROPIC_API_KEY) {
|
|
1409
|
+
console.error(
|
|
1410
|
+
"\x1B[31mError:\x1B[0m ANTHROPIC_API_KEY environment variable is required.\n Set it with: export ANTHROPIC_API_KEY=sk-ant-...\n Get a key at: https://console.anthropic.com/settings/keys"
|
|
1411
|
+
);
|
|
1412
|
+
process.exit(1);
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
//# sourceMappingURL=cli.js.map
|