llm-wiki-compiler 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -8
- package/dist/cli.js +846 -223
- package/dist/cli.js.map +1 -1
- package/package.json +5 -2
package/dist/cli.js
CHANGED
|
@@ -13,6 +13,12 @@ import { mkdir as mkdir2, writeFile as writeFile2 } from "fs/promises";
|
|
|
13
13
|
import { writeFile, rename, readFile, mkdir } from "fs/promises";
|
|
14
14
|
import path from "path";
|
|
15
15
|
import yaml from "js-yaml";
|
|
16
|
+
var VALID_PROVENANCE_STATES = /* @__PURE__ */ new Set([
|
|
17
|
+
"extracted",
|
|
18
|
+
"merged",
|
|
19
|
+
"inferred",
|
|
20
|
+
"ambiguous"
|
|
21
|
+
]);
|
|
16
22
|
function slugify(title) {
|
|
17
23
|
return title.toLowerCase().replace(/['']/g, "").replace(/[^\w\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
18
24
|
}
|
|
@@ -50,6 +56,46 @@ async function safeReadFile(filePath) {
|
|
|
50
56
|
return "";
|
|
51
57
|
}
|
|
52
58
|
}
|
|
59
|
+
function parseConfidence(raw) {
|
|
60
|
+
if (typeof raw !== "number" || !Number.isFinite(raw)) return void 0;
|
|
61
|
+
if (raw < 0) return 0;
|
|
62
|
+
if (raw > 1) return 1;
|
|
63
|
+
return raw;
|
|
64
|
+
}
|
|
65
|
+
function parseProvenanceState(raw) {
|
|
66
|
+
if (typeof raw !== "string") return void 0;
|
|
67
|
+
return VALID_PROVENANCE_STATES.has(raw) ? raw : void 0;
|
|
68
|
+
}
|
|
69
|
+
function coerceContradictionEntry(entry) {
|
|
70
|
+
if (typeof entry === "string" && entry.trim().length > 0) {
|
|
71
|
+
return { slug: entry.trim() };
|
|
72
|
+
}
|
|
73
|
+
if (entry && typeof entry === "object" && "slug" in entry) {
|
|
74
|
+
const obj = entry;
|
|
75
|
+
if (typeof obj.slug !== "string" || obj.slug.trim().length === 0) return null;
|
|
76
|
+
const ref = { slug: obj.slug.trim() };
|
|
77
|
+
if (typeof obj.reason === "string") ref.reason = obj.reason;
|
|
78
|
+
return ref;
|
|
79
|
+
}
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
function parseContradictedBy(raw) {
|
|
83
|
+
if (!Array.isArray(raw)) return void 0;
|
|
84
|
+
const refs = raw.map(coerceContradictionEntry).filter((ref) => ref !== null);
|
|
85
|
+
return refs.length > 0 ? refs : void 0;
|
|
86
|
+
}
|
|
87
|
+
function parseInferredParagraphs(raw) {
|
|
88
|
+
if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 0) return void 0;
|
|
89
|
+
return raw;
|
|
90
|
+
}
|
|
91
|
+
function parseProvenanceMetadata(meta) {
|
|
92
|
+
return {
|
|
93
|
+
confidence: parseConfidence(meta.confidence),
|
|
94
|
+
provenanceState: parseProvenanceState(meta.provenanceState),
|
|
95
|
+
contradictedBy: parseContradictedBy(meta.contradictedBy),
|
|
96
|
+
inferredParagraphs: parseInferredParagraphs(meta.inferredParagraphs)
|
|
97
|
+
};
|
|
98
|
+
}
|
|
53
99
|
function validateWikiPage(content) {
|
|
54
100
|
if (!content || content.trim().length === 0) return false;
|
|
55
101
|
const { meta, body } = parseFrontmatter(content);
|
|
@@ -83,7 +129,11 @@ var LOCK_FILE = ".llmwiki/lock";
|
|
|
83
129
|
var INDEX_FILE = "wiki/index.md";
|
|
84
130
|
var MOC_FILE = "wiki/MOC.md";
|
|
85
131
|
var EMBEDDINGS_FILE = ".llmwiki/embeddings.json";
|
|
132
|
+
var CANDIDATES_DIR = ".llmwiki/candidates";
|
|
133
|
+
var CANDIDATES_ARCHIVE_DIR = ".llmwiki/candidates/archive";
|
|
86
134
|
var EMBEDDING_TOP_K = 15;
|
|
135
|
+
var LOW_CONFIDENCE_THRESHOLD = 0.5;
|
|
136
|
+
var MAX_INFERRED_PARAGRAPHS_WITHOUT_CITATIONS = 2;
|
|
87
137
|
var EMBEDDING_MODELS = {
|
|
88
138
|
anthropic: "voyage-3-lite",
|
|
89
139
|
openai: "text-embedding-3-small",
|
|
@@ -274,11 +324,11 @@ async function ingest(source2) {
|
|
|
274
324
|
}
|
|
275
325
|
|
|
276
326
|
// src/commands/compile.ts
|
|
277
|
-
import { existsSync as
|
|
327
|
+
import { existsSync as existsSync5 } from "fs";
|
|
278
328
|
|
|
279
329
|
// src/compiler/index.ts
|
|
280
|
-
import { readFile as readFile8
|
|
281
|
-
import
|
|
330
|
+
import { readFile as readFile8 } from "fs/promises";
|
|
331
|
+
import path16 from "path";
|
|
282
332
|
|
|
283
333
|
// src/utils/state.ts
|
|
284
334
|
import { readFile as readFile3, writeFile as writeFile3, rename as rename2, mkdir as mkdir3, copyFile } from "fs/promises";
|
|
@@ -321,6 +371,78 @@ async function removeSourceState(root, sourceFile) {
|
|
|
321
371
|
await writeState(root, state);
|
|
322
372
|
}
|
|
323
373
|
|
|
374
|
+
// src/compiler/source-state.ts
|
|
375
|
+
import path6 from "path";
|
|
376
|
+
|
|
377
|
+
// src/compiler/hasher.ts
|
|
378
|
+
import { createHash } from "crypto";
|
|
379
|
+
import { readFile as readFile4, readdir } from "fs/promises";
|
|
380
|
+
import path5 from "path";
|
|
381
|
+
async function hashFile(filePath) {
|
|
382
|
+
const content = await readFile4(filePath, "utf-8");
|
|
383
|
+
return createHash("sha256").update(content).digest("hex");
|
|
384
|
+
}
|
|
385
|
+
async function detectChanges(root, prevState) {
|
|
386
|
+
const sourcesPath = path5.join(root, SOURCES_DIR);
|
|
387
|
+
const currentFiles = await listSourceFiles(sourcesPath);
|
|
388
|
+
const changes = [];
|
|
389
|
+
for (const file of currentFiles) {
|
|
390
|
+
const status2 = await classifyFile(root, file, prevState);
|
|
391
|
+
changes.push({ file, status: status2 });
|
|
392
|
+
}
|
|
393
|
+
const deletedChanges = findDeletedFiles(currentFiles, prevState);
|
|
394
|
+
changes.push(...deletedChanges);
|
|
395
|
+
return changes;
|
|
396
|
+
}
|
|
397
|
+
async function listSourceFiles(sourcesPath) {
|
|
398
|
+
try {
|
|
399
|
+
const entries = await readdir(sourcesPath);
|
|
400
|
+
return entries.filter((f) => f.endsWith(".md"));
|
|
401
|
+
} catch {
|
|
402
|
+
return [];
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
async function classifyFile(root, file, prevState) {
|
|
406
|
+
const filePath = path5.join(root, SOURCES_DIR, file);
|
|
407
|
+
const hash = await hashFile(filePath);
|
|
408
|
+
const prev = prevState.sources[file];
|
|
409
|
+
if (!prev) return "new";
|
|
410
|
+
if (prev.hash !== hash) return "changed";
|
|
411
|
+
return "unchanged";
|
|
412
|
+
}
|
|
413
|
+
function findDeletedFiles(currentFiles, prevState) {
|
|
414
|
+
const currentSet = new Set(currentFiles);
|
|
415
|
+
return Object.keys(prevState.sources).filter((file) => !currentSet.has(file)).map((file) => ({ file, status: "deleted" }));
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// src/compiler/source-state.ts
|
|
419
|
+
async function buildExtractionSourceStates(root, extractions) {
|
|
420
|
+
const snapshot = {};
|
|
421
|
+
const compiledAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
422
|
+
for (const result of extractions) {
|
|
423
|
+
if (result.concepts.length === 0) continue;
|
|
424
|
+
snapshot[result.sourceFile] = await buildEntry(root, result, compiledAt);
|
|
425
|
+
}
|
|
426
|
+
return snapshot;
|
|
427
|
+
}
|
|
428
|
+
async function buildEntry(root, result, compiledAt) {
|
|
429
|
+
const filePath = path6.join(root, SOURCES_DIR, result.sourceFile);
|
|
430
|
+
const hash = await hashFile(filePath);
|
|
431
|
+
return {
|
|
432
|
+
hash,
|
|
433
|
+
concepts: result.concepts.map((concept) => slugify(concept.concept)),
|
|
434
|
+
compiledAt
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
function pickStatesForSources(allStates, sourceFiles) {
|
|
438
|
+
const picked = {};
|
|
439
|
+
for (const file of sourceFiles) {
|
|
440
|
+
const entry = allStates[file];
|
|
441
|
+
if (entry) picked[file] = entry;
|
|
442
|
+
}
|
|
443
|
+
return picked;
|
|
444
|
+
}
|
|
445
|
+
|
|
324
446
|
// src/providers/anthropic.ts
|
|
325
447
|
import Anthropic from "@anthropic-ai/sdk";
|
|
326
448
|
var VOYAGE_EMBEDDINGS_URL = "https://api.voyageai.com/v1/embeddings";
|
|
@@ -446,14 +568,18 @@ function translateToolToOpenAI(tool) {
|
|
|
446
568
|
}
|
|
447
569
|
var OpenAIProvider = class {
|
|
448
570
|
client;
|
|
571
|
+
embeddingsClient;
|
|
449
572
|
model;
|
|
450
|
-
|
|
573
|
+
configuredEmbeddingModel;
|
|
574
|
+
constructor(model, options = {}) {
|
|
451
575
|
this.model = model;
|
|
452
|
-
|
|
576
|
+
this.configuredEmbeddingModel = options.embeddingModel;
|
|
577
|
+
const resolvedKey = options.apiKey ?? process.env.OPENAI_API_KEY ?? "";
|
|
453
578
|
this.client = new OpenAI({
|
|
454
579
|
apiKey: resolvedKey,
|
|
455
|
-
|
|
580
|
+
baseURL: options.baseURL ?? null
|
|
456
581
|
});
|
|
582
|
+
this.embeddingsClient = options.embeddingsBaseURL ? new OpenAI({ apiKey: resolvedKey, baseURL: options.embeddingsBaseURL }) : this.client;
|
|
457
583
|
}
|
|
458
584
|
/** Send a single non-streaming completion request. */
|
|
459
585
|
async complete(system, messages, maxTokens) {
|
|
@@ -502,7 +628,7 @@ var OpenAIProvider = class {
|
|
|
502
628
|
* Subclasses (e.g. Ollama) override embeddingModel() to pick a different model.
|
|
503
629
|
*/
|
|
504
630
|
async embed(text) {
|
|
505
|
-
const response = await this.
|
|
631
|
+
const response = await this.embeddingsClient.embeddings.create({
|
|
506
632
|
model: this.embeddingModel(),
|
|
507
633
|
input: text
|
|
508
634
|
});
|
|
@@ -514,18 +640,23 @@ var OpenAIProvider = class {
|
|
|
514
640
|
}
|
|
515
641
|
/** Default embedding model for this provider. Subclasses may override. */
|
|
516
642
|
embeddingModel() {
|
|
517
|
-
return EMBEDDING_MODELS.openai;
|
|
643
|
+
return this.configuredEmbeddingModel ?? EMBEDDING_MODELS.openai;
|
|
518
644
|
}
|
|
519
645
|
};
|
|
520
646
|
|
|
521
647
|
// src/providers/ollama.ts
|
|
522
648
|
var OllamaProvider = class extends OpenAIProvider {
|
|
523
|
-
constructor(model,
|
|
524
|
-
super(model,
|
|
649
|
+
constructor(model, options) {
|
|
650
|
+
super(model, {
|
|
651
|
+
baseURL: options.baseURL,
|
|
652
|
+
apiKey: "ollama",
|
|
653
|
+
embeddingsBaseURL: options.embeddingsBaseURL,
|
|
654
|
+
embeddingModel: options.embeddingModel
|
|
655
|
+
});
|
|
525
656
|
}
|
|
526
657
|
/** Ollama ships a dedicated embedding model (nomic-embed-text). */
|
|
527
658
|
embeddingModel() {
|
|
528
|
-
return EMBEDDING_MODELS.ollama;
|
|
659
|
+
return this.configuredEmbeddingModel ?? EMBEDDING_MODELS.ollama;
|
|
529
660
|
}
|
|
530
661
|
};
|
|
531
662
|
|
|
@@ -533,14 +664,14 @@ var OllamaProvider = class extends OpenAIProvider {
|
|
|
533
664
|
var MINIMAX_BASE_URL = "https://api.minimax.io/v1";
|
|
534
665
|
var MiniMaxProvider = class extends OpenAIProvider {
|
|
535
666
|
constructor(model, apiKey) {
|
|
536
|
-
super(model, MINIMAX_BASE_URL, apiKey);
|
|
667
|
+
super(model, { baseURL: MINIMAX_BASE_URL, apiKey });
|
|
537
668
|
}
|
|
538
669
|
};
|
|
539
670
|
|
|
540
671
|
// src/utils/claude-settings.ts
|
|
541
672
|
import { readFileSync } from "fs";
|
|
542
673
|
import { homedir } from "os";
|
|
543
|
-
import
|
|
674
|
+
import path7 from "path";
|
|
544
675
|
var CLAUDE_SETTINGS_PATH_ENV = "LLMWIKI_CLAUDE_SETTINGS_PATH";
|
|
545
676
|
function isRecord(value) {
|
|
546
677
|
return typeof value === "object" && value !== null;
|
|
@@ -551,7 +682,7 @@ function normalize(value) {
|
|
|
551
682
|
return trimmed.length > 0 ? trimmed : void 0;
|
|
552
683
|
}
|
|
553
684
|
function resolveClaudeSettingsPath(env) {
|
|
554
|
-
return env[CLAUDE_SETTINGS_PATH_ENV] ??
|
|
685
|
+
return env[CLAUDE_SETTINGS_PATH_ENV] ?? path7.join(homedir(), ".claude", "settings.json");
|
|
555
686
|
}
|
|
556
687
|
function readClaudeSettingsFile(settingsPath) {
|
|
557
688
|
try {
|
|
@@ -640,18 +771,27 @@ function getProvider() {
|
|
|
640
771
|
case "anthropic":
|
|
641
772
|
return getAnthropicProvider();
|
|
642
773
|
case "openai":
|
|
643
|
-
return new OpenAIProvider(getModelForProvider("openai")
|
|
774
|
+
return new OpenAIProvider(getModelForProvider("openai"), {
|
|
775
|
+
baseURL: readOptionalEnv("OPENAI_BASE_URL"),
|
|
776
|
+
embeddingsBaseURL: readOptionalEnv("OPENAI_EMBEDDINGS_BASE_URL"),
|
|
777
|
+
embeddingModel: readOptionalEnv("LLMWIKI_EMBEDDING_MODEL")
|
|
778
|
+
});
|
|
644
779
|
case "ollama":
|
|
645
|
-
return new OllamaProvider(
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
780
|
+
return new OllamaProvider(getModelForProvider("ollama"), {
|
|
781
|
+
baseURL: readOptionalEnv("OLLAMA_HOST") ?? OLLAMA_DEFAULT_HOST,
|
|
782
|
+
embeddingsBaseURL: readOptionalEnv("OLLAMA_EMBEDDINGS_HOST"),
|
|
783
|
+
embeddingModel: readOptionalEnv("LLMWIKI_EMBEDDING_MODEL")
|
|
784
|
+
});
|
|
649
785
|
case "minimax":
|
|
650
786
|
return getMiniMaxProvider();
|
|
651
787
|
default:
|
|
652
788
|
throw new Error(`Unhandled provider: ${providerName}`);
|
|
653
789
|
}
|
|
654
790
|
}
|
|
791
|
+
function readOptionalEnv(name) {
|
|
792
|
+
const value = process.env[name]?.trim();
|
|
793
|
+
return value ? value : void 0;
|
|
794
|
+
}
|
|
655
795
|
function getModelForProvider(providerName) {
|
|
656
796
|
return process.env.LLMWIKI_MODEL ?? PROVIDER_MODELS[providerName];
|
|
657
797
|
}
|
|
@@ -715,8 +855,8 @@ async function callClaude(options) {
|
|
|
715
855
|
}
|
|
716
856
|
|
|
717
857
|
// src/utils/lock.ts
|
|
718
|
-
import { open, readFile as
|
|
719
|
-
import
|
|
858
|
+
import { open, readFile as readFile5, unlink, mkdir as mkdir4 } from "fs/promises";
|
|
859
|
+
import path8 from "path";
|
|
720
860
|
var RECLAIM_SUFFIX = ".reclaim";
|
|
721
861
|
var MAX_ACQUIRE_ATTEMPTS = 2;
|
|
722
862
|
function isProcessAlive(pid) {
|
|
@@ -728,8 +868,8 @@ function isProcessAlive(pid) {
|
|
|
728
868
|
}
|
|
729
869
|
}
|
|
730
870
|
async function acquireLock(root) {
|
|
731
|
-
const lockPath =
|
|
732
|
-
await mkdir4(
|
|
871
|
+
const lockPath = path8.join(root, LOCK_FILE);
|
|
872
|
+
await mkdir4(path8.join(root, LLMWIKI_DIR), { recursive: true });
|
|
733
873
|
for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt++) {
|
|
734
874
|
const created = await tryCreateLock(lockPath);
|
|
735
875
|
if (created) return true;
|
|
@@ -792,7 +932,7 @@ async function tryCreateLock(lockPath) {
|
|
|
792
932
|
}
|
|
793
933
|
async function isLockStale(lockPath) {
|
|
794
934
|
try {
|
|
795
|
-
const content = await
|
|
935
|
+
const content = await readFile5(lockPath, "utf-8");
|
|
796
936
|
const pid = parseInt(content.trim(), 10);
|
|
797
937
|
if (isNaN(pid)) return true;
|
|
798
938
|
return !isProcessAlive(pid);
|
|
@@ -801,7 +941,7 @@ async function isLockStale(lockPath) {
|
|
|
801
941
|
}
|
|
802
942
|
}
|
|
803
943
|
async function releaseLock(root) {
|
|
804
|
-
const lockPath =
|
|
944
|
+
const lockPath = path8.join(root, LOCK_FILE);
|
|
805
945
|
try {
|
|
806
946
|
await unlink(lockPath);
|
|
807
947
|
} catch {
|
|
@@ -809,6 +949,12 @@ async function releaseLock(root) {
|
|
|
809
949
|
}
|
|
810
950
|
|
|
811
951
|
// src/compiler/prompts.ts
|
|
952
|
+
var PROVENANCE_STATE_VALUES = [
|
|
953
|
+
"extracted",
|
|
954
|
+
"merged",
|
|
955
|
+
"inferred",
|
|
956
|
+
"ambiguous"
|
|
957
|
+
];
|
|
812
958
|
var CONCEPT_EXTRACTION_TOOL = {
|
|
813
959
|
name: "extract_concepts",
|
|
814
960
|
description: "Extract knowledge concepts from a source document",
|
|
@@ -836,6 +982,31 @@ var CONCEPT_EXTRACTION_TOOL = {
|
|
|
836
982
|
type: "array",
|
|
837
983
|
items: { type: "string" },
|
|
838
984
|
description: "2-4 categorical tags for organizing this concept (e.g., 'machine-learning', 'optimization')"
|
|
985
|
+
},
|
|
986
|
+
confidence: {
|
|
987
|
+
type: "number",
|
|
988
|
+
description: "Confidence in this concept on a 0..1 scale (1 = directly stated, 0 = highly speculative)."
|
|
989
|
+
},
|
|
990
|
+
provenance_state: {
|
|
991
|
+
type: "string",
|
|
992
|
+
enum: PROVENANCE_STATE_VALUES,
|
|
993
|
+
description: "How this concept was produced: 'extracted' (direct from source), 'merged' (synthesised across sources), 'inferred' (model deduction), or 'ambiguous' (sources disagree)."
|
|
994
|
+
},
|
|
995
|
+
contradicted_by: {
|
|
996
|
+
type: "array",
|
|
997
|
+
items: {
|
|
998
|
+
type: "object",
|
|
999
|
+
properties: {
|
|
1000
|
+
slug: { type: "string", description: "Slug of the contradicting concept." },
|
|
1001
|
+
reason: { type: "string", description: "Brief reason for the contradiction." }
|
|
1002
|
+
},
|
|
1003
|
+
required: ["slug"]
|
|
1004
|
+
},
|
|
1005
|
+
description: "Slugs of other concepts whose evidence contradicts this one."
|
|
1006
|
+
},
|
|
1007
|
+
inferred_paragraphs: {
|
|
1008
|
+
type: "integer",
|
|
1009
|
+
description: "Estimated number of paragraphs in the page that will be inferred rather than directly cited."
|
|
839
1010
|
}
|
|
840
1011
|
},
|
|
841
1012
|
required: ["concept", "summary", "is_new"]
|
|
@@ -857,6 +1028,17 @@ ${existingIndex}` : "\n\nNo existing wiki pages yet.";
|
|
|
857
1028
|
"Each concept should be a standalone topic that someone might look up.",
|
|
858
1029
|
"Focus on key ideas, techniques, patterns, or entities \u2014 not trivial details.",
|
|
859
1030
|
"Use the extract_concepts tool to return your findings.",
|
|
1031
|
+
"",
|
|
1032
|
+
"For every concept, emit provenance metadata so downstream tools can reason",
|
|
1033
|
+
"about reliability:",
|
|
1034
|
+
" - confidence: 0..1 \u2014 how certain you are the source supports this concept.",
|
|
1035
|
+
" - provenance_state: 'extracted' if directly stated, 'merged' if synthesised",
|
|
1036
|
+
" from multiple parts of the source, 'inferred' if reasoned from context,",
|
|
1037
|
+
" or 'ambiguous' if the source is contradictory or unclear.",
|
|
1038
|
+
" - contradicted_by: slugs of other concepts (in this batch or the index)",
|
|
1039
|
+
" whose evidence conflicts with this one.",
|
|
1040
|
+
" - inferred_paragraphs: estimated number of paragraphs in the resulting",
|
|
1041
|
+
" page that will be inferred rather than directly citable.",
|
|
860
1042
|
indexSection,
|
|
861
1043
|
"\n\n--- SOURCE DOCUMENT ---\n\n",
|
|
862
1044
|
sourceContent
|
|
@@ -885,69 +1067,54 @@ ${relatedPages}` : "";
|
|
|
885
1067
|
"Format: ^[filename.md] for single-source, ^[source-a.md, source-b.md] for multi-source.",
|
|
886
1068
|
"Place citations only at the end of prose paragraphs \u2014 not on headings, list items, or code blocks.",
|
|
887
1069
|
"Source filenames are visible as `--- SOURCE: filename.md ---` headers in the content below.",
|
|
1070
|
+
"",
|
|
1071
|
+
"If a paragraph is your inference rather than a direct extraction, leave it",
|
|
1072
|
+
"uncited \u2014 downstream lint rules will count uncited paragraphs as 'inferred'",
|
|
1073
|
+
"to compute the page's provenance metadata.",
|
|
888
1074
|
existingSection,
|
|
889
1075
|
relatedSection,
|
|
890
1076
|
"\n\n--- SOURCE MATERIAL ---\n\n",
|
|
891
1077
|
sourceContent
|
|
892
1078
|
].join("\n");
|
|
893
1079
|
}
|
|
894
|
-
function
|
|
895
|
-
|
|
896
|
-
const parsed = JSON.parse(toolOutput);
|
|
897
|
-
const concepts = parsed.concepts ?? [];
|
|
898
|
-
return concepts.filter(
|
|
899
|
-
(c) => typeof c.concept === "string" && typeof c.summary === "string" && typeof c.is_new === "boolean" && (c.tags === void 0 || Array.isArray(c.tags))
|
|
900
|
-
).map((c) => ({
|
|
901
|
-
concept: c.concept,
|
|
902
|
-
summary: c.summary,
|
|
903
|
-
is_new: c.is_new,
|
|
904
|
-
tags: Array.isArray(c.tags) ? c.tags : void 0
|
|
905
|
-
}));
|
|
906
|
-
} catch {
|
|
907
|
-
return [];
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
|
|
911
|
-
// src/compiler/hasher.ts
|
|
912
|
-
import { createHash } from "crypto";
|
|
913
|
-
import { readFile as readFile5, readdir } from "fs/promises";
|
|
914
|
-
import path7 from "path";
|
|
915
|
-
async function hashFile(filePath) {
|
|
916
|
-
const content = await readFile5(filePath, "utf-8");
|
|
917
|
-
return createHash("sha256").update(content).digest("hex");
|
|
1080
|
+
function isValidRawConcept(c) {
|
|
1081
|
+
return typeof c.concept === "string" && typeof c.summary === "string" && typeof c.is_new === "boolean" && (c.tags === void 0 || Array.isArray(c.tags));
|
|
918
1082
|
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
const
|
|
922
|
-
const
|
|
923
|
-
|
|
924
|
-
const
|
|
925
|
-
|
|
1083
|
+
function coerceContradictedBy(raw) {
|
|
1084
|
+
if (!Array.isArray(raw)) return void 0;
|
|
1085
|
+
const refs = [];
|
|
1086
|
+
for (const entry of raw) {
|
|
1087
|
+
if (!entry || typeof entry !== "object") continue;
|
|
1088
|
+
const obj = entry;
|
|
1089
|
+
if (typeof obj.slug !== "string" || obj.slug.trim().length === 0) continue;
|
|
1090
|
+
const ref = { slug: obj.slug.trim() };
|
|
1091
|
+
if (typeof obj.reason === "string") ref.reason = obj.reason;
|
|
1092
|
+
refs.push(ref);
|
|
926
1093
|
}
|
|
927
|
-
|
|
928
|
-
changes.push(...deletedChanges);
|
|
929
|
-
return changes;
|
|
1094
|
+
return refs.length > 0 ? refs : void 0;
|
|
930
1095
|
}
|
|
931
|
-
|
|
1096
|
+
function mapRawConcept(c) {
|
|
1097
|
+
const provenance = typeof c.provenance_state === "string" && PROVENANCE_STATE_VALUES.includes(c.provenance_state) ? c.provenance_state : void 0;
|
|
1098
|
+
return {
|
|
1099
|
+
concept: c.concept,
|
|
1100
|
+
summary: c.summary,
|
|
1101
|
+
is_new: c.is_new,
|
|
1102
|
+
tags: Array.isArray(c.tags) ? c.tags : void 0,
|
|
1103
|
+
confidence: typeof c.confidence === "number" ? c.confidence : void 0,
|
|
1104
|
+
provenanceState: provenance,
|
|
1105
|
+
contradictedBy: coerceContradictedBy(c.contradicted_by),
|
|
1106
|
+
inferredParagraphs: typeof c.inferred_paragraphs === "number" && Number.isInteger(c.inferred_paragraphs) && c.inferred_paragraphs >= 0 ? c.inferred_paragraphs : void 0
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
function parseConcepts(toolOutput) {
|
|
932
1110
|
try {
|
|
933
|
-
const
|
|
934
|
-
|
|
1111
|
+
const parsed = JSON.parse(toolOutput);
|
|
1112
|
+
const concepts = parsed.concepts ?? [];
|
|
1113
|
+
return concepts.filter(isValidRawConcept).map(mapRawConcept);
|
|
935
1114
|
} catch {
|
|
936
1115
|
return [];
|
|
937
1116
|
}
|
|
938
1117
|
}
|
|
939
|
-
async function classifyFile(root, file, prevState) {
|
|
940
|
-
const filePath = path7.join(root, SOURCES_DIR, file);
|
|
941
|
-
const hash = await hashFile(filePath);
|
|
942
|
-
const prev = prevState.sources[file];
|
|
943
|
-
if (!prev) return "new";
|
|
944
|
-
if (prev.hash !== hash) return "changed";
|
|
945
|
-
return "unchanged";
|
|
946
|
-
}
|
|
947
|
-
function findDeletedFiles(currentFiles, prevState) {
|
|
948
|
-
const currentSet = new Set(currentFiles);
|
|
949
|
-
return Object.keys(prevState.sources).filter((file) => !currentSet.has(file)).map((file) => ({ file, status: "deleted" }));
|
|
950
|
-
}
|
|
951
1118
|
|
|
952
1119
|
// src/compiler/deps.ts
|
|
953
1120
|
function buildConceptToSourcesMap(sources) {
|
|
@@ -1095,7 +1262,7 @@ async function freezeFailedExtractions(root, results, frozenSlugs) {
|
|
|
1095
1262
|
}
|
|
1096
1263
|
|
|
1097
1264
|
// src/compiler/orphan.ts
|
|
1098
|
-
import
|
|
1265
|
+
import path9 from "path";
|
|
1099
1266
|
async function markOrphaned(root, sourceFile, state) {
|
|
1100
1267
|
const sourceEntry = state.sources[sourceFile];
|
|
1101
1268
|
if (!sourceEntry) return;
|
|
@@ -1121,7 +1288,7 @@ async function orphanUnownedFrozenPages(root, frozenSlugs) {
|
|
|
1121
1288
|
}
|
|
1122
1289
|
}
|
|
1123
1290
|
async function orphanPage(root, slug, reason) {
|
|
1124
|
-
const pagePath =
|
|
1291
|
+
const pagePath = path9.join(root, CONCEPTS_DIR, `${slug}.md`);
|
|
1125
1292
|
const content = await safeReadFile(pagePath);
|
|
1126
1293
|
if (!content) return;
|
|
1127
1294
|
const { meta } = parseFrontmatter(content);
|
|
@@ -1133,16 +1300,16 @@ async function orphanPage(root, slug, reason) {
|
|
|
1133
1300
|
|
|
1134
1301
|
// src/compiler/resolver.ts
|
|
1135
1302
|
import { readdir as readdir2, readFile as readFile6 } from "fs/promises";
|
|
1136
|
-
import
|
|
1303
|
+
import path10 from "path";
|
|
1137
1304
|
import { existsSync as existsSync2 } from "fs";
|
|
1138
1305
|
async function buildTitleIndex(root) {
|
|
1139
|
-
const conceptsDir =
|
|
1306
|
+
const conceptsDir = path10.join(root, CONCEPTS_DIR);
|
|
1140
1307
|
if (!existsSync2(conceptsDir)) return [];
|
|
1141
1308
|
const files = await readdir2(conceptsDir);
|
|
1142
1309
|
const pages = [];
|
|
1143
1310
|
for (const file of files) {
|
|
1144
1311
|
if (!file.endsWith(".md")) continue;
|
|
1145
|
-
const filePath =
|
|
1312
|
+
const filePath = path10.join(conceptsDir, file);
|
|
1146
1313
|
const content = await readFile6(filePath, "utf-8");
|
|
1147
1314
|
const { meta } = parseFrontmatter(content);
|
|
1148
1315
|
if (meta.title && typeof meta.title === "string" && !meta.orphaned) {
|
|
@@ -1252,17 +1419,17 @@ async function linkPage(page, titleIndex) {
|
|
|
1252
1419
|
|
|
1253
1420
|
// src/compiler/indexgen.ts
|
|
1254
1421
|
import { readdir as readdir3 } from "fs/promises";
|
|
1255
|
-
import
|
|
1422
|
+
import path11 from "path";
|
|
1256
1423
|
async function generateIndex(root) {
|
|
1257
1424
|
status("*", info("Generating index..."));
|
|
1258
|
-
const conceptsPath =
|
|
1259
|
-
const queriesPath =
|
|
1425
|
+
const conceptsPath = path11.join(root, CONCEPTS_DIR);
|
|
1426
|
+
const queriesPath = path11.join(root, QUERIES_DIR);
|
|
1260
1427
|
const concepts = await collectPageSummaries(conceptsPath);
|
|
1261
1428
|
const queries = await collectPageSummaries(queriesPath);
|
|
1262
1429
|
concepts.sort((a, b) => a.title.localeCompare(b.title));
|
|
1263
1430
|
queries.sort((a, b) => a.title.localeCompare(b.title));
|
|
1264
1431
|
const indexContent = buildIndexContent(concepts, queries);
|
|
1265
|
-
const indexPath =
|
|
1432
|
+
const indexPath = path11.join(root, INDEX_FILE);
|
|
1266
1433
|
await atomicWrite(indexPath, indexContent);
|
|
1267
1434
|
const total = concepts.length + queries.length;
|
|
1268
1435
|
status("+", success(`Index updated with ${total} pages.`));
|
|
@@ -1276,7 +1443,7 @@ async function scanWikiPages(dirPath) {
|
|
|
1276
1443
|
}
|
|
1277
1444
|
const scanned = [];
|
|
1278
1445
|
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
1279
|
-
const content = await safeReadFile(
|
|
1446
|
+
const content = await safeReadFile(path11.join(dirPath, file));
|
|
1280
1447
|
const { meta } = parseFrontmatter(content);
|
|
1281
1448
|
scanned.push({ slug: file.replace(/\.md$/, ""), meta });
|
|
1282
1449
|
}
|
|
@@ -1313,7 +1480,7 @@ function buildIndexContent(concepts, queries) {
|
|
|
1313
1480
|
|
|
1314
1481
|
// src/compiler/obsidian.ts
|
|
1315
1482
|
import { readdir as readdir4 } from "fs/promises";
|
|
1316
|
-
import
|
|
1483
|
+
import path12 from "path";
|
|
1317
1484
|
var ABBREVIATION_MIN_WORDS = 3;
|
|
1318
1485
|
var SWAP_CONJUNCTIONS = [" and ", " or "];
|
|
1319
1486
|
function addObsidianMeta(frontmatter, conceptTitle, tags) {
|
|
@@ -1355,11 +1522,11 @@ function generateAbbreviation(title) {
|
|
|
1355
1522
|
return abbreviation;
|
|
1356
1523
|
}
|
|
1357
1524
|
async function generateMOC(root) {
|
|
1358
|
-
const conceptsPath =
|
|
1525
|
+
const conceptsPath = path12.join(root, CONCEPTS_DIR);
|
|
1359
1526
|
const pages = await loadConceptPages(conceptsPath);
|
|
1360
1527
|
const tagGroups = groupPagesByTag(pages);
|
|
1361
1528
|
const content = buildMOCContent(tagGroups);
|
|
1362
|
-
await atomicWrite(
|
|
1529
|
+
await atomicWrite(path12.join(root, MOC_FILE), content);
|
|
1363
1530
|
}
|
|
1364
1531
|
async function loadConceptPages(conceptsPath) {
|
|
1365
1532
|
let files;
|
|
@@ -1371,7 +1538,7 @@ async function loadConceptPages(conceptsPath) {
|
|
|
1371
1538
|
const pages = [];
|
|
1372
1539
|
for (const file of files) {
|
|
1373
1540
|
if (!file.endsWith(".md")) continue;
|
|
1374
|
-
const content = await safeReadFile(
|
|
1541
|
+
const content = await safeReadFile(path12.join(conceptsPath, file));
|
|
1375
1542
|
if (!content) continue;
|
|
1376
1543
|
const { meta } = parseFrontmatter(content);
|
|
1377
1544
|
if (meta.orphaned) continue;
|
|
@@ -1423,7 +1590,7 @@ function buildMOCContent(tagGroups) {
|
|
|
1423
1590
|
// src/utils/embeddings.ts
|
|
1424
1591
|
import { readFile as readFile7, readdir as readdir5 } from "fs/promises";
|
|
1425
1592
|
import { existsSync as existsSync3 } from "fs";
|
|
1426
|
-
import
|
|
1593
|
+
import path13 from "path";
|
|
1427
1594
|
function cosineSimilarity(a, b) {
|
|
1428
1595
|
if (a.length !== b.length || a.length === 0) return 0;
|
|
1429
1596
|
let dot = 0;
|
|
@@ -1446,18 +1613,23 @@ function findTopK(queryVec, store, k) {
|
|
|
1446
1613
|
return scored.slice(0, k).map((item) => item.entry);
|
|
1447
1614
|
}
|
|
1448
1615
|
async function readEmbeddingStore(root) {
|
|
1449
|
-
const filePath =
|
|
1616
|
+
const filePath = path13.join(root, EMBEDDINGS_FILE);
|
|
1450
1617
|
if (!existsSync3(filePath)) return null;
|
|
1451
1618
|
const raw = await readFile7(filePath, "utf-8");
|
|
1452
1619
|
return JSON.parse(raw);
|
|
1453
1620
|
}
|
|
1454
1621
|
async function writeEmbeddingStore(root, store) {
|
|
1455
|
-
const filePath =
|
|
1622
|
+
const filePath = path13.join(root, EMBEDDINGS_FILE);
|
|
1456
1623
|
await atomicWrite(filePath, JSON.stringify(store, null, 2));
|
|
1457
1624
|
}
|
|
1458
1625
|
async function findRelevantPages(root, question) {
|
|
1459
1626
|
const store = await readEmbeddingStore(root);
|
|
1460
1627
|
if (!store || store.entries.length === 0) return [];
|
|
1628
|
+
const activeModel = resolveEmbeddingModel();
|
|
1629
|
+
if (store.model !== activeModel) {
|
|
1630
|
+
warnStaleEmbeddingStore(store.model, activeModel);
|
|
1631
|
+
return [];
|
|
1632
|
+
}
|
|
1461
1633
|
const queryVec = await getProvider().embed(question);
|
|
1462
1634
|
return findTopK(queryVec, store, EMBEDDING_TOP_K).map((entry) => ({
|
|
1463
1635
|
slug: entry.slug,
|
|
@@ -1468,7 +1640,7 @@ async function findRelevantPages(root, question) {
|
|
|
1468
1640
|
async function collectPageRecords(root) {
|
|
1469
1641
|
const records = [];
|
|
1470
1642
|
for (const dir of [CONCEPTS_DIR, QUERIES_DIR]) {
|
|
1471
|
-
const absDir =
|
|
1643
|
+
const absDir = path13.join(root, dir);
|
|
1472
1644
|
let files;
|
|
1473
1645
|
try {
|
|
1474
1646
|
files = await readdir5(absDir);
|
|
@@ -1476,7 +1648,7 @@ async function collectPageRecords(root) {
|
|
|
1476
1648
|
continue;
|
|
1477
1649
|
}
|
|
1478
1650
|
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
1479
|
-
const content = await safeReadFile(
|
|
1651
|
+
const content = await safeReadFile(path13.join(absDir, file));
|
|
1480
1652
|
const { meta } = parseFrontmatter(content);
|
|
1481
1653
|
if (meta.orphaned || typeof meta.title !== "string") continue;
|
|
1482
1654
|
records.push({
|
|
@@ -1510,8 +1682,25 @@ async function embedPages(records, slugsToEmbed) {
|
|
|
1510
1682
|
}
|
|
1511
1683
|
return fresh;
|
|
1512
1684
|
}
|
|
1685
|
+
var warnedStaleModels = /* @__PURE__ */ new Set();
|
|
1686
|
+
function warnStaleEmbeddingStore(storedModel, activeModel) {
|
|
1687
|
+
const key = `${storedModel}\u2192${activeModel}`;
|
|
1688
|
+
if (warnedStaleModels.has(key)) return;
|
|
1689
|
+
warnedStaleModels.add(key);
|
|
1690
|
+
status(
|
|
1691
|
+
"!",
|
|
1692
|
+
warn(
|
|
1693
|
+
`Embedding store was built with "${storedModel}" but active embedding model is "${activeModel}". Falling back to full-index selection. Run 'llmwiki compile' to rebuild embeddings.`
|
|
1694
|
+
)
|
|
1695
|
+
);
|
|
1696
|
+
}
|
|
1513
1697
|
function resolveEmbeddingModel() {
|
|
1514
|
-
|
|
1698
|
+
const providerName = getActiveProviderName();
|
|
1699
|
+
const configuredModel = process.env.LLMWIKI_EMBEDDING_MODEL?.trim();
|
|
1700
|
+
if (configuredModel && (providerName === "openai" || providerName === "ollama")) {
|
|
1701
|
+
return configuredModel;
|
|
1702
|
+
}
|
|
1703
|
+
return EMBEDDING_MODELS[providerName] ?? EMBEDDING_MODELS.anthropic;
|
|
1515
1704
|
}
|
|
1516
1705
|
function mergeEntries(existing, fresh, liveSlugs) {
|
|
1517
1706
|
const bySlug = /* @__PURE__ */ new Map();
|
|
@@ -1526,13 +1715,15 @@ function mergeEntries(existing, fresh, liveSlugs) {
|
|
|
1526
1715
|
async function updateEmbeddings(root, changedSlugs) {
|
|
1527
1716
|
const records = await collectPageRecords(root);
|
|
1528
1717
|
const liveSlugs = new Set(records.map((r) => r.slug));
|
|
1529
|
-
const
|
|
1718
|
+
const embeddingModel = resolveEmbeddingModel();
|
|
1530
1719
|
const existingStore = await readEmbeddingStore(root);
|
|
1531
|
-
const
|
|
1532
|
-
|
|
1720
|
+
const modelChanged = Boolean(existingStore && existingStore.model !== embeddingModel);
|
|
1721
|
+
const toEmbed = new Set(changedSlugs.filter((slug) => liveSlugs.has(slug)));
|
|
1722
|
+
const previousEntries = modelChanged ? [] : existingStore?.entries ?? [];
|
|
1723
|
+
if (!existingStore || modelChanged) {
|
|
1533
1724
|
for (const record of records) toEmbed.add(record.slug);
|
|
1534
1725
|
}
|
|
1535
|
-
if (toEmbed.size === 0 && previousEntries.every((e) => liveSlugs.has(e.slug))) {
|
|
1726
|
+
if (!modelChanged && toEmbed.size === 0 && previousEntries.every((e) => liveSlugs.has(e.slug))) {
|
|
1536
1727
|
return;
|
|
1537
1728
|
}
|
|
1538
1729
|
const freshEntries = await embedPages(records, toEmbed);
|
|
@@ -1540,7 +1731,7 @@ async function updateEmbeddings(root, changedSlugs) {
|
|
|
1540
1731
|
const dimensions = mergedEntries[0]?.vector.length ?? 0;
|
|
1541
1732
|
const store = {
|
|
1542
1733
|
version: 1,
|
|
1543
|
-
model:
|
|
1734
|
+
model: embeddingModel,
|
|
1544
1735
|
dimensions,
|
|
1545
1736
|
entries: mergedEntries
|
|
1546
1737
|
};
|
|
@@ -1548,15 +1739,207 @@ async function updateEmbeddings(root, changedSlugs) {
|
|
|
1548
1739
|
status("*", dim(`Embeddings updated (${mergedEntries.length} pages).`));
|
|
1549
1740
|
}
|
|
1550
1741
|
|
|
1742
|
+
// src/compiler/candidates.ts
|
|
1743
|
+
import { readdir as readdir6, rename as rename3, unlink as unlink2, writeFile as writeFile4, mkdir as mkdir5 } from "fs/promises";
|
|
1744
|
+
import { existsSync as existsSync4 } from "fs";
|
|
1745
|
+
import path14 from "path";
|
|
1746
|
+
import { randomBytes } from "crypto";
|
|
1747
|
+
var ID_SUFFIX_BYTES = 4;
|
|
1748
|
+
var CANDIDATE_EXT = ".json";
|
|
1749
|
+
function buildCandidateId(slug) {
|
|
1750
|
+
const suffix = randomBytes(ID_SUFFIX_BYTES).toString("hex");
|
|
1751
|
+
return `${slug}-${suffix}`;
|
|
1752
|
+
}
|
|
1753
|
+
function candidatePath(root, id) {
|
|
1754
|
+
return path14.join(root, CANDIDATES_DIR, `${id}${CANDIDATE_EXT}`);
|
|
1755
|
+
}
|
|
1756
|
+
function archivePath(root, id) {
|
|
1757
|
+
return path14.join(root, CANDIDATES_ARCHIVE_DIR, `${id}${CANDIDATE_EXT}`);
|
|
1758
|
+
}
|
|
1759
|
+
async function writeCandidate(root, draft) {
|
|
1760
|
+
const candidate = {
|
|
1761
|
+
id: buildCandidateId(draft.slug),
|
|
1762
|
+
title: draft.title,
|
|
1763
|
+
slug: draft.slug,
|
|
1764
|
+
summary: draft.summary,
|
|
1765
|
+
sources: draft.sources,
|
|
1766
|
+
body: draft.body,
|
|
1767
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1768
|
+
...draft.sourceStates ? { sourceStates: draft.sourceStates } : {}
|
|
1769
|
+
};
|
|
1770
|
+
await atomicWrite(candidatePath(root, candidate.id), JSON.stringify(candidate, null, 2));
|
|
1771
|
+
return candidate;
|
|
1772
|
+
}
|
|
1773
|
+
function failWithError(message) {
|
|
1774
|
+
status("!", error(message));
|
|
1775
|
+
process.exitCode = 1;
|
|
1776
|
+
return null;
|
|
1777
|
+
}
|
|
1778
|
+
async function loadCandidateOrFail(root, id) {
|
|
1779
|
+
const candidate = await readCandidate(root, id);
|
|
1780
|
+
if (!candidate) return failWithError(`Candidate not found: ${id}`);
|
|
1781
|
+
return candidate;
|
|
1782
|
+
}
|
|
1783
|
+
async function loadCandidateUnderLockOrFail(root, id) {
|
|
1784
|
+
const candidate = await readCandidate(root, id);
|
|
1785
|
+
if (!candidate) {
|
|
1786
|
+
return failWithError(`Candidate ${id} was removed by another process during review.`);
|
|
1787
|
+
}
|
|
1788
|
+
return candidate;
|
|
1789
|
+
}
|
|
1790
|
+
async function readCandidate(root, id) {
|
|
1791
|
+
const raw = await safeReadFile(candidatePath(root, id));
|
|
1792
|
+
if (!raw) return null;
|
|
1793
|
+
try {
|
|
1794
|
+
const parsed = JSON.parse(raw);
|
|
1795
|
+
if (!isValidCandidate(parsed)) return null;
|
|
1796
|
+
return parsed;
|
|
1797
|
+
} catch {
|
|
1798
|
+
return null;
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
function isValidCandidate(value) {
|
|
1802
|
+
if (!value || typeof value !== "object") return false;
|
|
1803
|
+
const candidate = value;
|
|
1804
|
+
return typeof candidate.id === "string" && typeof candidate.title === "string" && typeof candidate.slug === "string" && typeof candidate.body === "string" && Array.isArray(candidate.sources);
|
|
1805
|
+
}
|
|
1806
|
+
async function listCandidates(root) {
|
|
1807
|
+
const dir = path14.join(root, CANDIDATES_DIR);
|
|
1808
|
+
if (!existsSync4(dir)) return [];
|
|
1809
|
+
const entries = await readdir6(dir, { withFileTypes: true });
|
|
1810
|
+
const candidates = [];
|
|
1811
|
+
for (const entry of entries) {
|
|
1812
|
+
if (!entry.isFile() || !entry.name.endsWith(CANDIDATE_EXT)) continue;
|
|
1813
|
+
const id = entry.name.slice(0, -CANDIDATE_EXT.length);
|
|
1814
|
+
const candidate = await readCandidate(root, id);
|
|
1815
|
+
if (candidate) candidates.push(candidate);
|
|
1816
|
+
}
|
|
1817
|
+
candidates.sort((a, b) => a.generatedAt.localeCompare(b.generatedAt));
|
|
1818
|
+
return candidates;
|
|
1819
|
+
}
|
|
1820
|
+
async function countCandidates(root) {
|
|
1821
|
+
const candidates = await listCandidates(root);
|
|
1822
|
+
return candidates.length;
|
|
1823
|
+
}
|
|
1824
|
+
async function deleteCandidate(root, id) {
|
|
1825
|
+
const filePath = candidatePath(root, id);
|
|
1826
|
+
if (!existsSync4(filePath)) return false;
|
|
1827
|
+
await unlink2(filePath);
|
|
1828
|
+
return true;
|
|
1829
|
+
}
|
|
1830
|
+
async function archiveCandidate(root, id) {
|
|
1831
|
+
const sourcePath = candidatePath(root, id);
|
|
1832
|
+
if (!existsSync4(sourcePath)) return false;
|
|
1833
|
+
const target = archivePath(root, id);
|
|
1834
|
+
await mkdir5(path14.dirname(target), { recursive: true });
|
|
1835
|
+
try {
|
|
1836
|
+
await rename3(sourcePath, target);
|
|
1837
|
+
} catch {
|
|
1838
|
+
const raw = await safeReadFile(sourcePath);
|
|
1839
|
+
await writeFile4(target, raw, "utf-8");
|
|
1840
|
+
await unlink2(sourcePath);
|
|
1841
|
+
}
|
|
1842
|
+
return true;
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
// src/compiler/page-renderer.ts
|
|
1846
|
+
import { readdir as readdir7 } from "fs/promises";
|
|
1847
|
+
import path15 from "path";
|
|
1848
|
+
|
|
1849
|
+
// src/compiler/provenance.ts
|
|
1850
|
+
function addProvenanceMeta(fields, concept) {
|
|
1851
|
+
if (typeof concept.confidence === "number") {
|
|
1852
|
+
fields.confidence = concept.confidence;
|
|
1853
|
+
}
|
|
1854
|
+
if (concept.provenanceState) {
|
|
1855
|
+
fields.provenanceState = concept.provenanceState;
|
|
1856
|
+
}
|
|
1857
|
+
if (concept.contradictedBy && concept.contradictedBy.length > 0) {
|
|
1858
|
+
fields.contradictedBy = concept.contradictedBy;
|
|
1859
|
+
}
|
|
1860
|
+
if (typeof concept.inferredParagraphs === "number") {
|
|
1861
|
+
fields.inferredParagraphs = concept.inferredParagraphs;
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
function reportContradictionWarnings(conceptTitle, concept) {
|
|
1865
|
+
const refs = concept.contradictedBy;
|
|
1866
|
+
if (!refs || refs.length === 0) return;
|
|
1867
|
+
const slugs = refs.map((r) => r.slug).join(", ");
|
|
1868
|
+
status(
|
|
1869
|
+
"!",
|
|
1870
|
+
warn(`Contradiction reported on "${conceptTitle}" \u2014 conflicts with: ${slugs}`)
|
|
1871
|
+
);
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
// src/compiler/page-renderer.ts
|
|
1875
|
+
var RELATED_PAGE_CONTEXT_LIMIT = 5;
|
|
1876
|
+
async function renderMergedPageContent(root, entry) {
|
|
1877
|
+
const pagePath = path15.join(root, CONCEPTS_DIR, `${entry.slug}.md`);
|
|
1878
|
+
const existingPage = await safeReadFile(pagePath);
|
|
1879
|
+
const relatedPages = await loadRelatedPages(root, entry.slug);
|
|
1880
|
+
const system = buildPagePrompt(
|
|
1881
|
+
entry.concept.concept,
|
|
1882
|
+
entry.combinedContent,
|
|
1883
|
+
existingPage,
|
|
1884
|
+
relatedPages
|
|
1885
|
+
);
|
|
1886
|
+
const pageBody = await callClaude({
|
|
1887
|
+
system,
|
|
1888
|
+
messages: [
|
|
1889
|
+
{ role: "user", content: `Write the wiki page for "${entry.concept.concept}".` }
|
|
1890
|
+
]
|
|
1891
|
+
});
|
|
1892
|
+
const frontmatter = buildMergedFrontmatter(entry, existingPage);
|
|
1893
|
+
reportContradictionWarnings(entry.concept.concept, entry.concept);
|
|
1894
|
+
return `${frontmatter}
|
|
1895
|
+
|
|
1896
|
+
${pageBody}
|
|
1897
|
+
`;
|
|
1898
|
+
}
|
|
1899
|
+
function buildMergedFrontmatter(entry, existingPage) {
|
|
1900
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1901
|
+
const existing = existingPage ? parseFrontmatter(existingPage) : null;
|
|
1902
|
+
const createdAt = existing?.meta.createdAt && typeof existing.meta.createdAt === "string" ? existing.meta.createdAt : now;
|
|
1903
|
+
const frontmatterFields = {
|
|
1904
|
+
title: entry.concept.concept,
|
|
1905
|
+
summary: entry.concept.summary,
|
|
1906
|
+
sources: entry.sourceFiles,
|
|
1907
|
+
createdAt,
|
|
1908
|
+
updatedAt: now
|
|
1909
|
+
};
|
|
1910
|
+
addObsidianMeta(frontmatterFields, entry.concept.concept, entry.concept.tags ?? []);
|
|
1911
|
+
addProvenanceMeta(frontmatterFields, entry.concept);
|
|
1912
|
+
return buildFrontmatter(frontmatterFields);
|
|
1913
|
+
}
|
|
1914
|
+
async function loadRelatedPages(root, excludeSlug) {
|
|
1915
|
+
const conceptsPath = path15.join(root, CONCEPTS_DIR);
|
|
1916
|
+
let files;
|
|
1917
|
+
try {
|
|
1918
|
+
files = await readdir7(conceptsPath);
|
|
1919
|
+
} catch {
|
|
1920
|
+
return "";
|
|
1921
|
+
}
|
|
1922
|
+
const related = files.filter((f) => f.endsWith(".md") && f !== `${excludeSlug}.md`).slice(0, RELATED_PAGE_CONTEXT_LIMIT);
|
|
1923
|
+
const contents = [];
|
|
1924
|
+
for (const f of related) {
|
|
1925
|
+
const content = await safeReadFile(path15.join(conceptsPath, f));
|
|
1926
|
+
if (!content) continue;
|
|
1927
|
+
const { meta } = parseFrontmatter(content);
|
|
1928
|
+
if (meta.orphaned) continue;
|
|
1929
|
+
contents.push(content);
|
|
1930
|
+
}
|
|
1931
|
+
return contents.join("\n\n---\n\n");
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1551
1934
|
// src/compiler/index.ts
|
|
1552
1935
|
import pLimit from "p-limit";
|
|
1553
1936
|
function emptyCompileResult() {
|
|
1554
1937
|
return { compiled: 0, skipped: 0, deleted: 0, concepts: [], pages: [], errors: [] };
|
|
1555
1938
|
}
|
|
1556
|
-
async function compile(root) {
|
|
1557
|
-
await compileAndReport(root);
|
|
1939
|
+
async function compile(root, options = {}) {
|
|
1940
|
+
await compileAndReport(root, options);
|
|
1558
1941
|
}
|
|
1559
|
-
async function compileAndReport(root) {
|
|
1942
|
+
async function compileAndReport(root, options = {}) {
|
|
1560
1943
|
header("llmwiki compile");
|
|
1561
1944
|
const locked = await acquireLock(root);
|
|
1562
1945
|
if (!locked) {
|
|
@@ -1567,7 +1950,7 @@ async function compileAndReport(root) {
|
|
|
1567
1950
|
};
|
|
1568
1951
|
}
|
|
1569
1952
|
try {
|
|
1570
|
-
return await runCompilePipeline(root);
|
|
1953
|
+
return await runCompilePipeline(root, options);
|
|
1571
1954
|
} finally {
|
|
1572
1955
|
await releaseLock(root);
|
|
1573
1956
|
}
|
|
@@ -1579,18 +1962,21 @@ function bucketChanges(changes) {
|
|
|
1579
1962
|
unchanged: changes.filter((c) => c.status === "unchanged")
|
|
1580
1963
|
};
|
|
1581
1964
|
}
|
|
1582
|
-
async function generatePagesPhase(root, extractions, frozenSlugs) {
|
|
1965
|
+
async function generatePagesPhase(root, extractions, frozenSlugs, options) {
|
|
1583
1966
|
const merged = mergeExtractions(extractions, frozenSlugs);
|
|
1967
|
+
const sourceStates = options.review ? await buildExtractionSourceStates(root, extractions) : {};
|
|
1584
1968
|
const limit = pLimit(COMPILE_CONCURRENCY);
|
|
1585
1969
|
const errors = [];
|
|
1970
|
+
const candidates = [];
|
|
1586
1971
|
const pages = await Promise.all(
|
|
1587
1972
|
merged.map((entry) => limit(async () => {
|
|
1588
|
-
const
|
|
1589
|
-
if (
|
|
1973
|
+
const result = await generateMergedPage(root, entry, options, sourceStates);
|
|
1974
|
+
if (result.error) errors.push(result.error);
|
|
1975
|
+
if (result.candidateId) candidates.push(result.candidateId);
|
|
1590
1976
|
return entry;
|
|
1591
1977
|
}))
|
|
1592
1978
|
);
|
|
1593
|
-
return { pages, errors };
|
|
1979
|
+
return { pages, errors, candidates };
|
|
1594
1980
|
}
|
|
1595
1981
|
async function persistExtractionStates(root, extractions) {
|
|
1596
1982
|
for (const result of extractions) {
|
|
@@ -1598,12 +1984,16 @@ async function persistExtractionStates(root, extractions) {
|
|
|
1598
1984
|
await persistSourceState(root, result.sourcePath, result.sourceFile, result.concepts);
|
|
1599
1985
|
}
|
|
1600
1986
|
}
|
|
1601
|
-
function summarizeCompile(buckets, generation, extractions) {
|
|
1987
|
+
function summarizeCompile(buckets, generation, extractions, options) {
|
|
1602
1988
|
header("Compilation complete");
|
|
1603
1989
|
status("\u2713", success(
|
|
1604
1990
|
`${buckets.toCompile.length} compiled, ${buckets.unchanged.length} skipped, ${buckets.deleted.length} deleted`
|
|
1605
1991
|
));
|
|
1606
|
-
if (
|
|
1992
|
+
if (options.review && generation.candidates.length > 0) {
|
|
1993
|
+
status("?", info(
|
|
1994
|
+
`${generation.candidates.length} candidate(s) awaiting review \u2014 run \`llmwiki review list\``
|
|
1995
|
+
));
|
|
1996
|
+
} else if (buckets.toCompile.length > 0) {
|
|
1607
1997
|
status("\u2192", dim('Next: llmwiki query "your question here"'));
|
|
1608
1998
|
}
|
|
1609
1999
|
const errors = [...generation.errors];
|
|
@@ -1612,7 +2002,7 @@ function summarizeCompile(buckets, generation, extractions) {
|
|
|
1612
2002
|
errors.push(`No concepts extracted from ${result.sourceFile}`);
|
|
1613
2003
|
}
|
|
1614
2004
|
}
|
|
1615
|
-
|
|
2005
|
+
const baseResult = {
|
|
1616
2006
|
compiled: buckets.toCompile.length,
|
|
1617
2007
|
skipped: buckets.unchanged.length,
|
|
1618
2008
|
deleted: buckets.deleted.length,
|
|
@@ -1620,8 +2010,12 @@ function summarizeCompile(buckets, generation, extractions) {
|
|
|
1620
2010
|
pages: generation.pages.map((entry) => entry.slug),
|
|
1621
2011
|
errors
|
|
1622
2012
|
};
|
|
2013
|
+
if (options.review) {
|
|
2014
|
+
baseResult.candidates = generation.candidates;
|
|
2015
|
+
}
|
|
2016
|
+
return baseResult;
|
|
1623
2017
|
}
|
|
1624
|
-
async function runCompilePipeline(root) {
|
|
2018
|
+
async function runCompilePipeline(root, options) {
|
|
1625
2019
|
const state = await readState(root);
|
|
1626
2020
|
const changes = await detectChanges(root, state);
|
|
1627
2021
|
augmentWithAffectedSources(changes, findAffectedSources(state, changes));
|
|
@@ -1631,19 +2025,25 @@ async function runCompilePipeline(root) {
|
|
|
1631
2025
|
return { ...emptyCompileResult(), skipped: buckets.unchanged.length };
|
|
1632
2026
|
}
|
|
1633
2027
|
printChangesSummary(changes);
|
|
1634
|
-
|
|
2028
|
+
if (!options.review) {
|
|
2029
|
+
await markDeletedAsOrphaned(root, buckets.deleted, state);
|
|
2030
|
+
}
|
|
1635
2031
|
const frozenSlugs = findFrozenSlugs(state, changes);
|
|
1636
2032
|
reportFrozenSlugs(frozenSlugs);
|
|
1637
2033
|
const extractions = await runExtractionPhases(root, buckets.toCompile, state, changes);
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
2034
|
+
if (!options.review) {
|
|
2035
|
+
await freezeFailedExtractions(root, extractions, frozenSlugs);
|
|
2036
|
+
}
|
|
2037
|
+
const generation = await generatePagesPhase(root, extractions, frozenSlugs, options);
|
|
2038
|
+
if (!options.review) {
|
|
2039
|
+
await persistExtractionStates(root, extractions);
|
|
2040
|
+
if (frozenSlugs.size > 0) {
|
|
2041
|
+
await orphanUnownedFrozenPages(root, frozenSlugs);
|
|
2042
|
+
}
|
|
2043
|
+
await persistFrozenSlugs(root, frozenSlugs, extractions);
|
|
2044
|
+
await finalizeWiki(root, generation.pages);
|
|
1643
2045
|
}
|
|
1644
|
-
|
|
1645
|
-
await finalizeWiki(root, generation.pages);
|
|
1646
|
-
return summarizeCompile(buckets, generation, extractions);
|
|
2046
|
+
return summarizeCompile(buckets, generation, extractions, options);
|
|
1647
2047
|
}
|
|
1648
2048
|
function augmentWithAffectedSources(changes, affected) {
|
|
1649
2049
|
for (const file of affected) {
|
|
@@ -1705,9 +2105,9 @@ function printChangesSummary(changes) {
|
|
|
1705
2105
|
}
|
|
1706
2106
|
async function extractForSource(root, sourceFile) {
|
|
1707
2107
|
status("*", info(`Extracting: ${sourceFile}`));
|
|
1708
|
-
const sourcePath =
|
|
2108
|
+
const sourcePath = path16.join(root, SOURCES_DIR, sourceFile);
|
|
1709
2109
|
const sourceContent = await readFile8(sourcePath, "utf-8");
|
|
1710
|
-
const existingIndex = await safeReadFile(
|
|
2110
|
+
const existingIndex = await safeReadFile(path16.join(root, INDEX_FILE));
|
|
1711
2111
|
const concepts = await extractConcepts(sourceContent, existingIndex);
|
|
1712
2112
|
if (concepts.length > 0) {
|
|
1713
2113
|
const names = concepts.map((c) => c.concept).join(", ");
|
|
@@ -1715,6 +2115,26 @@ async function extractForSource(root, sourceFile) {
|
|
|
1715
2115
|
}
|
|
1716
2116
|
return { sourceFile, sourcePath, sourceContent, concepts };
|
|
1717
2117
|
}
|
|
2118
|
+
function reconcileConceptMetadata(existing, incoming) {
|
|
2119
|
+
const reconciled = { ...existing };
|
|
2120
|
+
if (typeof incoming.confidence === "number") {
|
|
2121
|
+
reconciled.confidence = typeof existing.confidence === "number" ? Math.min(existing.confidence, incoming.confidence) : incoming.confidence;
|
|
2122
|
+
}
|
|
2123
|
+
reconciled.provenanceState = "merged";
|
|
2124
|
+
const refs = [...existing.contradictedBy ?? []];
|
|
2125
|
+
const seenSlugs = new Set(refs.map((r) => r.slug));
|
|
2126
|
+
for (const ref of incoming.contradictedBy ?? []) {
|
|
2127
|
+
if (!seenSlugs.has(ref.slug)) {
|
|
2128
|
+
refs.push(ref);
|
|
2129
|
+
seenSlugs.add(ref.slug);
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
reconciled.contradictedBy = refs.length > 0 ? refs : void 0;
|
|
2133
|
+
if (typeof incoming.inferredParagraphs === "number") {
|
|
2134
|
+
reconciled.inferredParagraphs = typeof existing.inferredParagraphs === "number" ? Math.max(existing.inferredParagraphs, incoming.inferredParagraphs) : incoming.inferredParagraphs;
|
|
2135
|
+
}
|
|
2136
|
+
return reconciled;
|
|
2137
|
+
}
|
|
1718
2138
|
function mergeExtractions(extractions, frozenSlugs) {
|
|
1719
2139
|
const bySlug = /* @__PURE__ */ new Map();
|
|
1720
2140
|
for (const result of extractions) {
|
|
@@ -1724,6 +2144,7 @@ function mergeExtractions(extractions, frozenSlugs) {
|
|
|
1724
2144
|
if (frozenSlugs.has(slug)) continue;
|
|
1725
2145
|
const existing = bySlug.get(slug);
|
|
1726
2146
|
if (existing) {
|
|
2147
|
+
existing.concept = reconcileConceptMetadata(existing.concept, concept);
|
|
1727
2148
|
existing.sourceFiles.push(result.sourceFile);
|
|
1728
2149
|
existing.combinedContent += `
|
|
1729
2150
|
|
|
@@ -1744,39 +2165,26 @@ ${result.sourceContent}`
|
|
|
1744
2165
|
}
|
|
1745
2166
|
return Array.from(bySlug.values());
|
|
1746
2167
|
}
|
|
1747
|
-
async function generateMergedPage(root, entry) {
|
|
1748
|
-
const
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
const
|
|
1758
|
-
system,
|
|
1759
|
-
messages: [
|
|
1760
|
-
{ role: "user", content: `Write the wiki page for "${entry.concept.concept}".` }
|
|
1761
|
-
]
|
|
1762
|
-
});
|
|
1763
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1764
|
-
const existing = existingPage ? parseFrontmatter(existingPage) : null;
|
|
1765
|
-
const createdAt = existing?.meta.createdAt && typeof existing.meta.createdAt === "string" ? existing.meta.createdAt : now;
|
|
1766
|
-
const frontmatterFields = {
|
|
2168
|
+
async function generateMergedPage(root, entry, options, sourceStates) {
|
|
2169
|
+
const fullPage = await renderMergedPageContent(root, entry);
|
|
2170
|
+
if (options.review) {
|
|
2171
|
+
return await persistReviewCandidate(root, entry, fullPage, sourceStates);
|
|
2172
|
+
}
|
|
2173
|
+
const pagePath = path16.join(root, CONCEPTS_DIR, `${entry.slug}.md`);
|
|
2174
|
+
const error2 = await writePageIfValid(pagePath, fullPage, entry.concept.concept);
|
|
2175
|
+
return { error: error2 ?? void 0 };
|
|
2176
|
+
}
|
|
2177
|
+
async function persistReviewCandidate(root, entry, fullPage, sourceStates) {
|
|
2178
|
+
const candidate = await writeCandidate(root, {
|
|
1767
2179
|
title: entry.concept.concept,
|
|
2180
|
+
slug: entry.slug,
|
|
1768
2181
|
summary: entry.concept.summary,
|
|
1769
2182
|
sources: entry.sourceFiles,
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
};
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
const fullPage = `${frontmatter}
|
|
1776
|
-
|
|
1777
|
-
${pageBody}
|
|
1778
|
-
`;
|
|
1779
|
-
return await writePageIfValid(pagePath, fullPage, entry.concept.concept);
|
|
2183
|
+
body: fullPage,
|
|
2184
|
+
sourceStates: pickStatesForSources(sourceStates, entry.sourceFiles)
|
|
2185
|
+
});
|
|
2186
|
+
status("?", info(`Candidate ready: ${candidate.id} (${entry.slug})`));
|
|
2187
|
+
return { candidateId: candidate.id };
|
|
1780
2188
|
}
|
|
1781
2189
|
async function extractConcepts(sourceContent, existingIndex) {
|
|
1782
2190
|
const system = buildExtractionPrompt(sourceContent, existingIndex);
|
|
@@ -1787,25 +2195,6 @@ async function extractConcepts(sourceContent, existingIndex) {
|
|
|
1787
2195
|
});
|
|
1788
2196
|
return parseConcepts(rawOutput);
|
|
1789
2197
|
}
|
|
1790
|
-
async function loadRelatedPages(root, excludeSlug) {
|
|
1791
|
-
const conceptsPath = path13.join(root, CONCEPTS_DIR);
|
|
1792
|
-
let files;
|
|
1793
|
-
try {
|
|
1794
|
-
files = await readdir6(conceptsPath);
|
|
1795
|
-
} catch {
|
|
1796
|
-
return "";
|
|
1797
|
-
}
|
|
1798
|
-
const related = files.filter((f) => f.endsWith(".md") && f !== `${excludeSlug}.md`).slice(0, 5);
|
|
1799
|
-
const contents = [];
|
|
1800
|
-
for (const f of related) {
|
|
1801
|
-
const content = await safeReadFile(path13.join(conceptsPath, f));
|
|
1802
|
-
if (!content) continue;
|
|
1803
|
-
const { meta } = parseFrontmatter(content);
|
|
1804
|
-
if (meta.orphaned) continue;
|
|
1805
|
-
contents.push(content);
|
|
1806
|
-
}
|
|
1807
|
-
return contents.join("\n\n---\n\n");
|
|
1808
|
-
}
|
|
1809
2198
|
async function writePageIfValid(pagePath, content, conceptTitle) {
|
|
1810
2199
|
if (!validateWikiPage(content)) {
|
|
1811
2200
|
status("!", warn(`Invalid page for "${conceptTitle}" \u2014 skipped.`));
|
|
@@ -1833,20 +2222,20 @@ async function persistSourceState(root, sourcePath, sourceFile, concepts) {
|
|
|
1833
2222
|
}
|
|
1834
2223
|
|
|
1835
2224
|
// src/commands/compile.ts
|
|
1836
|
-
async function compileCommand() {
|
|
1837
|
-
if (!
|
|
2225
|
+
async function compileCommand(options = {}) {
|
|
2226
|
+
if (!existsSync5(SOURCES_DIR)) {
|
|
1838
2227
|
status(
|
|
1839
2228
|
"!",
|
|
1840
2229
|
warn("No sources found. Run `llmwiki ingest <url>` first.")
|
|
1841
2230
|
);
|
|
1842
2231
|
return;
|
|
1843
2232
|
}
|
|
1844
|
-
await compile(process.cwd());
|
|
2233
|
+
await compile(process.cwd(), options);
|
|
1845
2234
|
}
|
|
1846
2235
|
|
|
1847
2236
|
// src/commands/query.ts
|
|
1848
|
-
import { existsSync as
|
|
1849
|
-
import
|
|
2237
|
+
import { existsSync as existsSync6 } from "fs";
|
|
2238
|
+
import path17 from "path";
|
|
1850
2239
|
var PAGE_DIRS = [CONCEPTS_DIR, QUERIES_DIR];
|
|
1851
2240
|
var PAGE_SELECTION_TOOL = {
|
|
1852
2241
|
name: "select_pages",
|
|
@@ -1901,7 +2290,7 @@ async function selectRelevantPages(root, question) {
|
|
|
1901
2290
|
const { pages: rawPages2, reasoning: reasoning2 } = await selectPages(question, filteredIndex);
|
|
1902
2291
|
return { pages: rawPages2, rawPages: rawPages2, reasoning: reasoning2 };
|
|
1903
2292
|
}
|
|
1904
|
-
const indexContent = await safeReadFile(
|
|
2293
|
+
const indexContent = await safeReadFile(path17.join(root, INDEX_FILE));
|
|
1905
2294
|
const { pages: rawPages, reasoning } = await selectPages(question, indexContent);
|
|
1906
2295
|
return { pages: rawPages.map((p) => slugify(p)), rawPages, reasoning };
|
|
1907
2296
|
}
|
|
@@ -1919,7 +2308,7 @@ async function loadSelectedPages(root, slugs) {
|
|
|
1919
2308
|
for (const slug of slugs) {
|
|
1920
2309
|
let content = "";
|
|
1921
2310
|
for (const dir of PAGE_DIRS) {
|
|
1922
|
-
const candidate = await safeReadFile(
|
|
2311
|
+
const candidate = await safeReadFile(path17.join(root, dir, `${slug}.md`));
|
|
1923
2312
|
if (!candidate) continue;
|
|
1924
2313
|
const { meta } = parseFrontmatter(candidate);
|
|
1925
2314
|
if (meta.orphaned) continue;
|
|
@@ -1955,7 +2344,7 @@ function summarizeAnswer(answer) {
|
|
|
1955
2344
|
}
|
|
1956
2345
|
async function saveQueryPage(root, question, answer) {
|
|
1957
2346
|
const slug = slugify(question);
|
|
1958
|
-
const filePath =
|
|
2347
|
+
const filePath = path17.join(root, QUERIES_DIR, `${slug}.md`);
|
|
1959
2348
|
const frontmatter = buildFrontmatter({
|
|
1960
2349
|
title: question,
|
|
1961
2350
|
summary: summarizeAnswer(answer),
|
|
@@ -1981,7 +2370,7 @@ ${answer}
|
|
|
1981
2370
|
return slug;
|
|
1982
2371
|
}
|
|
1983
2372
|
async function generateAnswer(root, question, options = {}) {
|
|
1984
|
-
if (!
|
|
2373
|
+
if (!existsSync6(path17.join(root, INDEX_FILE))) {
|
|
1985
2374
|
throw new Error("Wiki index not found. Run `llmwiki compile` first.");
|
|
1986
2375
|
}
|
|
1987
2376
|
const { pages, reasoning } = await selectRelevantPages(root, question);
|
|
@@ -1998,7 +2387,7 @@ async function generateAnswer(root, question, options = {}) {
|
|
|
1998
2387
|
return { answer, selectedPages: pages, reasoning, saved };
|
|
1999
2388
|
}
|
|
2000
2389
|
async function queryCommand(root, question, options) {
|
|
2001
|
-
if (!
|
|
2390
|
+
if (!existsSync6(path17.join(root, INDEX_FILE))) {
|
|
2002
2391
|
status("!", error("Wiki index not found. Run `llmwiki compile` first."));
|
|
2003
2392
|
return;
|
|
2004
2393
|
}
|
|
@@ -2026,12 +2415,12 @@ async function queryCommand(root, question, options) {
|
|
|
2026
2415
|
|
|
2027
2416
|
// src/commands/watch.ts
|
|
2028
2417
|
import { watch as chokidarWatch } from "chokidar";
|
|
2029
|
-
import { existsSync as
|
|
2030
|
-
import
|
|
2418
|
+
import { existsSync as existsSync7 } from "fs";
|
|
2419
|
+
import path18 from "path";
|
|
2031
2420
|
var DEBOUNCE_MS = 500;
|
|
2032
2421
|
async function watchCommand() {
|
|
2033
|
-
const sourcesPath =
|
|
2034
|
-
if (!
|
|
2422
|
+
const sourcesPath = path18.resolve(SOURCES_DIR);
|
|
2423
|
+
if (!existsSync7(sourcesPath)) {
|
|
2035
2424
|
status(
|
|
2036
2425
|
"!",
|
|
2037
2426
|
warn("No sources/ directory found. Run `llmwiki ingest <url>` first.")
|
|
@@ -2065,7 +2454,7 @@ async function watchCommand() {
|
|
|
2065
2454
|
const scheduleCompile = (eventPath, event) => {
|
|
2066
2455
|
status(
|
|
2067
2456
|
"~",
|
|
2068
|
-
dim(`${event}: ${
|
|
2457
|
+
dim(`${event}: ${path18.basename(eventPath)}`)
|
|
2069
2458
|
);
|
|
2070
2459
|
if (debounceTimer) clearTimeout(debounceTimer);
|
|
2071
2460
|
debounceTimer = setTimeout(triggerCompile, DEBOUNCE_MS);
|
|
@@ -2080,9 +2469,9 @@ async function watchCommand() {
|
|
|
2080
2469
|
}
|
|
2081
2470
|
|
|
2082
2471
|
// src/linter/rules.ts
|
|
2083
|
-
import { readdir as
|
|
2084
|
-
import { existsSync as
|
|
2085
|
-
import
|
|
2472
|
+
import { readdir as readdir8, readFile as readFile9 } from "fs/promises";
|
|
2473
|
+
import { existsSync as existsSync8 } from "fs";
|
|
2474
|
+
import path19 from "path";
|
|
2086
2475
|
var MIN_BODY_LENGTH = 50;
|
|
2087
2476
|
var WIKILINK_PATTERN = /\[\[([^\]]+)\]\]/g;
|
|
2088
2477
|
var CITATION_PATTERN = /\^\[([^\]]+)\]/g;
|
|
@@ -2098,12 +2487,12 @@ function findMatchesInContent(content, pattern) {
|
|
|
2098
2487
|
return results;
|
|
2099
2488
|
}
|
|
2100
2489
|
async function readMarkdownFiles(dirPath) {
|
|
2101
|
-
if (!
|
|
2102
|
-
const entries = await
|
|
2490
|
+
if (!existsSync8(dirPath)) return [];
|
|
2491
|
+
const entries = await readdir8(dirPath);
|
|
2103
2492
|
const mdFiles = entries.filter((f) => f.endsWith(".md"));
|
|
2104
2493
|
const results = await Promise.all(
|
|
2105
2494
|
mdFiles.map(async (fileName) => {
|
|
2106
|
-
const filePath =
|
|
2495
|
+
const filePath = path19.join(dirPath, fileName);
|
|
2107
2496
|
const content = await readFile9(filePath, "utf-8");
|
|
2108
2497
|
return { filePath, content };
|
|
2109
2498
|
})
|
|
@@ -2111,14 +2500,14 @@ async function readMarkdownFiles(dirPath) {
|
|
|
2111
2500
|
return results;
|
|
2112
2501
|
}
|
|
2113
2502
|
async function collectAllPages(root) {
|
|
2114
|
-
const conceptPages = await readMarkdownFiles(
|
|
2115
|
-
const queryPages = await readMarkdownFiles(
|
|
2503
|
+
const conceptPages = await readMarkdownFiles(path19.join(root, CONCEPTS_DIR));
|
|
2504
|
+
const queryPages = await readMarkdownFiles(path19.join(root, QUERIES_DIR));
|
|
2116
2505
|
return [...conceptPages, ...queryPages];
|
|
2117
2506
|
}
|
|
2118
2507
|
function buildPageSlugSet(pages) {
|
|
2119
2508
|
const slugs = /* @__PURE__ */ new Set();
|
|
2120
2509
|
for (const page of pages) {
|
|
2121
|
-
const baseName =
|
|
2510
|
+
const baseName = path19.basename(page.filePath, ".md");
|
|
2122
2511
|
slugs.add(baseName.toLowerCase());
|
|
2123
2512
|
}
|
|
2124
2513
|
return slugs;
|
|
@@ -2221,21 +2610,93 @@ async function checkEmptyPages(root) {
|
|
|
2221
2610
|
}
|
|
2222
2611
|
return results;
|
|
2223
2612
|
}
|
|
2613
|
+
async function checkLowConfidencePages(root) {
|
|
2614
|
+
const pages = await collectAllPages(root);
|
|
2615
|
+
const results = [];
|
|
2616
|
+
for (const page of pages) {
|
|
2617
|
+
const { meta } = parseFrontmatter(page.content);
|
|
2618
|
+
const { confidence } = parseProvenanceMetadata(meta);
|
|
2619
|
+
if (confidence === void 0 || confidence >= LOW_CONFIDENCE_THRESHOLD) continue;
|
|
2620
|
+
results.push({
|
|
2621
|
+
rule: "low-confidence",
|
|
2622
|
+
severity: "warning",
|
|
2623
|
+
file: page.filePath,
|
|
2624
|
+
message: `Page confidence ${confidence.toFixed(2)} is below ${LOW_CONFIDENCE_THRESHOLD}`
|
|
2625
|
+
});
|
|
2626
|
+
}
|
|
2627
|
+
return results;
|
|
2628
|
+
}
|
|
2629
|
+
async function checkContradictedPages(root) {
|
|
2630
|
+
const pages = await collectAllPages(root);
|
|
2631
|
+
const results = [];
|
|
2632
|
+
for (const page of pages) {
|
|
2633
|
+
const { meta } = parseFrontmatter(page.content);
|
|
2634
|
+
const { contradictedBy } = parseProvenanceMetadata(meta);
|
|
2635
|
+
if (!contradictedBy || contradictedBy.length === 0) continue;
|
|
2636
|
+
const slugs = contradictedBy.map((r) => r.slug).join(", ");
|
|
2637
|
+
results.push({
|
|
2638
|
+
rule: "contradicted-page",
|
|
2639
|
+
severity: "warning",
|
|
2640
|
+
file: page.filePath,
|
|
2641
|
+
message: `Page contradicts: ${slugs}`
|
|
2642
|
+
});
|
|
2643
|
+
}
|
|
2644
|
+
return results;
|
|
2645
|
+
}
|
|
2646
|
+
async function checkInferredWithoutCitations(root) {
|
|
2647
|
+
const pages = await collectAllPages(root);
|
|
2648
|
+
const results = [];
|
|
2649
|
+
for (const page of pages) {
|
|
2650
|
+
const { meta, body } = parseFrontmatter(page.content);
|
|
2651
|
+
const provenance = parseProvenanceMetadata(meta);
|
|
2652
|
+
const inferred = provenance.inferredParagraphs ?? countUncitedProseParagraphs(body);
|
|
2653
|
+
if (inferred <= MAX_INFERRED_PARAGRAPHS_WITHOUT_CITATIONS) continue;
|
|
2654
|
+
results.push({
|
|
2655
|
+
rule: "excess-inferred-paragraphs",
|
|
2656
|
+
severity: "warning",
|
|
2657
|
+
file: page.filePath,
|
|
2658
|
+
message: `Page has ${inferred} inferred paragraphs without citations (max ${MAX_INFERRED_PARAGRAPHS_WITHOUT_CITATIONS})`
|
|
2659
|
+
});
|
|
2660
|
+
}
|
|
2661
|
+
return results;
|
|
2662
|
+
}
|
|
2663
|
+
var PROSE_PARAGRAPH_LEAD = /^[A-Za-z]/;
|
|
2664
|
+
function countUncitedProseParagraphs(body) {
|
|
2665
|
+
const paragraphs = body.split(/\n\s*\n/);
|
|
2666
|
+
let count = 0;
|
|
2667
|
+
for (const block of paragraphs) {
|
|
2668
|
+
const trimmed = block.trim();
|
|
2669
|
+
if (trimmed.length === 0) continue;
|
|
2670
|
+
if (!PROSE_PARAGRAPH_LEAD.test(trimmed)) continue;
|
|
2671
|
+
if (CITATION_PATTERN.test(trimmed)) {
|
|
2672
|
+
CITATION_PATTERN.lastIndex = 0;
|
|
2673
|
+
continue;
|
|
2674
|
+
}
|
|
2675
|
+
CITATION_PATTERN.lastIndex = 0;
|
|
2676
|
+
count += 1;
|
|
2677
|
+
}
|
|
2678
|
+
return count;
|
|
2679
|
+
}
|
|
2680
|
+
function splitCitationFilenames(captured) {
|
|
2681
|
+
return captured.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
2682
|
+
}
|
|
2224
2683
|
async function checkBrokenCitations(root) {
|
|
2225
2684
|
const pages = await collectAllPages(root);
|
|
2226
|
-
const sourcesDir =
|
|
2685
|
+
const sourcesDir = path19.join(root, SOURCES_DIR);
|
|
2227
2686
|
const results = [];
|
|
2228
2687
|
for (const page of pages) {
|
|
2229
2688
|
for (const { captured, line } of findMatchesInContent(page.content, CITATION_PATTERN)) {
|
|
2230
|
-
const
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2689
|
+
for (const filename of splitCitationFilenames(captured)) {
|
|
2690
|
+
const citedPath = path19.join(sourcesDir, filename);
|
|
2691
|
+
if (!existsSync8(citedPath)) {
|
|
2692
|
+
results.push({
|
|
2693
|
+
rule: "broken-citation",
|
|
2694
|
+
severity: "error",
|
|
2695
|
+
file: page.filePath,
|
|
2696
|
+
message: `Broken citation ^[${filename}] \u2014 source file not found`,
|
|
2697
|
+
line
|
|
2698
|
+
});
|
|
2699
|
+
}
|
|
2239
2700
|
}
|
|
2240
2701
|
}
|
|
2241
2702
|
}
|
|
@@ -2249,7 +2710,10 @@ var ALL_RULES = [
|
|
|
2249
2710
|
checkMissingSummaries,
|
|
2250
2711
|
checkDuplicateConcepts,
|
|
2251
2712
|
checkEmptyPages,
|
|
2252
|
-
checkBrokenCitations
|
|
2713
|
+
checkBrokenCitations,
|
|
2714
|
+
checkLowConfidencePages,
|
|
2715
|
+
checkContradictedPages,
|
|
2716
|
+
checkInferredWithoutCitations
|
|
2253
2717
|
];
|
|
2254
2718
|
function countBySeverity(results, severity) {
|
|
2255
2719
|
return results.filter((r) => r.severity === severity).length;
|
|
@@ -2302,12 +2766,133 @@ async function lintCommand() {
|
|
|
2302
2766
|
}
|
|
2303
2767
|
}
|
|
2304
2768
|
|
|
2769
|
+
// src/commands/review-list.ts
|
|
2770
|
+
async function reviewListCommand() {
|
|
2771
|
+
header("Pending review candidates");
|
|
2772
|
+
const candidates = await listCandidates(process.cwd());
|
|
2773
|
+
if (candidates.length === 0) {
|
|
2774
|
+
status("\u2713", success("No pending candidates."));
|
|
2775
|
+
return;
|
|
2776
|
+
}
|
|
2777
|
+
for (const candidate of candidates) {
|
|
2778
|
+
const sources = candidate.sources.join(", ");
|
|
2779
|
+
const meta = dim(`${candidate.generatedAt} | sources: ${sources}`);
|
|
2780
|
+
status("?", `${info(candidate.id)} \u2192 ${candidate.slug} ${meta}`);
|
|
2781
|
+
}
|
|
2782
|
+
status(
|
|
2783
|
+
"\u2192",
|
|
2784
|
+
dim(`Use \`llmwiki review show <id>\` to inspect a candidate.`)
|
|
2785
|
+
);
|
|
2786
|
+
}
|
|
2787
|
+
|
|
2788
|
+
// src/commands/review-show.ts
|
|
2789
|
+
async function reviewShowCommand(id) {
|
|
2790
|
+
const candidate = await loadCandidateOrFail(process.cwd(), id);
|
|
2791
|
+
if (!candidate) return;
|
|
2792
|
+
header(`Candidate ${candidate.id}`);
|
|
2793
|
+
status("i", dim(`title: ${candidate.title}`));
|
|
2794
|
+
status("i", dim(`slug: ${candidate.slug}`));
|
|
2795
|
+
status("i", dim(`summary: ${candidate.summary}`));
|
|
2796
|
+
status("i", dim(`sources: ${candidate.sources.join(", ")}`));
|
|
2797
|
+
status("i", dim(`generated: ${candidate.generatedAt}`));
|
|
2798
|
+
console.log();
|
|
2799
|
+
console.log(candidate.body);
|
|
2800
|
+
}
|
|
2801
|
+
|
|
2802
|
+
// src/commands/review-approve.ts
|
|
2803
|
+
import path20 from "path";
|
|
2804
|
+
|
|
2805
|
+
// src/commands/review-helpers.ts
|
|
2806
|
+
async function runReviewUnderLock(id, underLock) {
|
|
2807
|
+
const root = process.cwd();
|
|
2808
|
+
const preCheck = await loadCandidateOrFail(root, id);
|
|
2809
|
+
if (!preCheck) return;
|
|
2810
|
+
const locked = await acquireLock(root);
|
|
2811
|
+
if (!locked) {
|
|
2812
|
+
status("!", error("Could not acquire lock. Try again later."));
|
|
2813
|
+
process.exitCode = 1;
|
|
2814
|
+
return;
|
|
2815
|
+
}
|
|
2816
|
+
try {
|
|
2817
|
+
await underLock(root, id);
|
|
2818
|
+
} finally {
|
|
2819
|
+
await releaseLock(root);
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
|
|
2823
|
+
// src/commands/review-approve.ts
|
|
2824
|
+
async function reviewApproveCommand(id) {
|
|
2825
|
+
await runReviewUnderLock(id, approveUnderLock);
|
|
2826
|
+
}
|
|
2827
|
+
async function approveUnderLock(root, id) {
|
|
2828
|
+
const candidate = await loadCandidateUnderLockOrFail(root, id);
|
|
2829
|
+
if (!candidate) return;
|
|
2830
|
+
if (!validateWikiPage(candidate.body)) {
|
|
2831
|
+
status("!", error(`Candidate ${id} failed page validation; not approved.`));
|
|
2832
|
+
process.exitCode = 1;
|
|
2833
|
+
return;
|
|
2834
|
+
}
|
|
2835
|
+
const pagePath = path20.join(root, CONCEPTS_DIR, `${candidate.slug}.md`);
|
|
2836
|
+
await atomicWrite(pagePath, candidate.body);
|
|
2837
|
+
status("+", success(`Approved \u2192 ${source(pagePath)}`));
|
|
2838
|
+
await persistCandidateSourceStates(root, candidate);
|
|
2839
|
+
await refreshWikiAfterApproval(root, candidate.slug);
|
|
2840
|
+
await deleteCandidate(root, id);
|
|
2841
|
+
status("\u2713", dim(`Candidate ${id} cleared.`));
|
|
2842
|
+
}
|
|
2843
|
+
async function persistCandidateSourceStates(root, candidate) {
|
|
2844
|
+
const states = candidate.sourceStates;
|
|
2845
|
+
if (!states) return;
|
|
2846
|
+
const otherSources = await collectOtherCandidateSources(root, candidate.id);
|
|
2847
|
+
for (const [sourceFile, entry] of Object.entries(states)) {
|
|
2848
|
+
if (otherSources.has(sourceFile)) continue;
|
|
2849
|
+
await updateSourceState(root, sourceFile, entry);
|
|
2850
|
+
}
|
|
2851
|
+
}
|
|
2852
|
+
async function collectOtherCandidateSources(root, approvingId) {
|
|
2853
|
+
const pending = await listCandidates(root);
|
|
2854
|
+
const sources = /* @__PURE__ */ new Set();
|
|
2855
|
+
for (const candidate of pending) {
|
|
2856
|
+
if (candidate.id === approvingId) continue;
|
|
2857
|
+
for (const source2 of candidate.sources) sources.add(source2);
|
|
2858
|
+
}
|
|
2859
|
+
return sources;
|
|
2860
|
+
}
|
|
2861
|
+
async function refreshWikiAfterApproval(root, slug) {
|
|
2862
|
+
await resolveLinks(root, [slug], [slug]);
|
|
2863
|
+
await generateIndex(root);
|
|
2864
|
+
await generateMOC(root);
|
|
2865
|
+
await safelyUpdateEmbeddings2(root, [slug]);
|
|
2866
|
+
}
|
|
2867
|
+
async function safelyUpdateEmbeddings2(root, slugs) {
|
|
2868
|
+
try {
|
|
2869
|
+
await updateEmbeddings(root, slugs);
|
|
2870
|
+
} catch (err) {
|
|
2871
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2872
|
+
status("!", warn(`Skipped embeddings update: ${message}`));
|
|
2873
|
+
}
|
|
2874
|
+
}
|
|
2875
|
+
|
|
2876
|
+
// src/commands/review-reject.ts
|
|
2877
|
+
async function reviewRejectCommand(id) {
|
|
2878
|
+
await runReviewUnderLock(id, rejectUnderLock);
|
|
2879
|
+
}
|
|
2880
|
+
async function rejectUnderLock(root, id) {
|
|
2881
|
+
const candidate = await loadCandidateUnderLockOrFail(root, id);
|
|
2882
|
+
if (!candidate) return;
|
|
2883
|
+
await archiveCandidate(root, id);
|
|
2884
|
+
status(
|
|
2885
|
+
"-",
|
|
2886
|
+
warn(`Rejected candidate ${id} (${candidate.slug}) \u2014 archived, wiki unchanged.`)
|
|
2887
|
+
);
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2305
2890
|
// src/mcp/server.ts
|
|
2306
2891
|
import { McpServer as McpServer2 } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2307
2892
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
2308
2893
|
|
|
2309
2894
|
// src/mcp/tools.ts
|
|
2310
|
-
import
|
|
2895
|
+
import path21 from "path";
|
|
2311
2896
|
import { z } from "zod";
|
|
2312
2897
|
|
|
2313
2898
|
// src/mcp/provider-check.ts
|
|
@@ -2437,7 +3022,7 @@ async function pickSearchSlugs(root, question) {
|
|
|
2437
3022
|
if (candidates.length > 0) return candidates.map((c) => c.slug);
|
|
2438
3023
|
} catch {
|
|
2439
3024
|
}
|
|
2440
|
-
const indexContent = await safeReadFile(
|
|
3025
|
+
const indexContent = await safeReadFile(path21.join(root, INDEX_FILE));
|
|
2441
3026
|
const { pages } = await selectPages(question, indexContent);
|
|
2442
3027
|
return pages;
|
|
2443
3028
|
}
|
|
@@ -2486,11 +3071,12 @@ function registerStatusTool(server, root) {
|
|
|
2486
3071
|
);
|
|
2487
3072
|
}
|
|
2488
3073
|
async function collectStatus(root) {
|
|
2489
|
-
const concepts = await collectPageSummaries(
|
|
2490
|
-
const queries = await collectPageSummaries(
|
|
3074
|
+
const concepts = await collectPageSummaries(path21.join(root, CONCEPTS_DIR));
|
|
3075
|
+
const queries = await collectPageSummaries(path21.join(root, QUERIES_DIR));
|
|
2491
3076
|
const state = await readState(root);
|
|
2492
3077
|
const changes = await detectChanges(root, state);
|
|
2493
3078
|
const orphans = await findOrphanedSlugs(root);
|
|
3079
|
+
const pendingCandidates = await countCandidates(root);
|
|
2494
3080
|
const compileTimes = Object.values(state.sources).map((s) => s.compiledAt);
|
|
2495
3081
|
const lastCompile = compileTimes.length > 0 ? compileTimes.sort().slice(-1)[0] : null;
|
|
2496
3082
|
return {
|
|
@@ -2498,11 +3084,12 @@ async function collectStatus(root) {
|
|
|
2498
3084
|
sources: Object.keys(state.sources).length,
|
|
2499
3085
|
lastCompiledAt: lastCompile,
|
|
2500
3086
|
orphanedPages: orphans,
|
|
3087
|
+
pendingCandidates,
|
|
2501
3088
|
pendingChanges: changes.filter((c) => c.status !== "unchanged").map((c) => ({ file: c.file, status: c.status }))
|
|
2502
3089
|
};
|
|
2503
3090
|
}
|
|
2504
3091
|
async function findOrphanedSlugs(root) {
|
|
2505
|
-
const scanned = await scanWikiPages(
|
|
3092
|
+
const scanned = await scanWikiPages(path21.join(root, CONCEPTS_DIR));
|
|
2506
3093
|
return scanned.filter(({ meta }) => meta.orphaned).map(({ slug }) => slug);
|
|
2507
3094
|
}
|
|
2508
3095
|
async function loadPageRecords(root, slugs) {
|
|
@@ -2515,7 +3102,7 @@ async function loadPageRecords(root, slugs) {
|
|
|
2515
3102
|
}
|
|
2516
3103
|
async function readPage(root, slug) {
|
|
2517
3104
|
for (const dir of PAGE_DIRS2) {
|
|
2518
|
-
const content = await safeReadFile(
|
|
3105
|
+
const content = await safeReadFile(path21.join(root, dir, `${slug}.md`));
|
|
2519
3106
|
if (!content) continue;
|
|
2520
3107
|
const { meta, body } = parseFrontmatter(content);
|
|
2521
3108
|
if (meta.orphaned) continue;
|
|
@@ -2530,8 +3117,8 @@ async function readPage(root, slug) {
|
|
|
2530
3117
|
}
|
|
2531
3118
|
|
|
2532
3119
|
// src/mcp/resources.ts
|
|
2533
|
-
import
|
|
2534
|
-
import { readdir as
|
|
3120
|
+
import path22 from "path";
|
|
3121
|
+
import { readdir as readdir9 } from "fs/promises";
|
|
2535
3122
|
import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2536
3123
|
function jsonContent(uri, payload) {
|
|
2537
3124
|
return {
|
|
@@ -2564,7 +3151,7 @@ function registerIndexResource(server, root) {
|
|
|
2564
3151
|
mimeType: "text/markdown"
|
|
2565
3152
|
},
|
|
2566
3153
|
async (uri) => {
|
|
2567
|
-
const content = await safeReadFile(
|
|
3154
|
+
const content = await safeReadFile(path22.join(root, INDEX_FILE));
|
|
2568
3155
|
return { contents: [markdownContent(uri, content)] };
|
|
2569
3156
|
}
|
|
2570
3157
|
);
|
|
@@ -2631,23 +3218,23 @@ function registerQueryResource(server, root) {
|
|
|
2631
3218
|
);
|
|
2632
3219
|
}
|
|
2633
3220
|
async function listSources(root) {
|
|
2634
|
-
const sourcesPath =
|
|
3221
|
+
const sourcesPath = path22.join(root, SOURCES_DIR);
|
|
2635
3222
|
let files;
|
|
2636
3223
|
try {
|
|
2637
|
-
files = await
|
|
3224
|
+
files = await readdir9(sourcesPath);
|
|
2638
3225
|
} catch {
|
|
2639
3226
|
return [];
|
|
2640
3227
|
}
|
|
2641
3228
|
const records = [];
|
|
2642
3229
|
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
2643
|
-
const content = await safeReadFile(
|
|
3230
|
+
const content = await safeReadFile(path22.join(sourcesPath, file));
|
|
2644
3231
|
const { meta } = parseFrontmatter(content);
|
|
2645
3232
|
records.push({ filename: file, ...meta });
|
|
2646
3233
|
}
|
|
2647
3234
|
return records;
|
|
2648
3235
|
}
|
|
2649
3236
|
async function loadPageWithMeta(root, dir, slug) {
|
|
2650
|
-
const filePath =
|
|
3237
|
+
const filePath = path22.join(root, dir, `${slug}.md`);
|
|
2651
3238
|
const content = await safeReadFile(filePath);
|
|
2652
3239
|
if (!content) {
|
|
2653
3240
|
throw new Error(`Page not found: ${dir}/${slug}.md`);
|
|
@@ -2656,10 +3243,10 @@ async function loadPageWithMeta(root, dir, slug) {
|
|
|
2656
3243
|
return { slug, meta, body: body.trim() };
|
|
2657
3244
|
}
|
|
2658
3245
|
async function listPagesUnder(root, dir, scheme) {
|
|
2659
|
-
const pagesPath =
|
|
3246
|
+
const pagesPath = path22.join(root, dir);
|
|
2660
3247
|
let files;
|
|
2661
3248
|
try {
|
|
2662
|
-
files = await
|
|
3249
|
+
files = await readdir9(pagesPath);
|
|
2663
3250
|
} catch {
|
|
2664
3251
|
return { resources: [] };
|
|
2665
3252
|
}
|
|
@@ -2695,10 +3282,46 @@ program.command("ingest <source>").description("Ingest a URL or local file into
|
|
|
2695
3282
|
process.exit(1);
|
|
2696
3283
|
}
|
|
2697
3284
|
});
|
|
2698
|
-
program.command("compile").description("Compile sources/ into an interlinked wiki").
|
|
3285
|
+
program.command("compile").description("Compile sources/ into an interlinked wiki").option(
|
|
3286
|
+
"--review",
|
|
3287
|
+
"Write generated pages as review candidates under .llmwiki/candidates/ instead of mutating wiki/. Orphan-marking for deleted sources is deferred until the next non-review compile."
|
|
3288
|
+
).action(async (options) => {
|
|
2699
3289
|
try {
|
|
2700
3290
|
requireProvider();
|
|
2701
|
-
await compileCommand();
|
|
3291
|
+
await compileCommand({ review: options.review });
|
|
3292
|
+
} catch (err) {
|
|
3293
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
3294
|
+
process.exit(1);
|
|
3295
|
+
}
|
|
3296
|
+
});
|
|
3297
|
+
var reviewCommand = program.command("review").description("Inspect and act on pending compile review candidates");
|
|
3298
|
+
reviewCommand.command("list").description("List pending review candidates").action(async () => {
|
|
3299
|
+
try {
|
|
3300
|
+
await reviewListCommand();
|
|
3301
|
+
} catch (err) {
|
|
3302
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
3303
|
+
process.exit(1);
|
|
3304
|
+
}
|
|
3305
|
+
});
|
|
3306
|
+
reviewCommand.command("show <id>").description("Print a single candidate's metadata and body").action(async (id) => {
|
|
3307
|
+
try {
|
|
3308
|
+
await reviewShowCommand(id);
|
|
3309
|
+
} catch (err) {
|
|
3310
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
3311
|
+
process.exit(1);
|
|
3312
|
+
}
|
|
3313
|
+
});
|
|
3314
|
+
reviewCommand.command("approve <id>").description("Approve a candidate and promote it into wiki/concepts/").action(async (id) => {
|
|
3315
|
+
try {
|
|
3316
|
+
await reviewApproveCommand(id);
|
|
3317
|
+
} catch (err) {
|
|
3318
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
3319
|
+
process.exit(1);
|
|
3320
|
+
}
|
|
3321
|
+
});
|
|
3322
|
+
reviewCommand.command("reject <id>").description("Reject a candidate and archive it without touching wiki/").action(async (id) => {
|
|
3323
|
+
try {
|
|
3324
|
+
await reviewRejectCommand(id);
|
|
2702
3325
|
} catch (err) {
|
|
2703
3326
|
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
2704
3327
|
process.exit(1);
|