llm-wiki-compiler 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +196 -720
- package/dist/cli.js +1466 -252
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +92 -3
- package/dist/index.js +1249 -192
- package/dist/index.js.map +1 -1
- package/package.json +4 -1
package/dist/cli.js
CHANGED
|
@@ -22,7 +22,8 @@ var VALID_PROVENANCE_STATES = /* @__PURE__ */ new Set([
|
|
|
22
22
|
"extracted",
|
|
23
23
|
"merged",
|
|
24
24
|
"inferred",
|
|
25
|
-
"ambiguous"
|
|
25
|
+
"ambiguous",
|
|
26
|
+
"imported"
|
|
26
27
|
]);
|
|
27
28
|
function slugify(title) {
|
|
28
29
|
return title.toLowerCase().replace(/['']/g, "").replace(/[^\p{L}\p{N}\s-]/gu, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
@@ -218,6 +219,7 @@ var OLLAMA_DEFAULT_HOST = "http://localhost:11434/v1";
|
|
|
218
219
|
var COPILOT_BASE_URL = "https://api.githubcopilot.com";
|
|
219
220
|
var OPENAI_DEFAULT_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
220
221
|
var OLLAMA_DEFAULT_TIMEOUT_MS = 30 * 60 * 1e3;
|
|
222
|
+
var EXPORT_DIR = "dist/exports";
|
|
221
223
|
var SOURCES_DIR = "sources";
|
|
222
224
|
var CONCEPTS_DIR = "wiki/concepts";
|
|
223
225
|
var QUERIES_DIR = "wiki/queries";
|
|
@@ -280,6 +282,25 @@ async function confinedRegularFile(dir, name) {
|
|
|
280
282
|
if (real === null || !isInsideDir(real, dir)) return null;
|
|
281
283
|
return real;
|
|
282
284
|
}
|
|
285
|
+
async function confineUnderRoot(target, root, opts) {
|
|
286
|
+
const realRoot = await safeRealpath(root) ?? path2.resolve(root);
|
|
287
|
+
const abs = path2.normalize(path2.resolve(realRoot, target));
|
|
288
|
+
if (!isInsideDir(abs, realRoot)) throw new Error(`path escapes project root: ${target}`);
|
|
289
|
+
if (opts.mustExist) {
|
|
290
|
+
const real = await safeRealpath(abs);
|
|
291
|
+
if (real === null || !isInsideDir(real, realRoot)) throw new Error(`path escapes project root: ${target}`);
|
|
292
|
+
return real;
|
|
293
|
+
}
|
|
294
|
+
for (let cur = abs; ; cur = path2.dirname(cur)) {
|
|
295
|
+
const real = await safeRealpath(cur);
|
|
296
|
+
if (real !== null) {
|
|
297
|
+
if (!isInsideDir(real, realRoot)) throw new Error(`path escapes project root: ${target}`);
|
|
298
|
+
break;
|
|
299
|
+
}
|
|
300
|
+
if (path2.dirname(cur) === cur) break;
|
|
301
|
+
}
|
|
302
|
+
return abs;
|
|
303
|
+
}
|
|
283
304
|
|
|
284
305
|
// src/viewer/path-safety.ts
|
|
285
306
|
import path3 from "path";
|
|
@@ -428,6 +449,9 @@ function source(text) {
|
|
|
428
449
|
}
|
|
429
450
|
var quietMode = false;
|
|
430
451
|
var quietScope = new AsyncLocalStorage();
|
|
452
|
+
function withQuiet(fn) {
|
|
453
|
+
return quietScope.run(true, fn);
|
|
454
|
+
}
|
|
431
455
|
function isQuiet() {
|
|
432
456
|
return quietScope.getStore() ?? quietMode;
|
|
433
457
|
}
|
|
@@ -2642,6 +2666,17 @@ async function moveCandidateToArchive(sourcePath, targetPath) {
|
|
|
2642
2666
|
// src/compiler/candidates.ts
|
|
2643
2667
|
var ID_SUFFIX_BYTES = 4;
|
|
2644
2668
|
var CANDIDATE_EXT = ".json";
|
|
2669
|
+
var DEFAULT_HELD_REASONS = [{ code: "manual-review-requested" }];
|
|
2670
|
+
var VALID_REVIEW_MODES = ["policy", "forced", "imported"];
|
|
2671
|
+
var VALID_HELD_REASON_CODES = [
|
|
2672
|
+
"low-confidence",
|
|
2673
|
+
"contradicted",
|
|
2674
|
+
"schema-violating",
|
|
2675
|
+
"provenance-violating",
|
|
2676
|
+
"all",
|
|
2677
|
+
"manual-review-requested",
|
|
2678
|
+
"imported-okf"
|
|
2679
|
+
];
|
|
2645
2680
|
function buildCandidateId(slug) {
|
|
2646
2681
|
const suffix = randomBytes(ID_SUFFIX_BYTES).toString("hex");
|
|
2647
2682
|
return `${slug}-${suffix}`;
|
|
@@ -2653,21 +2688,48 @@ function archivePath(root, id) {
|
|
|
2653
2688
|
return path22.join(root, CANDIDATES_ARCHIVE_DIR, `${id}${CANDIDATE_EXT}`);
|
|
2654
2689
|
}
|
|
2655
2690
|
async function writeCandidate(root, draft) {
|
|
2691
|
+
const { canonicalId, duplicateIds } = await findSlugDuplicates(root, draft.slug);
|
|
2656
2692
|
const candidate = {
|
|
2657
|
-
id: buildCandidateId(draft.slug),
|
|
2693
|
+
id: canonicalId ?? buildCandidateId(draft.slug),
|
|
2658
2694
|
title: draft.title,
|
|
2659
2695
|
slug: draft.slug,
|
|
2660
2696
|
summary: draft.summary,
|
|
2661
2697
|
sources: draft.sources,
|
|
2662
2698
|
body: draft.body,
|
|
2663
2699
|
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2700
|
+
reviewMode: draft.reviewMode ?? "forced",
|
|
2701
|
+
heldReasons: draft.heldReasons ?? DEFAULT_HELD_REASONS,
|
|
2664
2702
|
...draft.sourceStates ? { sourceStates: draft.sourceStates } : {},
|
|
2665
2703
|
...draft.schemaViolations ? { schemaViolations: draft.schemaViolations } : {},
|
|
2666
|
-
...draft.provenanceViolations ? { provenanceViolations: draft.provenanceViolations } : {}
|
|
2704
|
+
...draft.provenanceViolations ? { provenanceViolations: draft.provenanceViolations } : {},
|
|
2705
|
+
...draft.confidence !== void 0 ? { confidence: draft.confidence } : {},
|
|
2706
|
+
...draft.contradicted !== void 0 ? { contradicted: draft.contradicted } : {},
|
|
2707
|
+
...draft.targetDirectory ? { targetDirectory: draft.targetDirectory } : {},
|
|
2708
|
+
...draft.okfPath ? { okfPath: draft.okfPath } : {}
|
|
2667
2709
|
};
|
|
2668
2710
|
await atomicWrite(candidatePath(root, candidate.id), JSON.stringify(candidate, null, 2));
|
|
2711
|
+
await deleteDuplicates(root, duplicateIds);
|
|
2669
2712
|
return candidate;
|
|
2670
2713
|
}
|
|
2714
|
+
async function findSlugDuplicates(root, slug) {
|
|
2715
|
+
const all = await listCandidates(root);
|
|
2716
|
+
const matching = all.filter((c) => c.slug === slug);
|
|
2717
|
+
if (matching.length === 0) return { canonicalId: null, duplicateIds: [] };
|
|
2718
|
+
const [canonical, ...extras] = matching;
|
|
2719
|
+
return {
|
|
2720
|
+
canonicalId: canonical.id,
|
|
2721
|
+
duplicateIds: extras.map((c) => c.id)
|
|
2722
|
+
};
|
|
2723
|
+
}
|
|
2724
|
+
async function deleteDuplicates(root, ids) {
|
|
2725
|
+
for (const id of ids) {
|
|
2726
|
+
await deleteCandidate(root, id);
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
async function listPendingCandidateSlugs(root) {
|
|
2730
|
+
const candidates = await listCandidates(root);
|
|
2731
|
+
return new Set(candidates.map((candidate) => candidate.slug));
|
|
2732
|
+
}
|
|
2671
2733
|
function failWithError(message) {
|
|
2672
2734
|
status("!", error(message));
|
|
2673
2735
|
process.exitCode = 1;
|
|
@@ -2690,12 +2752,51 @@ async function readCandidate(root, id) {
|
|
|
2690
2752
|
if (!raw) return null;
|
|
2691
2753
|
try {
|
|
2692
2754
|
const parsed = JSON.parse(raw);
|
|
2693
|
-
if (!isValidCandidate(parsed))
|
|
2694
|
-
|
|
2755
|
+
if (!isValidCandidate(parsed)) {
|
|
2756
|
+
note(`[llmwiki] Skipping malformed candidate file: ${id}.json (missing required fields)`);
|
|
2757
|
+
return null;
|
|
2758
|
+
}
|
|
2759
|
+
return sanitizeCandidate(parsed);
|
|
2695
2760
|
} catch {
|
|
2761
|
+
note(`[llmwiki] Skipping unparseable candidate file: ${id}.json`);
|
|
2696
2762
|
return null;
|
|
2697
2763
|
}
|
|
2698
2764
|
}
|
|
2765
|
+
function sanitizeCandidate(candidate) {
|
|
2766
|
+
const generatedAt = typeof candidate.generatedAt === "string" ? candidate.generatedAt : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
2767
|
+
const reviewMode = VALID_REVIEW_MODES.includes(candidate.reviewMode) ? candidate.reviewMode : "forced";
|
|
2768
|
+
const heldReasons = sanitizeHeldReasons(candidate.heldReasons);
|
|
2769
|
+
const sourceStates = sanitizeSourceStates(candidate.sourceStates);
|
|
2770
|
+
const result = { ...candidate, generatedAt, reviewMode, heldReasons };
|
|
2771
|
+
if (sourceStates !== void 0) result.sourceStates = sourceStates;
|
|
2772
|
+
else delete result.sourceStates;
|
|
2773
|
+
return result;
|
|
2774
|
+
}
|
|
2775
|
+
function sanitizeHeldReasons(raw) {
|
|
2776
|
+
if (!Array.isArray(raw)) return DEFAULT_HELD_REASONS;
|
|
2777
|
+
const valid = raw.filter(
|
|
2778
|
+
(r) => r !== null && typeof r === "object" && typeof r.code === "string" && VALID_HELD_REASON_CODES.includes(r.code)
|
|
2779
|
+
);
|
|
2780
|
+
return valid.length > 0 ? valid : DEFAULT_HELD_REASONS;
|
|
2781
|
+
}
|
|
2782
|
+
function isSourceKeysafe(key) {
|
|
2783
|
+
return !key.includes("/") && !key.includes("\\") && !key.includes("..");
|
|
2784
|
+
}
|
|
2785
|
+
function sanitizeSourceStates(raw) {
|
|
2786
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return void 0;
|
|
2787
|
+
const result = {};
|
|
2788
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
2789
|
+
if (!isSourceKeyValid(key, value)) continue;
|
|
2790
|
+
result[key] = value;
|
|
2791
|
+
}
|
|
2792
|
+
return result;
|
|
2793
|
+
}
|
|
2794
|
+
function isSourceKeyValid(key, value) {
|
|
2795
|
+
if (!isSourceKeysafe(key)) return false;
|
|
2796
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2797
|
+
const entry = value;
|
|
2798
|
+
return typeof entry.hash === "string" && entry.hash.length > 0 && Array.isArray(entry.concepts) && entry.concepts.every((c) => typeof c === "string") && typeof entry.compiledAt === "string";
|
|
2799
|
+
}
|
|
2699
2800
|
function isValidCandidate(value) {
|
|
2700
2801
|
if (!value || typeof value !== "object") return false;
|
|
2701
2802
|
const candidate = value;
|
|
@@ -2722,6 +2823,15 @@ async function deleteCandidate(root, id) {
|
|
|
2722
2823
|
await unlink2(filePath);
|
|
2723
2824
|
return true;
|
|
2724
2825
|
}
|
|
2826
|
+
async function deleteCandidateBySlug(root, slug) {
|
|
2827
|
+
const all = await listCandidates(root);
|
|
2828
|
+
const matching = all.filter((c) => c.slug === slug);
|
|
2829
|
+
if (matching.length === 0) return false;
|
|
2830
|
+
for (const candidate of matching) {
|
|
2831
|
+
await deleteCandidate(root, candidate.id);
|
|
2832
|
+
}
|
|
2833
|
+
return true;
|
|
2834
|
+
}
|
|
2725
2835
|
async function archiveCandidate(root, id) {
|
|
2726
2836
|
return moveCandidateToArchive(candidatePath(root, id), archivePath(root, id));
|
|
2727
2837
|
}
|
|
@@ -3110,11 +3220,11 @@ function registerShutdown(close) {
|
|
|
3110
3220
|
}
|
|
3111
3221
|
|
|
3112
3222
|
// src/commands/compile.ts
|
|
3113
|
-
import { existsSync as
|
|
3223
|
+
import { existsSync as existsSync10 } from "fs";
|
|
3114
3224
|
|
|
3115
3225
|
// src/compiler/index.ts
|
|
3116
|
-
import { readFile as
|
|
3117
|
-
import
|
|
3226
|
+
import { readFile as readFile24, readdir as readdir13 } from "fs/promises";
|
|
3227
|
+
import path38 from "path";
|
|
3118
3228
|
|
|
3119
3229
|
// src/compiler/source-state.ts
|
|
3120
3230
|
import path27 from "path";
|
|
@@ -5027,12 +5137,21 @@ function buildPageSlugSet(pages) {
|
|
|
5027
5137
|
async function checkBrokenWikilinks(root) {
|
|
5028
5138
|
const pages = await collectAllPages(root);
|
|
5029
5139
|
const existingSlugs = buildPageSlugSet(pages);
|
|
5140
|
+
const pendingSlugs = await listPendingCandidateSlugs(root);
|
|
5030
5141
|
const results = [];
|
|
5031
5142
|
for (const page of pages) {
|
|
5032
5143
|
for (const { captured, line: line2 } of findMatchesInContent(page.content, WIKILINK_PATTERN2)) {
|
|
5033
5144
|
const linkTarget = captured.split("|")[0].trim();
|
|
5034
5145
|
const linkSlug = slugify(linkTarget);
|
|
5035
|
-
if (!existingSlugs.has(linkSlug)) {
|
|
5146
|
+
if (!existingSlugs.has(linkSlug) && pendingSlugs.has(linkSlug)) {
|
|
5147
|
+
results.push({
|
|
5148
|
+
rule: "pending-target",
|
|
5149
|
+
severity: "info",
|
|
5150
|
+
file: page.filePath,
|
|
5151
|
+
message: `Wikilink [[${captured}]] points to a page awaiting review`,
|
|
5152
|
+
line: line2
|
|
5153
|
+
});
|
|
5154
|
+
} else if (!existingSlugs.has(linkSlug)) {
|
|
5036
5155
|
results.push({
|
|
5037
5156
|
rule: "broken-wikilink",
|
|
5038
5157
|
severity: "error",
|
|
@@ -5274,6 +5393,8 @@ async function checkBrokenCitations(root) {
|
|
|
5274
5393
|
const results = [];
|
|
5275
5394
|
const lineCountCache = /* @__PURE__ */ new Map();
|
|
5276
5395
|
for (const page of pages) {
|
|
5396
|
+
const { meta } = parseFrontmatter(page.content);
|
|
5397
|
+
if (meta.provenanceState === "imported") continue;
|
|
5277
5398
|
const pageFindings = await checkPageBrokenCitations(
|
|
5278
5399
|
page.content,
|
|
5279
5400
|
page.filePath,
|
|
@@ -5417,6 +5538,154 @@ async function loadRelatedPages(root, excludeSlug) {
|
|
|
5417
5538
|
return contents.join("\n\n---\n\n");
|
|
5418
5539
|
}
|
|
5419
5540
|
|
|
5541
|
+
// src/review/config.ts
|
|
5542
|
+
import { readFile as readFile23 } from "fs/promises";
|
|
5543
|
+
import { existsSync as existsSync9 } from "fs";
|
|
5544
|
+
import path37 from "path";
|
|
5545
|
+
var PROJECT_CONFIG_FILE = path37.join(LLMWIKI_DIR, "config.json");
|
|
5546
|
+
var REVIEW_POLICY_OFF = {
|
|
5547
|
+
hold: [],
|
|
5548
|
+
lowConfidenceThreshold: LOW_CONFIDENCE_THRESHOLD,
|
|
5549
|
+
treatMissingConfidenceAs: "low"
|
|
5550
|
+
};
|
|
5551
|
+
var ReviewConfigError = class extends Error {
|
|
5552
|
+
constructor(message) {
|
|
5553
|
+
super(message);
|
|
5554
|
+
this.name = "ReviewConfigError";
|
|
5555
|
+
}
|
|
5556
|
+
};
|
|
5557
|
+
var SIGNAL_MODES = [
|
|
5558
|
+
"low-confidence",
|
|
5559
|
+
"contradicted",
|
|
5560
|
+
"schema-violating",
|
|
5561
|
+
"provenance-violating"
|
|
5562
|
+
];
|
|
5563
|
+
var MODE_VALUES = /* @__PURE__ */ new Set([
|
|
5564
|
+
...SIGNAL_MODES,
|
|
5565
|
+
"all",
|
|
5566
|
+
"off"
|
|
5567
|
+
]);
|
|
5568
|
+
async function loadReviewPolicy(root) {
|
|
5569
|
+
const filePath = path37.join(root, PROJECT_CONFIG_FILE);
|
|
5570
|
+
if (!existsSync9(filePath)) return { ...REVIEW_POLICY_OFF };
|
|
5571
|
+
const raw = await readConfigJson(filePath);
|
|
5572
|
+
return normalizeReviewPolicy(raw);
|
|
5573
|
+
}
|
|
5574
|
+
async function readConfigJson(filePath) {
|
|
5575
|
+
try {
|
|
5576
|
+
return JSON.parse(await readFile23(filePath, "utf-8"));
|
|
5577
|
+
} catch (err) {
|
|
5578
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
5579
|
+
throw new ReviewConfigError(`Invalid .llmwiki/config.json: ${message}`);
|
|
5580
|
+
}
|
|
5581
|
+
}
|
|
5582
|
+
function normalizeReviewPolicy(raw) {
|
|
5583
|
+
const config = requireRecord(raw, "config");
|
|
5584
|
+
validateVersion(config.version);
|
|
5585
|
+
const review = config.review;
|
|
5586
|
+
if (review === void 0) return { ...REVIEW_POLICY_OFF };
|
|
5587
|
+
const reviewConfig = requireRecord(review, "review");
|
|
5588
|
+
const hold = normalizeHoldModes(reviewConfig.hold);
|
|
5589
|
+
if (hold.length === 0 || hold.includes("off")) {
|
|
5590
|
+
return { ...REVIEW_POLICY_OFF };
|
|
5591
|
+
}
|
|
5592
|
+
return {
|
|
5593
|
+
hold,
|
|
5594
|
+
lowConfidenceThreshold: normalizeThreshold(reviewConfig.lowConfidenceThreshold),
|
|
5595
|
+
treatMissingConfidenceAs: normalizeMissingConfidence(reviewConfig.treatMissingConfidenceAs)
|
|
5596
|
+
};
|
|
5597
|
+
}
|
|
5598
|
+
function requireRecord(value, label) {
|
|
5599
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
5600
|
+
throw new ReviewConfigError(`${label} must be a JSON object`);
|
|
5601
|
+
}
|
|
5602
|
+
return value;
|
|
5603
|
+
}
|
|
5604
|
+
function validateVersion(version2) {
|
|
5605
|
+
if (version2 === 1) return;
|
|
5606
|
+
throw new ReviewConfigError('.llmwiki/config.json requires "version": 1');
|
|
5607
|
+
}
|
|
5608
|
+
function normalizeHoldModes(value) {
|
|
5609
|
+
if (value === void 0) return [];
|
|
5610
|
+
if (!Array.isArray(value)) throw new ReviewConfigError("review.hold must be an array");
|
|
5611
|
+
const modes = value.map(readMode);
|
|
5612
|
+
validateModeCombination(modes);
|
|
5613
|
+
return [...new Set(modes)];
|
|
5614
|
+
}
|
|
5615
|
+
function readMode(value) {
|
|
5616
|
+
if (typeof value !== "string" || !MODE_VALUES.has(value)) {
|
|
5617
|
+
throw new ReviewConfigError(`Unknown review.hold mode: ${String(value)}`);
|
|
5618
|
+
}
|
|
5619
|
+
return value;
|
|
5620
|
+
}
|
|
5621
|
+
function validateModeCombination(modes) {
|
|
5622
|
+
if (modes.length <= 1) return;
|
|
5623
|
+
if (modes.includes("off")) {
|
|
5624
|
+
throw new ReviewConfigError('review.hold mode "off" cannot be combined with other modes');
|
|
5625
|
+
}
|
|
5626
|
+
if (modes.includes("all")) {
|
|
5627
|
+
throw new ReviewConfigError('review.hold mode "all" cannot be combined with other modes');
|
|
5628
|
+
}
|
|
5629
|
+
}
|
|
5630
|
+
function normalizeThreshold(value) {
|
|
5631
|
+
if (value === void 0) return LOW_CONFIDENCE_THRESHOLD;
|
|
5632
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
|
|
5633
|
+
throw new ReviewConfigError("review.lowConfidenceThreshold must be a finite number in [0,1]");
|
|
5634
|
+
}
|
|
5635
|
+
return value;
|
|
5636
|
+
}
|
|
5637
|
+
function normalizeMissingConfidence(value) {
|
|
5638
|
+
if (value === void 0) return "low";
|
|
5639
|
+
if (value === "low" || value === "ok") return value;
|
|
5640
|
+
throw new ReviewConfigError('review.treatMissingConfidenceAs must be "low" or "ok"');
|
|
5641
|
+
}
|
|
5642
|
+
|
|
5643
|
+
// src/review/policy.ts
|
|
5644
|
+
function isPolicyOff(policy) {
|
|
5645
|
+
return policy.hold.length === 0 || policy.hold.includes("off");
|
|
5646
|
+
}
|
|
5647
|
+
function evaluatePolicy(signals, policy) {
|
|
5648
|
+
if (isPolicyOff(policy)) return [];
|
|
5649
|
+
const reasons = [];
|
|
5650
|
+
for (const mode of policy.hold) {
|
|
5651
|
+
const reason = reasonForMode(mode, signals, policy);
|
|
5652
|
+
if (reason) reasons.push(reason);
|
|
5653
|
+
}
|
|
5654
|
+
return reasons;
|
|
5655
|
+
}
|
|
5656
|
+
function reasonForMode(mode, signals, policy) {
|
|
5657
|
+
if (mode === "all") return { code: "all" };
|
|
5658
|
+
if (mode === "low-confidence") return lowConfidenceReason(signals, policy);
|
|
5659
|
+
if (mode === "contradicted" && signals.contradicted) return { code: "contradicted" };
|
|
5660
|
+
if (mode === "schema-violating" && signals.schemaViolations.length > 0) {
|
|
5661
|
+
return { code: "schema-violating", detail: summarizeViolations(signals.schemaViolations) };
|
|
5662
|
+
}
|
|
5663
|
+
if (mode === "provenance-violating" && signals.provenanceViolations.length > 0) {
|
|
5664
|
+
return { code: "provenance-violating", detail: summarizeViolations(signals.provenanceViolations) };
|
|
5665
|
+
}
|
|
5666
|
+
return null;
|
|
5667
|
+
}
|
|
5668
|
+
function lowConfidenceReason(signals, policy) {
|
|
5669
|
+
if (signals.confidence === void 0) {
|
|
5670
|
+
if (policy.treatMissingConfidenceAs === "low") {
|
|
5671
|
+
return { code: "low-confidence", detail: "confidence missing" };
|
|
5672
|
+
}
|
|
5673
|
+
return null;
|
|
5674
|
+
}
|
|
5675
|
+
if (signals.confidence < policy.lowConfidenceThreshold) {
|
|
5676
|
+
return {
|
|
5677
|
+
code: "low-confidence",
|
|
5678
|
+
detail: `confidence ${signals.confidence} < ${policy.lowConfidenceThreshold}`
|
|
5679
|
+
};
|
|
5680
|
+
}
|
|
5681
|
+
return null;
|
|
5682
|
+
}
|
|
5683
|
+
function summarizeViolations(violations) {
|
|
5684
|
+
const first = violations[0];
|
|
5685
|
+
const suffix = violations.length === 1 ? "" : ` (+${violations.length - 1} more)`;
|
|
5686
|
+
return `${first.rule}${suffix}`;
|
|
5687
|
+
}
|
|
5688
|
+
|
|
5420
5689
|
// src/compiler/index.ts
|
|
5421
5690
|
import pLimit from "p-limit";
|
|
5422
5691
|
function emptyCompileResult() {
|
|
@@ -5448,33 +5717,70 @@ function bucketChanges(changes) {
|
|
|
5448
5717
|
unchanged: changes.filter((c) => c.status === "unchanged")
|
|
5449
5718
|
};
|
|
5450
5719
|
}
|
|
5451
|
-
async function generatePagesPhase(root, extractions, frozenSlugs, schema, options) {
|
|
5720
|
+
async function generatePagesPhase(root, extractions, frozenSlugs, schema, options, policy) {
|
|
5452
5721
|
const merged = mergeExtractions(extractions, frozenSlugs);
|
|
5453
|
-
const
|
|
5722
|
+
const shouldBuildSourceStates = options.review || !isPolicyOff(policy);
|
|
5723
|
+
const sourceStates = shouldBuildSourceStates ? await buildExtractionSourceStates(root, extractions) : {};
|
|
5454
5724
|
const limit = pLimit(COMPILE_CONCURRENCY);
|
|
5455
|
-
const
|
|
5456
|
-
const candidates = [];
|
|
5457
|
-
const pages = await Promise.all(
|
|
5725
|
+
const outcomes = await Promise.all(
|
|
5458
5726
|
merged.map((entry) => limit(async () => {
|
|
5459
|
-
const result = await generateMergedPage(root, entry, schema, options, sourceStates);
|
|
5460
|
-
|
|
5461
|
-
if (result.candidateId) candidates.push(result.candidateId);
|
|
5462
|
-
return entry;
|
|
5727
|
+
const result = await generateMergedPage(root, entry, schema, options, sourceStates, policy);
|
|
5728
|
+
return { entry, result };
|
|
5463
5729
|
}))
|
|
5464
5730
|
);
|
|
5465
|
-
|
|
5731
|
+
const errors = [];
|
|
5732
|
+
const candidates = [];
|
|
5733
|
+
const review = { held: [], forced: [] };
|
|
5734
|
+
for (const { result } of outcomes) {
|
|
5735
|
+
if (result.error) errors.push(result.error);
|
|
5736
|
+
if (result.candidate) {
|
|
5737
|
+
candidates.push(result.candidate.id);
|
|
5738
|
+
review[result.candidate.mode].push(result.candidate.ref);
|
|
5739
|
+
}
|
|
5740
|
+
}
|
|
5741
|
+
const pages = outcomes.map(({ entry }) => entry);
|
|
5742
|
+
const writtenPages = outcomes.filter(({ result }) => result.wrotePage).map(({ entry }) => entry);
|
|
5743
|
+
return { pages, writtenPages, errors, candidates, review, seedSlugs: [] };
|
|
5466
5744
|
}
|
|
5467
|
-
async function persistExtractionStates(root, extractions) {
|
|
5745
|
+
async function persistExtractionStates(root, extractions, writtenPages) {
|
|
5746
|
+
const liveSlugsForSource = buildLiveSlugsForSource(writtenPages);
|
|
5468
5747
|
for (const result of extractions) {
|
|
5469
5748
|
if (result.concepts.length === 0) continue;
|
|
5470
|
-
|
|
5749
|
+
const liveSlugs = liveSlugsForSource.get(result.sourceFile) ?? [];
|
|
5750
|
+
await persistSourceStateFiltered(
|
|
5751
|
+
root,
|
|
5752
|
+
result.sourcePath,
|
|
5753
|
+
result.sourceFile,
|
|
5754
|
+
result.concepts,
|
|
5755
|
+
new Set(liveSlugs)
|
|
5756
|
+
);
|
|
5471
5757
|
}
|
|
5472
5758
|
}
|
|
5759
|
+
function buildLiveSlugsForSource(writtenPages) {
|
|
5760
|
+
const map = /* @__PURE__ */ new Map();
|
|
5761
|
+
for (const page of writtenPages) {
|
|
5762
|
+
for (const sourceFile of page.sourceFiles) {
|
|
5763
|
+
const existing = map.get(sourceFile) ?? [];
|
|
5764
|
+
existing.push(page.slug);
|
|
5765
|
+
map.set(sourceFile, existing);
|
|
5766
|
+
}
|
|
5767
|
+
}
|
|
5768
|
+
return map;
|
|
5769
|
+
}
|
|
5770
|
+
async function persistSourceStateFiltered(root, sourcePath, sourceFile, concepts, liveSlugs) {
|
|
5771
|
+
const hash = await hashFile(sourcePath);
|
|
5772
|
+
const entry = {
|
|
5773
|
+
hash,
|
|
5774
|
+
concepts: concepts.map((c) => slugify(c.concept)).filter((s) => liveSlugs.has(s)),
|
|
5775
|
+
compiledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
5776
|
+
};
|
|
5777
|
+
await updateSourceState(root, sourceFile, entry);
|
|
5778
|
+
}
|
|
5473
5779
|
async function listExistingPageIds(root) {
|
|
5474
5780
|
const ids = /* @__PURE__ */ new Set();
|
|
5475
5781
|
for (const dir of [CONCEPTS_DIR, QUERIES_DIR]) {
|
|
5476
5782
|
try {
|
|
5477
|
-
const files = await readdir13(
|
|
5783
|
+
const files = await readdir13(path38.join(root, dir));
|
|
5478
5784
|
for (const file of files) {
|
|
5479
5785
|
if (file.endsWith(".md")) ids.add(`${dir}/${file.slice(0, -3)}`);
|
|
5480
5786
|
}
|
|
@@ -5485,7 +5791,7 @@ async function listExistingPageIds(root) {
|
|
|
5485
5791
|
}
|
|
5486
5792
|
async function logCompile(root, buckets, generation, existingIds) {
|
|
5487
5793
|
if (buckets.toCompile.length === 0 && buckets.deleted.length === 0) return;
|
|
5488
|
-
const produced = [.../* @__PURE__ */ new Set([...generation.
|
|
5794
|
+
const produced = [.../* @__PURE__ */ new Set([...generation.writtenPages.map((entry) => entry.slug), ...generation.seedSlugs])];
|
|
5489
5795
|
const existed = (slug) => existingIds.has(`${CONCEPTS_DIR}/${slug}`);
|
|
5490
5796
|
const created = produced.filter((slug) => !existed(slug));
|
|
5491
5797
|
const updated = produced.filter((slug) => existed(slug));
|
|
@@ -5504,9 +5810,9 @@ function summarizeCompile(buckets, generation, extractions, options) {
|
|
|
5504
5810
|
status("\u2713", success(
|
|
5505
5811
|
`${buckets.toCompile.length} compiled, ${buckets.unchanged.length} skipped, ${buckets.deleted.length} deleted`
|
|
5506
5812
|
));
|
|
5507
|
-
if (
|
|
5813
|
+
if (generation.candidates.length > 0) {
|
|
5508
5814
|
status("?", info(
|
|
5509
|
-
|
|
5815
|
+
reviewSummaryLine(generation)
|
|
5510
5816
|
));
|
|
5511
5817
|
} else if (buckets.toCompile.length > 0) {
|
|
5512
5818
|
status("\u2192", dim('Next: llmwiki query "your question here"'));
|
|
@@ -5517,7 +5823,7 @@ function summarizeCompile(buckets, generation, extractions, options) {
|
|
|
5517
5823
|
errors.push(`No concepts extracted from ${result.sourceFile}`);
|
|
5518
5824
|
}
|
|
5519
5825
|
}
|
|
5520
|
-
const conceptSlugs = generation.
|
|
5826
|
+
const conceptSlugs = generation.writtenPages.map((entry) => entry.slug);
|
|
5521
5827
|
const baseResult = {
|
|
5522
5828
|
compiled: buckets.toCompile.length,
|
|
5523
5829
|
skipped: buckets.unchanged.length,
|
|
@@ -5526,11 +5832,20 @@ function summarizeCompile(buckets, generation, extractions, options) {
|
|
|
5526
5832
|
pages: [...conceptSlugs, ...generation.seedSlugs],
|
|
5527
5833
|
errors
|
|
5528
5834
|
};
|
|
5529
|
-
if (
|
|
5835
|
+
if (generation.candidates.length > 0) {
|
|
5530
5836
|
baseResult.candidates = generation.candidates;
|
|
5837
|
+
baseResult.review = generation.review;
|
|
5531
5838
|
}
|
|
5532
5839
|
return baseResult;
|
|
5533
5840
|
}
|
|
5841
|
+
function reviewSummaryLine(generation) {
|
|
5842
|
+
const held = generation.review.held.length;
|
|
5843
|
+
const forced = generation.review.forced.length;
|
|
5844
|
+
if (held > 0 && forced === 0) {
|
|
5845
|
+
return `Wrote ${generation.writtenPages.length} page(s), held ${held} for review \u2014 run \`llmwiki review list\``;
|
|
5846
|
+
}
|
|
5847
|
+
return `${generation.candidates.length} candidate(s) awaiting review \u2014 run \`llmwiki review list\``;
|
|
5848
|
+
}
|
|
5534
5849
|
function applyChangeFilter(detected, filter) {
|
|
5535
5850
|
return filter ? detected.filter(filter) : detected;
|
|
5536
5851
|
}
|
|
@@ -5541,10 +5856,12 @@ async function maybeSeedPages(root, schema, generation, options) {
|
|
|
5541
5856
|
}
|
|
5542
5857
|
async function runCompilePipeline(root, options) {
|
|
5543
5858
|
const schema = await loadSchema(root);
|
|
5859
|
+
const reviewPolicy = await loadReviewPolicy(root);
|
|
5544
5860
|
reportSchemaStatus(schema);
|
|
5545
5861
|
const state = await readState(root);
|
|
5546
5862
|
const detected = await detectChanges(root, state);
|
|
5547
5863
|
const changes = applyChangeFilter(detected, options.changeFilter);
|
|
5864
|
+
await markUnchangedPendingSources(root, changes);
|
|
5548
5865
|
augmentWithAffectedSources(changes, findAffectedSources(state, changes));
|
|
5549
5866
|
const buckets = bucketChanges(changes);
|
|
5550
5867
|
if (buckets.toCompile.length === 0 && buckets.deleted.length === 0) {
|
|
@@ -5552,12 +5869,14 @@ async function runCompilePipeline(root, options) {
|
|
|
5552
5869
|
if (!options.review) {
|
|
5553
5870
|
const emptyGeneration = {
|
|
5554
5871
|
pages: [],
|
|
5872
|
+
writtenPages: [],
|
|
5555
5873
|
errors: [],
|
|
5556
5874
|
candidates: [],
|
|
5875
|
+
review: { held: [], forced: [] },
|
|
5557
5876
|
seedSlugs: []
|
|
5558
5877
|
};
|
|
5559
5878
|
await maybeSeedPages(root, schema, emptyGeneration, options);
|
|
5560
|
-
await finalizeWiki(root, emptyGeneration.
|
|
5879
|
+
await finalizeWiki(root, emptyGeneration.writtenPages, emptyGeneration.seedSlugs);
|
|
5561
5880
|
return {
|
|
5562
5881
|
...emptyCompileResult(),
|
|
5563
5882
|
skipped: buckets.unchanged.length,
|
|
@@ -5581,19 +5900,46 @@ async function runCompilePipeline(root, options) {
|
|
|
5581
5900
|
await freezeFailedExtractions(root, extractions, frozenSlugs);
|
|
5582
5901
|
}
|
|
5583
5902
|
const existingIds = await listExistingPageIds(root);
|
|
5584
|
-
const generation = await generatePagesPhase(
|
|
5903
|
+
const generation = await generatePagesPhase(
|
|
5904
|
+
root,
|
|
5905
|
+
extractions,
|
|
5906
|
+
frozenSlugs,
|
|
5907
|
+
schema,
|
|
5908
|
+
options,
|
|
5909
|
+
reviewPolicy
|
|
5910
|
+
);
|
|
5585
5911
|
if (!options.review) {
|
|
5586
|
-
await persistExtractionStates(root, extractions);
|
|
5912
|
+
await persistExtractionStates(root, extractions, generation.writtenPages);
|
|
5587
5913
|
if (frozenSlugs.size > 0) {
|
|
5588
5914
|
await orphanUnownedFrozenPages(root, frozenSlugs);
|
|
5589
5915
|
}
|
|
5590
5916
|
await persistFrozenSlugs(root, frozenSlugs, extractions);
|
|
5591
5917
|
await maybeSeedPages(root, schema, generation, options);
|
|
5592
|
-
await finalizeWiki(root, generation.
|
|
5918
|
+
await finalizeWiki(root, generation.writtenPages, generation.seedSlugs);
|
|
5593
5919
|
await logCompile(root, buckets, generation, existingIds);
|
|
5594
5920
|
}
|
|
5595
5921
|
return summarizeCompile(buckets, generation, extractions, options);
|
|
5596
5922
|
}
|
|
5923
|
+
async function markUnchangedPendingSources(root, changes) {
|
|
5924
|
+
const pendingHashes = await collectPendingSourceHashes(root);
|
|
5925
|
+
if (pendingHashes.size === 0) return;
|
|
5926
|
+
for (const change of changes) {
|
|
5927
|
+
if (change.status !== "new" && change.status !== "changed") continue;
|
|
5928
|
+
const pendingHash = pendingHashes.get(change.file);
|
|
5929
|
+
if (!pendingHash) continue;
|
|
5930
|
+
const currentHash = await hashFile(path38.join(root, SOURCES_DIR, change.file));
|
|
5931
|
+
if (currentHash === pendingHash) change.status = "unchanged";
|
|
5932
|
+
}
|
|
5933
|
+
}
|
|
5934
|
+
async function collectPendingSourceHashes(root) {
|
|
5935
|
+
const hashes = /* @__PURE__ */ new Map();
|
|
5936
|
+
for (const candidate of await listCandidates(root)) {
|
|
5937
|
+
for (const [source2, entry] of Object.entries(candidate.sourceStates ?? {})) {
|
|
5938
|
+
hashes.set(source2, entry.hash);
|
|
5939
|
+
}
|
|
5940
|
+
}
|
|
5941
|
+
return hashes;
|
|
5942
|
+
}
|
|
5597
5943
|
function reportSchemaStatus(schema) {
|
|
5598
5944
|
if (schema.loadedFrom) {
|
|
5599
5945
|
status("i", dim(`Schema: ${schema.loadedFrom}`));
|
|
@@ -5661,9 +6007,9 @@ function printChangesSummary(changes) {
|
|
|
5661
6007
|
}
|
|
5662
6008
|
async function extractForSource(root, sourceFile) {
|
|
5663
6009
|
status("*", info(`Extracting: ${sourceFile}`));
|
|
5664
|
-
const sourcePath =
|
|
5665
|
-
const sourceContent = await
|
|
5666
|
-
const existingIndex = await safeReadFile(
|
|
6010
|
+
const sourcePath = path38.join(root, SOURCES_DIR, sourceFile);
|
|
6011
|
+
const sourceContent = await readFile24(sourcePath, "utf-8");
|
|
6012
|
+
const existingIndex = await safeReadFile(path38.join(root, INDEX_FILE));
|
|
5667
6013
|
const concepts = await extractConcepts(sourceContent, existingIndex);
|
|
5668
6014
|
if (concepts.length > 0) {
|
|
5669
6015
|
const names = concepts.map((c) => c.concept).join(", ");
|
|
@@ -5724,16 +6070,30 @@ function mergeExtractions(extractions, frozenSlugs) {
|
|
|
5724
6070
|
}
|
|
5725
6071
|
return Array.from(bySlug.values());
|
|
5726
6072
|
}
|
|
5727
|
-
async function generateMergedPage(root, entry, schema, options, sourceStates) {
|
|
6073
|
+
async function generateMergedPage(root, entry, schema, options, sourceStates, policy) {
|
|
5728
6074
|
const fullPage = await renderMergedPageContent(root, entry, schema);
|
|
6075
|
+
const diagnostics = await collectReviewDiagnostics(root, entry, fullPage, schema);
|
|
6076
|
+
const signals = buildPolicySignals(fullPage, diagnostics);
|
|
6077
|
+
const reasons = evaluatePolicy(signals, policy);
|
|
5729
6078
|
if (options.review) {
|
|
5730
|
-
|
|
6079
|
+
const heldReasons = [{ code: "manual-review-requested" }, ...reasons];
|
|
6080
|
+
return await persistReviewCandidate(root, entry, fullPage, sourceStates, diagnostics, signals, {
|
|
6081
|
+
reviewMode: "forced",
|
|
6082
|
+
heldReasons
|
|
6083
|
+
});
|
|
6084
|
+
}
|
|
6085
|
+
if (reasons.length > 0) {
|
|
6086
|
+
return await persistReviewCandidate(root, entry, fullPage, sourceStates, diagnostics, signals, {
|
|
6087
|
+
reviewMode: "policy",
|
|
6088
|
+
heldReasons: reasons
|
|
6089
|
+
});
|
|
5731
6090
|
}
|
|
5732
|
-
const pagePath =
|
|
6091
|
+
const pagePath = path38.join(root, CONCEPTS_DIR, `${entry.slug}.md`);
|
|
5733
6092
|
const error2 = await writePageIfValid(pagePath, fullPage, entry.concept.concept);
|
|
5734
|
-
|
|
6093
|
+
if (!error2) await deleteCandidateBySlug(root, entry.slug);
|
|
6094
|
+
return { error: error2 ?? void 0, wrotePage: !error2 };
|
|
5735
6095
|
}
|
|
5736
|
-
async function
|
|
6096
|
+
async function collectReviewDiagnostics(root, entry, fullPage, schema) {
|
|
5737
6097
|
const virtualPath = `wiki/concepts/${entry.slug}.md`;
|
|
5738
6098
|
const schemaViolations = checkPageCrossLinks(fullPage, virtualPath, schema);
|
|
5739
6099
|
const provenanceViolations = await collectCandidateProvenanceViolations(
|
|
@@ -5741,6 +6101,19 @@ async function persistReviewCandidate(root, entry, fullPage, sourceStates, schem
|
|
|
5741
6101
|
fullPage,
|
|
5742
6102
|
virtualPath
|
|
5743
6103
|
);
|
|
6104
|
+
return { schemaViolations, provenanceViolations };
|
|
6105
|
+
}
|
|
6106
|
+
function buildPolicySignals(fullPage, diagnostics) {
|
|
6107
|
+
const { meta } = parseFrontmatter(fullPage);
|
|
6108
|
+
const provenance = parseProvenanceMetadata(meta);
|
|
6109
|
+
return {
|
|
6110
|
+
confidence: provenance.confidence,
|
|
6111
|
+
contradicted: (provenance.contradictedBy?.length ?? 0) > 0,
|
|
6112
|
+
schemaViolations: diagnostics.schemaViolations,
|
|
6113
|
+
provenanceViolations: diagnostics.provenanceViolations
|
|
6114
|
+
};
|
|
6115
|
+
}
|
|
6116
|
+
async function persistReviewCandidate(root, entry, fullPage, sourceStates, diagnostics, signals, meta) {
|
|
5744
6117
|
const candidate = await writeCandidate(root, {
|
|
5745
6118
|
title: entry.concept.concept,
|
|
5746
6119
|
slug: entry.slug,
|
|
@@ -5748,18 +6121,28 @@ async function persistReviewCandidate(root, entry, fullPage, sourceStates, schem
|
|
|
5748
6121
|
sources: entry.sourceFiles,
|
|
5749
6122
|
body: fullPage,
|
|
5750
6123
|
sourceStates: pickStatesForSources(sourceStates, entry.sourceFiles),
|
|
5751
|
-
schemaViolations: schemaViolations.length > 0 ? schemaViolations : void 0,
|
|
5752
|
-
provenanceViolations: provenanceViolations.length > 0 ? provenanceViolations : void 0
|
|
6124
|
+
schemaViolations: diagnostics.schemaViolations.length > 0 ? diagnostics.schemaViolations : void 0,
|
|
6125
|
+
provenanceViolations: diagnostics.provenanceViolations.length > 0 ? diagnostics.provenanceViolations : void 0,
|
|
6126
|
+
reviewMode: meta.reviewMode,
|
|
6127
|
+
heldReasons: meta.heldReasons,
|
|
6128
|
+
confidence: signals.confidence,
|
|
6129
|
+
contradicted: signals.contradicted
|
|
5753
6130
|
});
|
|
5754
6131
|
status("?", info(`Candidate ready: ${candidate.id} (${entry.slug})`));
|
|
5755
|
-
return {
|
|
6132
|
+
return {
|
|
6133
|
+
candidate: {
|
|
6134
|
+
id: candidate.id,
|
|
6135
|
+
mode: meta.reviewMode === "policy" ? "held" : "forced",
|
|
6136
|
+
ref: { id: candidate.id, slug: entry.slug, reasons: meta.heldReasons.map((r) => r.code) }
|
|
6137
|
+
}
|
|
6138
|
+
};
|
|
5756
6139
|
}
|
|
5757
6140
|
async function collectCandidateProvenanceViolations(root, fullPage, virtualPath) {
|
|
5758
6141
|
const malformed = checkPageMalformedCitations(fullPage, virtualPath);
|
|
5759
6142
|
const broken = await checkPageBrokenCitations(
|
|
5760
6143
|
fullPage,
|
|
5761
6144
|
virtualPath,
|
|
5762
|
-
|
|
6145
|
+
path38.join(root, SOURCES_DIR)
|
|
5763
6146
|
);
|
|
5764
6147
|
return [...malformed, ...broken];
|
|
5765
6148
|
}
|
|
@@ -5776,7 +6159,7 @@ async function generateSeedPages(root, schema, generation) {
|
|
|
5776
6159
|
}
|
|
5777
6160
|
async function generateSingleSeedPage(root, schema, seed) {
|
|
5778
6161
|
const slug = slugify(seed.title);
|
|
5779
|
-
const pagePath =
|
|
6162
|
+
const pagePath = path38.join(root, CONCEPTS_DIR, `${slug}.md`);
|
|
5780
6163
|
const relatedContent = await loadSeedRelatedPages(root, seed.relatedSlugs ?? []);
|
|
5781
6164
|
const rule = schema.kinds[seed.kind];
|
|
5782
6165
|
const system = buildSeedPagePrompt(seed, rule, relatedContent);
|
|
@@ -5810,7 +6193,7 @@ async function loadSeedRelatedPages(root, slugs) {
|
|
|
5810
6193
|
if (slugs.length === 0) return "";
|
|
5811
6194
|
const contents = [];
|
|
5812
6195
|
for (const slug of slugs) {
|
|
5813
|
-
const pagePath =
|
|
6196
|
+
const pagePath = path38.join(root, CONCEPTS_DIR, `${slug}.md`);
|
|
5814
6197
|
const content = await safeReadFile(pagePath);
|
|
5815
6198
|
if (content) contents.push(content);
|
|
5816
6199
|
}
|
|
@@ -5841,19 +6224,10 @@ async function safelyUpdateEmbeddings(root, changedSlugs) {
|
|
|
5841
6224
|
status("!", warn(`Skipped embeddings update: ${message}`));
|
|
5842
6225
|
}
|
|
5843
6226
|
}
|
|
5844
|
-
async function persistSourceState(root, sourcePath, sourceFile, concepts) {
|
|
5845
|
-
const hash = await hashFile(sourcePath);
|
|
5846
|
-
const entry = {
|
|
5847
|
-
hash,
|
|
5848
|
-
concepts: concepts.map((c) => slugify(c.concept)),
|
|
5849
|
-
compiledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
5850
|
-
};
|
|
5851
|
-
await updateSourceState(root, sourceFile, entry);
|
|
5852
|
-
}
|
|
5853
6227
|
|
|
5854
6228
|
// src/commands/compile.ts
|
|
5855
6229
|
async function compileCommand(options = {}) {
|
|
5856
|
-
if (!
|
|
6230
|
+
if (!existsSync10(SOURCES_DIR)) {
|
|
5857
6231
|
status(
|
|
5858
6232
|
"!",
|
|
5859
6233
|
warn("No sources found. Run `llmwiki ingest <url>` first.")
|
|
@@ -5864,8 +6238,8 @@ async function compileCommand(options = {}) {
|
|
|
5864
6238
|
}
|
|
5865
6239
|
|
|
5866
6240
|
// src/commands/query.ts
|
|
5867
|
-
import { existsSync as
|
|
5868
|
-
import
|
|
6241
|
+
import { existsSync as existsSync11 } from "fs";
|
|
6242
|
+
import path39 from "path";
|
|
5869
6243
|
var PAGE_DIRS = [CONCEPTS_DIR, QUERIES_DIR];
|
|
5870
6244
|
var PAGE_SELECTION_TOOL = {
|
|
5871
6245
|
name: "select_pages",
|
|
@@ -5922,7 +6296,7 @@ async function selectRelevantPages(root, question, debug) {
|
|
|
5922
6296
|
const { pages: rawPages2, reasoning: reasoning2 } = await selectPages(question, filteredIndex);
|
|
5923
6297
|
return { pages: rawPages2, rawPages: rawPages2, reasoning: reasoning2, chunks: [] };
|
|
5924
6298
|
}
|
|
5925
|
-
const indexContent = await safeReadFile(
|
|
6299
|
+
const indexContent = await safeReadFile(path39.join(root, INDEX_FILE));
|
|
5926
6300
|
const { pages: rawPages, reasoning } = await selectPages(question, indexContent);
|
|
5927
6301
|
return { pages: rawPages.map((p) => slugify(p)), rawPages, reasoning, chunks: [] };
|
|
5928
6302
|
}
|
|
@@ -6014,7 +6388,7 @@ async function loadSelectedPages(root, slugs) {
|
|
|
6014
6388
|
for (const slug of slugs) {
|
|
6015
6389
|
let content = "";
|
|
6016
6390
|
for (const dir of PAGE_DIRS) {
|
|
6017
|
-
const candidate = await safeReadFile(
|
|
6391
|
+
const candidate = await safeReadFile(path39.join(root, dir, `${slug}.md`));
|
|
6018
6392
|
if (!candidate) continue;
|
|
6019
6393
|
const { meta } = parseFrontmatter(candidate);
|
|
6020
6394
|
if (meta.orphaned) continue;
|
|
@@ -6065,7 +6439,7 @@ function summarizeAnswer(answer) {
|
|
|
6065
6439
|
}
|
|
6066
6440
|
async function saveQueryPage(root, question, answer) {
|
|
6067
6441
|
const slug = slugify(question);
|
|
6068
|
-
const filePath =
|
|
6442
|
+
const filePath = path39.join(root, QUERIES_DIR, `${slug}.md`);
|
|
6069
6443
|
const frontmatter = buildFrontmatter({
|
|
6070
6444
|
title: question,
|
|
6071
6445
|
summary: summarizeAnswer(answer),
|
|
@@ -6091,7 +6465,7 @@ ${answer}
|
|
|
6091
6465
|
return slug;
|
|
6092
6466
|
}
|
|
6093
6467
|
async function generateAnswer(root, question, options = {}) {
|
|
6094
|
-
if (!
|
|
6468
|
+
if (!existsSync11(path39.join(root, INDEX_FILE))) {
|
|
6095
6469
|
throw new Error("Wiki index not found. Run `llmwiki compile` first.");
|
|
6096
6470
|
}
|
|
6097
6471
|
const selection = await selectRelevantPages(root, question, Boolean(options.debug));
|
|
@@ -6122,7 +6496,7 @@ function buildEmptyResult(selection) {
|
|
|
6122
6496
|
};
|
|
6123
6497
|
}
|
|
6124
6498
|
async function queryCommand(root, question, options) {
|
|
6125
|
-
if (!
|
|
6499
|
+
if (!existsSync11(path39.join(root, INDEX_FILE))) {
|
|
6126
6500
|
status("!", error("Wiki index not found. Run `llmwiki compile` first."));
|
|
6127
6501
|
return;
|
|
6128
6502
|
}
|
|
@@ -6172,12 +6546,12 @@ var DEBUG_CHUNK_PREVIEW_CHARS = 120;
|
|
|
6172
6546
|
|
|
6173
6547
|
// src/commands/watch.ts
|
|
6174
6548
|
import { watch as chokidarWatch } from "chokidar";
|
|
6175
|
-
import { existsSync as
|
|
6176
|
-
import
|
|
6549
|
+
import { existsSync as existsSync12 } from "fs";
|
|
6550
|
+
import path40 from "path";
|
|
6177
6551
|
var DEBOUNCE_MS = 500;
|
|
6178
6552
|
async function watchCommand() {
|
|
6179
|
-
const sourcesPath =
|
|
6180
|
-
if (!
|
|
6553
|
+
const sourcesPath = path40.resolve(SOURCES_DIR);
|
|
6554
|
+
if (!existsSync12(sourcesPath)) {
|
|
6181
6555
|
status(
|
|
6182
6556
|
"!",
|
|
6183
6557
|
warn("No sources/ directory found. Run `llmwiki ingest <url>` first.")
|
|
@@ -6214,7 +6588,7 @@ async function watchCommand() {
|
|
|
6214
6588
|
const scheduleCompile = (eventPath, event) => {
|
|
6215
6589
|
status(
|
|
6216
6590
|
"~",
|
|
6217
|
-
dim(`${event}: ${
|
|
6591
|
+
dim(`${event}: ${path40.basename(eventPath)}`)
|
|
6218
6592
|
);
|
|
6219
6593
|
if (debounceTimer) clearTimeout(debounceTimer);
|
|
6220
6594
|
debounceTimer = setTimeout(triggerCompile, DEBOUNCE_MS);
|
|
@@ -6304,19 +6678,19 @@ async function lintCommand() {
|
|
|
6304
6678
|
}
|
|
6305
6679
|
|
|
6306
6680
|
// src/eval/stats.ts
|
|
6307
|
-
import { readdir as readdir14, appendFile as appendFile2, mkdir as mkdir7, readFile as
|
|
6308
|
-
import { existsSync as
|
|
6309
|
-
import
|
|
6310
|
-
var HISTORY_DIR =
|
|
6311
|
-
var HISTORY_FILE =
|
|
6681
|
+
import { readdir as readdir14, appendFile as appendFile2, mkdir as mkdir7, readFile as readFile25 } from "fs/promises";
|
|
6682
|
+
import { existsSync as existsSync13 } from "fs";
|
|
6683
|
+
import path41 from "path";
|
|
6684
|
+
var HISTORY_DIR = path41.join(".llmwiki", "eval");
|
|
6685
|
+
var HISTORY_FILE = path41.join(HISTORY_DIR, "history.jsonl");
|
|
6312
6686
|
async function countFiles(dir) {
|
|
6313
|
-
if (!
|
|
6687
|
+
if (!existsSync13(dir)) return 0;
|
|
6314
6688
|
const entries = await readdir14(dir);
|
|
6315
6689
|
return entries.filter((e) => e.endsWith(".md")).length;
|
|
6316
6690
|
}
|
|
6317
6691
|
async function collectStats(root) {
|
|
6318
6692
|
const [sourceCount, pages, embeddingStore] = await Promise.all([
|
|
6319
|
-
countFiles(
|
|
6693
|
+
countFiles(path41.join(root, SOURCES_DIR)),
|
|
6320
6694
|
collectAllPages(root),
|
|
6321
6695
|
readEmbeddingStore(root)
|
|
6322
6696
|
]);
|
|
@@ -6340,14 +6714,14 @@ async function collectStats(root) {
|
|
|
6340
6714
|
};
|
|
6341
6715
|
}
|
|
6342
6716
|
async function appendHistory(root, report) {
|
|
6343
|
-
const historyDir =
|
|
6717
|
+
const historyDir = path41.join(root, HISTORY_DIR);
|
|
6344
6718
|
await mkdir7(historyDir, { recursive: true });
|
|
6345
|
-
await appendFile2(
|
|
6719
|
+
await appendFile2(path41.join(root, HISTORY_FILE), JSON.stringify(report) + "\n");
|
|
6346
6720
|
}
|
|
6347
6721
|
async function loadHistory(root, n = 10) {
|
|
6348
|
-
const historyPath =
|
|
6349
|
-
if (!
|
|
6350
|
-
const content = await
|
|
6722
|
+
const historyPath = path41.join(root, HISTORY_FILE);
|
|
6723
|
+
if (!existsSync13(historyPath)) return [];
|
|
6724
|
+
const content = await readFile25(historyPath, "utf-8");
|
|
6351
6725
|
const lines = content.trim().split("\n").filter(Boolean);
|
|
6352
6726
|
const reports = [];
|
|
6353
6727
|
for (const line2 of lines.slice(-n)) {
|
|
@@ -6359,9 +6733,9 @@ async function loadHistory(root, n = 10) {
|
|
|
6359
6733
|
return reports;
|
|
6360
6734
|
}
|
|
6361
6735
|
async function readHistoryLines(root) {
|
|
6362
|
-
const historyPath =
|
|
6363
|
-
if (!
|
|
6364
|
-
const content = await
|
|
6736
|
+
const historyPath = path41.join(root, HISTORY_FILE);
|
|
6737
|
+
if (!existsSync13(historyPath)) return null;
|
|
6738
|
+
const content = await readFile25(historyPath, "utf-8");
|
|
6365
6739
|
return content.trim().split("\n").filter(Boolean);
|
|
6366
6740
|
}
|
|
6367
6741
|
async function loadPreviousReport(root) {
|
|
@@ -6397,6 +6771,7 @@ var ERROR_RULES = /* @__PURE__ */ new Set([
|
|
|
6397
6771
|
"duplicate-concept"
|
|
6398
6772
|
]);
|
|
6399
6773
|
function deductionFor(result) {
|
|
6774
|
+
if (result.rule === "pending-target") return 0;
|
|
6400
6775
|
if (ERROR_RULES.has(result.rule)) return ERROR_DEDUCTION;
|
|
6401
6776
|
if (result.rule === "contradicted-page") return CONTRADICTED_DEDUCTION;
|
|
6402
6777
|
return DEFAULT_DEDUCTION;
|
|
@@ -6422,44 +6797,47 @@ function aggregateRules(results) {
|
|
|
6422
6797
|
}
|
|
6423
6798
|
async function evaluateHealth(root) {
|
|
6424
6799
|
const schema = await loadSchema(root);
|
|
6425
|
-
const allResults =
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
6430
|
-
|
|
6431
|
-
|
|
6432
|
-
|
|
6433
|
-
|
|
6434
|
-
|
|
6435
|
-
|
|
6436
|
-
|
|
6437
|
-
|
|
6800
|
+
const [allResults, pendingReviews] = await Promise.all([
|
|
6801
|
+
Promise.all([
|
|
6802
|
+
checkBrokenWikilinks(root),
|
|
6803
|
+
checkBrokenCitations(root),
|
|
6804
|
+
checkMalformedClaimCitations(root),
|
|
6805
|
+
checkOrphanedPages(root),
|
|
6806
|
+
checkMissingSummaries(root),
|
|
6807
|
+
checkDuplicateConcepts(root),
|
|
6808
|
+
checkEmptyPages(root),
|
|
6809
|
+
checkLowConfidencePages(root),
|
|
6810
|
+
checkContradictedPages(root),
|
|
6811
|
+
checkInferredWithoutCitations(root),
|
|
6812
|
+
checkSchemaCrossLinks(root, schema)
|
|
6813
|
+
]).then((results) => results.flat()),
|
|
6814
|
+
countCandidates(root).catch(() => 0)
|
|
6815
|
+
]);
|
|
6438
6816
|
const rules = aggregateRules(allResults);
|
|
6439
6817
|
const totalDeduction = rules.reduce((sum, r) => sum + r.deduction, 0);
|
|
6440
6818
|
const score = Math.max(0, MAX_SCORE - totalDeduction);
|
|
6441
|
-
return { score, maxScore: MAX_SCORE, rules };
|
|
6819
|
+
return { score, maxScore: MAX_SCORE, rules, pendingReviews };
|
|
6442
6820
|
}
|
|
6443
6821
|
|
|
6444
6822
|
// src/eval/citation-coverage.ts
|
|
6445
|
-
import
|
|
6823
|
+
import path43 from "path";
|
|
6446
6824
|
|
|
6447
6825
|
// src/eval/source-path.ts
|
|
6448
6826
|
import { realpath as realpath5 } from "fs/promises";
|
|
6449
|
-
import
|
|
6827
|
+
import path42 from "path";
|
|
6450
6828
|
function containsParentSegment(file) {
|
|
6451
6829
|
return file.split(/[/\\]/).some((seg) => seg === "..");
|
|
6452
6830
|
}
|
|
6453
6831
|
function isInside(parent, candidate) {
|
|
6454
6832
|
if (candidate === parent) return true;
|
|
6455
|
-
const parentWithSep = parent.endsWith(
|
|
6833
|
+
const parentWithSep = parent.endsWith(path42.sep) ? parent : parent + path42.sep;
|
|
6456
6834
|
return candidate.startsWith(parentWithSep);
|
|
6457
6835
|
}
|
|
6458
6836
|
async function resolveSourceFile(sourcesDir, file) {
|
|
6459
|
-
if (file.length === 0 ||
|
|
6837
|
+
if (file.length === 0 || path42.isAbsolute(file)) return null;
|
|
6460
6838
|
if (containsParentSegment(file)) return null;
|
|
6461
|
-
const joined =
|
|
6462
|
-
if (!isInside(sourcesDir,
|
|
6839
|
+
const joined = path42.join(sourcesDir, file);
|
|
6840
|
+
if (!isInside(sourcesDir, path42.resolve(joined))) return null;
|
|
6463
6841
|
try {
|
|
6464
6842
|
const realDir = await realpath5(sourcesDir);
|
|
6465
6843
|
const realFile = await realpath5(joined);
|
|
@@ -6497,7 +6875,7 @@ async function evaluatePage(slug, body, sourcesDir) {
|
|
|
6497
6875
|
}
|
|
6498
6876
|
async function evaluateCitationCoverage(root) {
|
|
6499
6877
|
const pages = await collectAllPages(root);
|
|
6500
|
-
const sourcesDir =
|
|
6878
|
+
const sourcesDir = path43.join(root, SOURCES_DIR);
|
|
6501
6879
|
let totalProse = 0;
|
|
6502
6880
|
let totalCited = 0;
|
|
6503
6881
|
let totalCitations = 0;
|
|
@@ -6505,7 +6883,7 @@ async function evaluateCitationCoverage(root) {
|
|
|
6505
6883
|
const perPage = [];
|
|
6506
6884
|
for (const { filePath, content } of pages) {
|
|
6507
6885
|
const { body } = parseFrontmatter(content);
|
|
6508
|
-
const slug =
|
|
6886
|
+
const slug = path43.basename(filePath, ".md");
|
|
6509
6887
|
const stats = await evaluatePage(slug, body, sourcesDir);
|
|
6510
6888
|
totalProse += stats.proseParagraphs;
|
|
6511
6889
|
totalCited += stats.citedParagraphs;
|
|
@@ -6528,8 +6906,8 @@ async function evaluateCitationCoverage(root) {
|
|
|
6528
6906
|
|
|
6529
6907
|
// src/eval/source-utilization.ts
|
|
6530
6908
|
import { readdir as readdir15, lstat as lstat3 } from "fs/promises";
|
|
6531
|
-
import { existsSync as
|
|
6532
|
-
import
|
|
6909
|
+
import { existsSync as existsSync14 } from "fs";
|
|
6910
|
+
import path44 from "path";
|
|
6533
6911
|
function collectRawCitedFiles(body) {
|
|
6534
6912
|
const files = /* @__PURE__ */ new Set();
|
|
6535
6913
|
for (const para of splitProseParagraphs(body)) {
|
|
@@ -6541,17 +6919,17 @@ function collectRawCitedFiles(body) {
|
|
|
6541
6919
|
return files;
|
|
6542
6920
|
}
|
|
6543
6921
|
async function listSourceFiles3(dir) {
|
|
6544
|
-
if (!
|
|
6922
|
+
if (!existsSync14(dir)) return [];
|
|
6545
6923
|
const entries = await readdir15(dir);
|
|
6546
6924
|
return entries.filter((e) => e.endsWith(".md"));
|
|
6547
6925
|
}
|
|
6548
6926
|
function pageSlug(filePath) {
|
|
6549
|
-
const dir =
|
|
6550
|
-
return dir + "/" +
|
|
6927
|
+
const dir = path44.basename(path44.dirname(filePath));
|
|
6928
|
+
return dir + "/" + path44.basename(filePath, ".md");
|
|
6551
6929
|
}
|
|
6552
6930
|
async function exclusionReason(sourcesDir, file) {
|
|
6553
|
-
const
|
|
6554
|
-
return
|
|
6931
|
+
const stat4 = await lstat3(path44.join(sourcesDir, file)).catch(() => null);
|
|
6932
|
+
return stat4?.isSymbolicLink() ? "symlink target missing or outside sources/ (excluded)" : "could not be resolved (excluded)";
|
|
6555
6933
|
}
|
|
6556
6934
|
async function resolveSourceInventory(sourcesDir, sourceFiles) {
|
|
6557
6935
|
const fileToReal = /* @__PURE__ */ new Map();
|
|
@@ -6600,7 +6978,7 @@ function buildPerSource(validFiles, fileToReal, citedRealToPages) {
|
|
|
6600
6978
|
return records;
|
|
6601
6979
|
}
|
|
6602
6980
|
async function evaluateSourceUtilization(root) {
|
|
6603
|
-
const sourcesDir =
|
|
6981
|
+
const sourcesDir = path44.join(root, SOURCES_DIR);
|
|
6604
6982
|
const rawFiles = await listSourceFiles3(sourcesDir);
|
|
6605
6983
|
const { validFiles, fileToReal, warnings } = await resolveSourceInventory(sourcesDir, rawFiles);
|
|
6606
6984
|
const totalSources = validFiles.length;
|
|
@@ -6670,11 +7048,11 @@ async function evaluateCitationDepth(root) {
|
|
|
6670
7048
|
|
|
6671
7049
|
// src/eval/citation-support.ts
|
|
6672
7050
|
import { createHash as createHash5 } from "crypto";
|
|
6673
|
-
import { readFile as
|
|
6674
|
-
import { existsSync as
|
|
6675
|
-
import
|
|
6676
|
-
var CACHE_DIR =
|
|
6677
|
-
var CACHE_FILE =
|
|
7051
|
+
import { readFile as readFile26, appendFile as appendFile3, mkdir as mkdir8 } from "fs/promises";
|
|
7052
|
+
import { existsSync as existsSync15 } from "fs";
|
|
7053
|
+
import path45 from "path";
|
|
7054
|
+
var CACHE_DIR = path45.join(".llmwiki", "eval");
|
|
7055
|
+
var CACHE_FILE = path45.join(CACHE_DIR, "citation-cache.jsonl");
|
|
6678
7056
|
var JUDGE_TOOL = {
|
|
6679
7057
|
name: "judge_citation",
|
|
6680
7058
|
description: "Rate how well the source excerpt supports the claim.",
|
|
@@ -6700,7 +7078,7 @@ function makeCacheKey(contentHash, model) {
|
|
|
6700
7078
|
return createHash5("sha256").update(contentHash + JUDGE_CONFIG_HASH + model).digest("hex").slice(0, 16);
|
|
6701
7079
|
}
|
|
6702
7080
|
async function readSourceLines(filePath, start, end) {
|
|
6703
|
-
const content = await
|
|
7081
|
+
const content = await readFile26(filePath, "utf-8");
|
|
6704
7082
|
return content.split("\n").slice(start - 1, end).join("\n");
|
|
6705
7083
|
}
|
|
6706
7084
|
function stripCitationMarkers(paragraph) {
|
|
@@ -6736,11 +7114,11 @@ async function extractPagePairs(slug, body, sourcesDir) {
|
|
|
6736
7114
|
}
|
|
6737
7115
|
async function extractCitationPairs(root) {
|
|
6738
7116
|
const pages = await collectAllPages(root);
|
|
6739
|
-
const sourcesDir =
|
|
7117
|
+
const sourcesDir = path45.join(root, SOURCES_DIR);
|
|
6740
7118
|
const all = [];
|
|
6741
7119
|
for (const { filePath, content } of pages) {
|
|
6742
7120
|
const { body } = parseFrontmatter(content);
|
|
6743
|
-
const slug =
|
|
7121
|
+
const slug = path45.basename(filePath, ".md");
|
|
6744
7122
|
const pairs = await extractPagePairs(slug, body, sourcesDir);
|
|
6745
7123
|
all.push(...pairs);
|
|
6746
7124
|
}
|
|
@@ -6758,9 +7136,9 @@ function selectDeterministicSample(pairs, sampleSize, previousHashes = []) {
|
|
|
6758
7136
|
return [...retained, ...newPairs].slice(0, sampleSize);
|
|
6759
7137
|
}
|
|
6760
7138
|
async function loadCachedJudgements(root) {
|
|
6761
|
-
const cachePath =
|
|
6762
|
-
if (!
|
|
6763
|
-
const content = await
|
|
7139
|
+
const cachePath = path45.join(root, CACHE_FILE);
|
|
7140
|
+
if (!existsSync15(cachePath)) return /* @__PURE__ */ new Map();
|
|
7141
|
+
const content = await readFile26(cachePath, "utf-8");
|
|
6764
7142
|
const map = /* @__PURE__ */ new Map();
|
|
6765
7143
|
for (const line2 of content.trim().split("\n").filter(Boolean)) {
|
|
6766
7144
|
try {
|
|
@@ -6772,8 +7150,8 @@ async function loadCachedJudgements(root) {
|
|
|
6772
7150
|
return map;
|
|
6773
7151
|
}
|
|
6774
7152
|
async function appendCachedJudgement(root, judgement) {
|
|
6775
|
-
await mkdir8(
|
|
6776
|
-
await appendFile3(
|
|
7153
|
+
await mkdir8(path45.join(root, CACHE_DIR), { recursive: true });
|
|
7154
|
+
await appendFile3(path45.join(root, CACHE_FILE), JSON.stringify(judgement) + "\n");
|
|
6777
7155
|
}
|
|
6778
7156
|
function resolveModel() {
|
|
6779
7157
|
const provider = process.env.LLMWIKI_PROVIDER ?? DEFAULT_PROVIDER;
|
|
@@ -6871,15 +7249,15 @@ function computeDelta(current, previous) {
|
|
|
6871
7249
|
}
|
|
6872
7250
|
|
|
6873
7251
|
// src/eval/thresholds.ts
|
|
6874
|
-
import { readFile as
|
|
6875
|
-
import { existsSync as
|
|
6876
|
-
import
|
|
7252
|
+
import { readFile as readFile27 } from "fs/promises";
|
|
7253
|
+
import { existsSync as existsSync16 } from "fs";
|
|
7254
|
+
import path46 from "path";
|
|
6877
7255
|
import yaml4 from "js-yaml";
|
|
6878
|
-
var THRESHOLDS_FILE =
|
|
7256
|
+
var THRESHOLDS_FILE = path46.join(".llmwiki", "eval", "thresholds.yaml");
|
|
6879
7257
|
async function loadThresholds(root) {
|
|
6880
|
-
const configPath =
|
|
6881
|
-
if (!
|
|
6882
|
-
const raw = await
|
|
7258
|
+
const configPath = path46.join(root, THRESHOLDS_FILE);
|
|
7259
|
+
if (!existsSync16(configPath)) return {};
|
|
7260
|
+
const raw = await readFile27(configPath, "utf-8");
|
|
6883
7261
|
return yaml4.load(raw) ?? {};
|
|
6884
7262
|
}
|
|
6885
7263
|
function checkNewThresholds(violations, config, report) {
|
|
@@ -7080,6 +7458,9 @@ function formatHealth(report, delta) {
|
|
|
7080
7458
|
const row = ruleRow(rule);
|
|
7081
7459
|
if (row) rows.push(row);
|
|
7082
7460
|
}
|
|
7461
|
+
if (report.health.pendingReviews > 0) {
|
|
7462
|
+
rows.push(line(` pending review: ${report.health.pendingReviews} candidate(s) awaiting approval`));
|
|
7463
|
+
}
|
|
7083
7464
|
return rows;
|
|
7084
7465
|
}
|
|
7085
7466
|
function formatCoverage(report, delta) {
|
|
@@ -7263,20 +7644,20 @@ function formatJudgementsDisplay(judgements) {
|
|
|
7263
7644
|
}
|
|
7264
7645
|
|
|
7265
7646
|
// src/eval/cache.ts
|
|
7266
|
-
import { unlink as unlink4, readFile as
|
|
7267
|
-
import { existsSync as
|
|
7268
|
-
import
|
|
7269
|
-
var CACHE_FILE2 =
|
|
7647
|
+
import { unlink as unlink4, readFile as readFile28 } from "fs/promises";
|
|
7648
|
+
import { existsSync as existsSync17 } from "fs";
|
|
7649
|
+
import path47 from "path";
|
|
7650
|
+
var CACHE_FILE2 = path47.join(".llmwiki", "eval", "citation-cache.jsonl");
|
|
7270
7651
|
async function clearCitationCache(root) {
|
|
7271
|
-
const cachePath =
|
|
7272
|
-
if (!
|
|
7652
|
+
const cachePath = path47.join(root, CACHE_FILE2);
|
|
7653
|
+
if (!existsSync17(cachePath)) return false;
|
|
7273
7654
|
await unlink4(cachePath);
|
|
7274
7655
|
return true;
|
|
7275
7656
|
}
|
|
7276
7657
|
async function loadCitationCache(root) {
|
|
7277
|
-
const cachePath =
|
|
7278
|
-
if (!
|
|
7279
|
-
const content = await
|
|
7658
|
+
const cachePath = path47.join(root, CACHE_FILE2);
|
|
7659
|
+
if (!existsSync17(cachePath)) return [];
|
|
7660
|
+
const content = await readFile28(cachePath, "utf-8");
|
|
7280
7661
|
const judgements = [];
|
|
7281
7662
|
for (const line2 of content.trim().split("\n").filter(Boolean)) {
|
|
7282
7663
|
try {
|
|
@@ -7370,15 +7751,15 @@ async function evalJudgementsCommand(options = {}) {
|
|
|
7370
7751
|
}
|
|
7371
7752
|
|
|
7372
7753
|
// src/commands/export.ts
|
|
7373
|
-
import
|
|
7754
|
+
import path55 from "path";
|
|
7374
7755
|
import { createRequire } from "module";
|
|
7375
7756
|
|
|
7376
7757
|
// src/export/collect.ts
|
|
7377
|
-
import
|
|
7758
|
+
import path49 from "path";
|
|
7378
7759
|
|
|
7379
7760
|
// src/context/provenance.ts
|
|
7380
7761
|
import { promises as fs } from "fs";
|
|
7381
|
-
import
|
|
7762
|
+
import path48 from "path";
|
|
7382
7763
|
var MAX_SOURCE_WINDOWS = 20;
|
|
7383
7764
|
var MAX_LINES_PER_WINDOW = 30;
|
|
7384
7765
|
var CITATION_KEY_SEPARATOR = " <#> ";
|
|
@@ -7430,7 +7811,7 @@ async function readSourceWindow(root, citation) {
|
|
|
7430
7811
|
return readClampedWindow(realPath, citation);
|
|
7431
7812
|
}
|
|
7432
7813
|
async function resolveSourcesRoot(root) {
|
|
7433
|
-
const candidate =
|
|
7814
|
+
const candidate = path48.join(root, SOURCES_DIR);
|
|
7434
7815
|
try {
|
|
7435
7816
|
return await fs.realpath(candidate);
|
|
7436
7817
|
} catch {
|
|
@@ -7439,10 +7820,10 @@ async function resolveSourcesRoot(root) {
|
|
|
7439
7820
|
}
|
|
7440
7821
|
async function resolveSafePath(sourcesRoot, file) {
|
|
7441
7822
|
if (file.length === 0) return null;
|
|
7442
|
-
if (
|
|
7823
|
+
if (path48.isAbsolute(file)) return null;
|
|
7443
7824
|
if (containsParentSegment2(file)) return null;
|
|
7444
|
-
const joined =
|
|
7445
|
-
const resolved =
|
|
7825
|
+
const joined = path48.join(sourcesRoot, file);
|
|
7826
|
+
const resolved = path48.resolve(joined);
|
|
7446
7827
|
if (!isInside2(sourcesRoot, resolved)) return null;
|
|
7447
7828
|
try {
|
|
7448
7829
|
const realPath = await fs.realpath(resolved);
|
|
@@ -7458,7 +7839,7 @@ function containsParentSegment2(file) {
|
|
|
7458
7839
|
}
|
|
7459
7840
|
function isInside2(parent, candidate) {
|
|
7460
7841
|
if (candidate === parent) return true;
|
|
7461
|
-
const normalizedParent = parent.endsWith(
|
|
7842
|
+
const normalizedParent = parent.endsWith(path48.sep) ? parent : `${parent}${path48.sep}`;
|
|
7462
7843
|
return candidate.startsWith(normalizedParent);
|
|
7463
7844
|
}
|
|
7464
7845
|
async function readClampedWindow(realPath, citation) {
|
|
@@ -7505,7 +7886,7 @@ function resolveSourceHashes(sources, lookup) {
|
|
|
7505
7886
|
// src/export/collect.ts
|
|
7506
7887
|
function buildPagePath(pageDirectory, slug) {
|
|
7507
7888
|
const dir = pageDirectory === "concepts" ? CONCEPTS_DIR : QUERIES_DIR;
|
|
7508
|
-
return
|
|
7889
|
+
return path49.posix.join(dir, `${slug}.md`);
|
|
7509
7890
|
}
|
|
7510
7891
|
function readStringArray(meta, field) {
|
|
7511
7892
|
const value = meta[field];
|
|
@@ -7520,7 +7901,7 @@ function readAdvisoryConfidence(meta) {
|
|
|
7520
7901
|
}
|
|
7521
7902
|
function readProvenanceState(meta) {
|
|
7522
7903
|
const value = meta.provenanceState;
|
|
7523
|
-
if (value === "extracted" || value === "merged" || value === "inferred" || value === "ambiguous") {
|
|
7904
|
+
if (value === "extracted" || value === "merged" || value === "inferred" || value === "ambiguous" || value === "imported") {
|
|
7524
7905
|
return value;
|
|
7525
7906
|
}
|
|
7526
7907
|
return void 0;
|
|
@@ -7539,6 +7920,18 @@ function readContradictedBy(meta) {
|
|
|
7539
7920
|
}
|
|
7540
7921
|
return refs.length > 0 ? refs : void 0;
|
|
7541
7922
|
}
|
|
7923
|
+
function readXOkf(meta) {
|
|
7924
|
+
const x = meta["x-okf"];
|
|
7925
|
+
if (!x || typeof x !== "object") return void 0;
|
|
7926
|
+
const of = x.originalFrontmatter;
|
|
7927
|
+
if (!of || typeof of !== "object") return void 0;
|
|
7928
|
+
const snap = { originalFrontmatter: of };
|
|
7929
|
+
const t = x.type;
|
|
7930
|
+
if (typeof t === "string") snap.type = t;
|
|
7931
|
+
const p = x.okfPath;
|
|
7932
|
+
if (typeof p === "string") snap.okfPath = p;
|
|
7933
|
+
return snap;
|
|
7934
|
+
}
|
|
7542
7935
|
function readPageKind(meta) {
|
|
7543
7936
|
const value = meta.kind;
|
|
7544
7937
|
if (value === "concept" || value === "entity" || value === "comparison" || value === "overview") {
|
|
@@ -7567,6 +7960,7 @@ function toExportPage(raw, snapshot, sourceHashes) {
|
|
|
7567
7960
|
links: extractWikilinkSlugs(raw.body),
|
|
7568
7961
|
body: raw.body,
|
|
7569
7962
|
kind: readPageKind(meta),
|
|
7963
|
+
...readXOkf(meta) ? { xOkf: readXOkf(meta) } : {},
|
|
7570
7964
|
advisoryConfidence: readAdvisoryConfidence(meta),
|
|
7571
7965
|
provenanceState: readProvenanceState(meta),
|
|
7572
7966
|
contradictedBy: readContradictedBy(meta),
|
|
@@ -7827,9 +8221,375 @@ ${pageToSlide(p)}`);
|
|
|
7827
8221
|
return [frontmatter, titleSlide, ...slides, ""].join("\n\n");
|
|
7828
8222
|
}
|
|
7829
8223
|
|
|
8224
|
+
// src/export/okf/run.ts
|
|
8225
|
+
import path54 from "path";
|
|
8226
|
+
|
|
8227
|
+
// src/export/okf/bundle.ts
|
|
8228
|
+
import { mkdir as mkdir9, copyFile as copyFile2, rm, readFile as readFile29, readdir as readdir16, lstat as lstat4 } from "fs/promises";
|
|
8229
|
+
import path53 from "path";
|
|
8230
|
+
|
|
8231
|
+
// src/export/okf/mapping.ts
|
|
8232
|
+
import { createHash as createHash7 } from "crypto";
|
|
8233
|
+
import path50 from "path";
|
|
8234
|
+
var DERIVED_CITATIONS = /\n+#\s+Citations\b[\s\S]*$/;
|
|
8235
|
+
function canonicalBody(body) {
|
|
8236
|
+
return body.replace(DERIVED_CITATIONS, "").replace(/\s*$/, "") + "\n";
|
|
8237
|
+
}
|
|
8238
|
+
function hashCanonicalBody(body) {
|
|
8239
|
+
return createHash7("sha256").update(canonicalBody(body), "utf-8").digest("hex");
|
|
8240
|
+
}
|
|
8241
|
+
function safeRefName(file) {
|
|
8242
|
+
const base = file.replace(/^[./\\]+/, "").replace(/[/\\]+/g, "__").replace(/[^\w.\-]+/g, "_");
|
|
8243
|
+
const hash = createHash7("sha256").update(file).digest("hex").slice(0, 8);
|
|
8244
|
+
const ext = path50.extname(base);
|
|
8245
|
+
const stem = ext ? base.slice(0, -ext.length) : base;
|
|
8246
|
+
return `${stem}-${hash}${ext}`;
|
|
8247
|
+
}
|
|
8248
|
+
var OPTIONAL_XLLMWIKI_FIELDS = [
|
|
8249
|
+
["sources", (p) => p.sources],
|
|
8250
|
+
["confidence", (p) => p.advisoryConfidence],
|
|
8251
|
+
["provenanceState", (p) => p.provenanceState],
|
|
8252
|
+
["contradictedBy", (p) => p.contradictedBy],
|
|
8253
|
+
["freshnessStatus", (p) => p.freshnessStatus],
|
|
8254
|
+
["aliases", (p) => p.aliases],
|
|
8255
|
+
["citations", (p) => p.citations]
|
|
8256
|
+
];
|
|
8257
|
+
function isPresent(value) {
|
|
8258
|
+
if (value === void 0 || value === null) return false;
|
|
8259
|
+
return Array.isArray(value) ? value.length > 0 : true;
|
|
8260
|
+
}
|
|
8261
|
+
function buildXLlmwiki(page) {
|
|
8262
|
+
const x = {
|
|
8263
|
+
schemaVersion: "0.1",
|
|
8264
|
+
contentHash: hashCanonicalBody(page.body),
|
|
8265
|
+
pageDirectory: page.pageDirectory
|
|
8266
|
+
};
|
|
8267
|
+
for (const [field, read] of OPTIONAL_XLLMWIKI_FIELDS) {
|
|
8268
|
+
const value = read(page);
|
|
8269
|
+
if (isPresent(value)) x[field] = value;
|
|
8270
|
+
}
|
|
8271
|
+
return x;
|
|
8272
|
+
}
|
|
8273
|
+
var RECONSTRUCT_STRIP = ["type", "title", "description", "tags", "timestamp", "x-llmwiki", "x-okf"];
|
|
8274
|
+
function applyStandardFields(fm, page) {
|
|
8275
|
+
if (page.title) fm.title = page.title;
|
|
8276
|
+
if (page.summary) fm.description = page.summary;
|
|
8277
|
+
if (page.tags?.length) fm.tags = page.tags;
|
|
8278
|
+
if (page.updatedAt) fm.timestamp = page.updatedAt;
|
|
8279
|
+
}
|
|
8280
|
+
function reconstructForeignFrontmatter(page, x) {
|
|
8281
|
+
const of = page.xOkf.originalFrontmatter;
|
|
8282
|
+
const rawType = typeof of.type === "string" && of.type.trim() ? of.type : page.xOkf.type ?? "concept";
|
|
8283
|
+
const extras = { ...of };
|
|
8284
|
+
for (const k of RECONSTRUCT_STRIP) delete extras[k];
|
|
8285
|
+
const fm = { ...extras, type: rawType, "x-llmwiki": x };
|
|
8286
|
+
applyStandardFields(fm, page);
|
|
8287
|
+
return fm;
|
|
8288
|
+
}
|
|
8289
|
+
function mapPageToOkfFrontmatter(page) {
|
|
8290
|
+
const x = buildXLlmwiki(page);
|
|
8291
|
+
if (page.xOkf) return reconstructForeignFrontmatter(page, x);
|
|
8292
|
+
const fm = { type: page.kind ?? "concept", "x-llmwiki": x };
|
|
8293
|
+
applyStandardFields(fm, page);
|
|
8294
|
+
return fm;
|
|
8295
|
+
}
|
|
8296
|
+
var WIKILINK = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
|
|
8297
|
+
var OKF_LINK = /\[([^\]]+)\]\(\/([^)]+?)\.md\)/g;
|
|
8298
|
+
var FENCE = /(```[\s\S]*?```|~~~[\s\S]*?~~~)/g;
|
|
8299
|
+
function wikilinksToOkf(body, resolve) {
|
|
8300
|
+
return body.split(FENCE).map(
|
|
8301
|
+
(seg, i) => i % 2 === 1 ? seg : seg.replace(WIKILINK, (match, rawSlug, disp) => {
|
|
8302
|
+
const slug = slugify(rawSlug);
|
|
8303
|
+
const target = resolve(slug);
|
|
8304
|
+
if (!target) return match;
|
|
8305
|
+
return `[${disp ?? target.title}](/${target.path})`;
|
|
8306
|
+
})
|
|
8307
|
+
).join("");
|
|
8308
|
+
}
|
|
8309
|
+
function okfLinksToWikilinks(body, resolveLink) {
|
|
8310
|
+
return body.replace(OKF_LINK, (match, text, linkPath) => {
|
|
8311
|
+
const r = resolveLink(linkPath);
|
|
8312
|
+
if (!r) return match;
|
|
8313
|
+
return r.title === text ? `[[${r.slug}]]` : `[[${r.slug}|${text}]]`;
|
|
8314
|
+
});
|
|
8315
|
+
}
|
|
8316
|
+
|
|
8317
|
+
// src/export/okf/citations.ts
|
|
8318
|
+
function renderCitationsSection(citations, refName) {
|
|
8319
|
+
if (citations.length === 0) return "";
|
|
8320
|
+
const lines = citations.map((c, i) => {
|
|
8321
|
+
const span = c.start !== void 0 ? `:${c.start}${c.end !== void 0 ? `-${c.end}` : ""}` : "";
|
|
8322
|
+
const label = `${c.file}${span}`;
|
|
8323
|
+
const name = refName(c.file);
|
|
8324
|
+
return name ? `${i + 1}. [${label}](/references/${name})` : `${i + 1}. ${label}`;
|
|
8325
|
+
});
|
|
8326
|
+
return `
|
|
8327
|
+
# Citations
|
|
8328
|
+
|
|
8329
|
+
${lines.join("\n")}
|
|
8330
|
+
`;
|
|
8331
|
+
}
|
|
8332
|
+
|
|
8333
|
+
// src/export/okf/render-doc.ts
|
|
8334
|
+
function renderOkfDoc(page, resolve, refName) {
|
|
8335
|
+
const fm = mapPageToOkfFrontmatter(page);
|
|
8336
|
+
const canonical = canonicalBody(page.body);
|
|
8337
|
+
const rewritten = wikilinksToOkf(canonical, resolve);
|
|
8338
|
+
const citations = renderCitationsSection(page.citations ?? [], refName);
|
|
8339
|
+
const frontmatter = buildFrontmatter(fm);
|
|
8340
|
+
return `${frontmatter}
|
|
8341
|
+
${rewritten}${citations}`;
|
|
8342
|
+
}
|
|
8343
|
+
|
|
8344
|
+
// src/export/okf/index-log.ts
|
|
8345
|
+
function buildOkfIndex(pages, paths) {
|
|
8346
|
+
const entry = (p) => `* [${p.title}](/${paths.get(p)}) - ${p.summary}`;
|
|
8347
|
+
const concepts = pages.filter((p) => p.pageDirectory === "concepts").map(entry);
|
|
8348
|
+
const queries = pages.filter((p) => p.pageDirectory === "queries").map(entry);
|
|
8349
|
+
const sections = [];
|
|
8350
|
+
if (concepts.length) sections.push(`## Concepts
|
|
8351
|
+
|
|
8352
|
+
${concepts.join("\n")}`);
|
|
8353
|
+
if (queries.length) sections.push(`## Queries
|
|
8354
|
+
|
|
8355
|
+
${queries.join("\n")}`);
|
|
8356
|
+
return `---
|
|
8357
|
+
okf_version: "0.1"
|
|
8358
|
+
---
|
|
8359
|
+
|
|
8360
|
+
# Knowledge Bundle
|
|
8361
|
+
|
|
8362
|
+
${sections.join("\n\n")}
|
|
8363
|
+
`;
|
|
8364
|
+
}
|
|
8365
|
+
function buildOkfLog(entries) {
|
|
8366
|
+
const bulletsByDate = /* @__PURE__ */ new Map();
|
|
8367
|
+
for (const e of entries) {
|
|
8368
|
+
const bullets = bulletsByDate.get(e.date) ?? [];
|
|
8369
|
+
bullets.push(`* **${e.action}** ${e.text}`);
|
|
8370
|
+
bulletsByDate.set(e.date, bullets);
|
|
8371
|
+
}
|
|
8372
|
+
const dates = [...bulletsByDate.keys()].sort((a, b) => a < b ? 1 : -1);
|
|
8373
|
+
const body = dates.map((date) => `## ${date}
|
|
8374
|
+
|
|
8375
|
+
${bulletsByDate.get(date).join("\n")}`).join("\n\n");
|
|
8376
|
+
return `# Log
|
|
8377
|
+
|
|
8378
|
+
${body}
|
|
8379
|
+
`;
|
|
8380
|
+
}
|
|
8381
|
+
var LOG_HEADING = /^##\s+\[(\d{4}-\d{2}-\d{2})T[^\]]*\]\s+([\w-]+)\s+\|\s+(.*)$/gm;
|
|
8382
|
+
function parseLlmwikiLog(content) {
|
|
8383
|
+
const entries = [];
|
|
8384
|
+
for (const m of content.matchAll(LOG_HEADING)) {
|
|
8385
|
+
const [, date, op, text] = m;
|
|
8386
|
+
entries.push({ date, action: op.charAt(0).toUpperCase() + op.slice(1), text: text.trim() });
|
|
8387
|
+
}
|
|
8388
|
+
return entries;
|
|
8389
|
+
}
|
|
8390
|
+
|
|
8391
|
+
// src/export/okf/references.ts
|
|
8392
|
+
import path51 from "path";
|
|
8393
|
+
function collectReferenceFiles(pages) {
|
|
8394
|
+
const files = /* @__PURE__ */ new Set();
|
|
8395
|
+
for (const page of pages) for (const c of page.citations ?? []) files.add(c.file);
|
|
8396
|
+
return [...files];
|
|
8397
|
+
}
|
|
8398
|
+
async function resolveReferences(root, pages) {
|
|
8399
|
+
const resolved = /* @__PURE__ */ new Map();
|
|
8400
|
+
const sourcesDir = await safeRealpath(path51.join(root, SOURCES_DIR)) ?? path51.join(root, SOURCES_DIR);
|
|
8401
|
+
for (const file of collectReferenceFiles(pages)) {
|
|
8402
|
+
const realSrc = await safeRealpath(path51.join(sourcesDir, file));
|
|
8403
|
+
if (!realSrc || !isInsideDir(realSrc, sourcesDir)) continue;
|
|
8404
|
+
resolved.set(file, { srcAbs: realSrc, destName: safeRefName(file) });
|
|
8405
|
+
}
|
|
8406
|
+
return resolved;
|
|
8407
|
+
}
|
|
8408
|
+
|
|
8409
|
+
// src/export/okf/output-paths.ts
|
|
8410
|
+
import path52 from "path";
|
|
8411
|
+
function slugPath(p) {
|
|
8412
|
+
return `${p.pageDirectory}/${p.slug}.md`;
|
|
8413
|
+
}
|
|
8414
|
+
function isSafeOkfPath(okfPath, realOut) {
|
|
8415
|
+
if (typeof okfPath !== "string" || okfPath.length === 0) return false;
|
|
8416
|
+
if (path52.isAbsolute(okfPath)) return false;
|
|
8417
|
+
if (!/^[A-Za-z0-9._/-]+$/.test(okfPath)) return false;
|
|
8418
|
+
const segments = okfPath.split(/[/\\]+/);
|
|
8419
|
+
if (segments.includes("..") || segments.includes(".")) return false;
|
|
8420
|
+
if (!okfPath.endsWith(".md")) return false;
|
|
8421
|
+
if (okfPath === "index.md" || okfPath === "log.md") return false;
|
|
8422
|
+
if (okfPath === "references" || okfPath.startsWith("references/")) return false;
|
|
8423
|
+
return isInsideDir(path52.normalize(path52.join(realOut, okfPath)), realOut);
|
|
8424
|
+
}
|
|
8425
|
+
function compareForeign(a, b) {
|
|
8426
|
+
const ap = a.xOkf.okfPath, bp = b.xOkf.okfPath;
|
|
8427
|
+
if (ap !== bp) return ap < bp ? -1 : 1;
|
|
8428
|
+
if (a.pageDirectory !== b.pageDirectory) return a.pageDirectory < b.pageDirectory ? -1 : 1;
|
|
8429
|
+
if (a.slug !== b.slug) return a.slug < b.slug ? -1 : 1;
|
|
8430
|
+
return 0;
|
|
8431
|
+
}
|
|
8432
|
+
function resolveOutputPaths(pages, realOut) {
|
|
8433
|
+
const owner = /* @__PURE__ */ new Map();
|
|
8434
|
+
for (const p of pages) owner.set(slugPath(p), p);
|
|
8435
|
+
const paths = /* @__PURE__ */ new Map();
|
|
8436
|
+
const warnings = [];
|
|
8437
|
+
const used = /* @__PURE__ */ new Set();
|
|
8438
|
+
const foreign = pages.filter((p) => typeof p.xOkf?.okfPath === "string").sort(compareForeign);
|
|
8439
|
+
const native = pages.filter((p) => typeof p.xOkf?.okfPath !== "string");
|
|
8440
|
+
for (const p of foreign) {
|
|
8441
|
+
const okfPath = p.xOkf.okfPath;
|
|
8442
|
+
const ownerOfOkf = owner.get(okfPath);
|
|
8443
|
+
const usable = isSafeOkfPath(okfPath, realOut) && !used.has(okfPath) && (ownerOfOkf === void 0 || ownerOfOkf === p);
|
|
8444
|
+
if (usable) {
|
|
8445
|
+
paths.set(p, okfPath);
|
|
8446
|
+
used.add(okfPath);
|
|
8447
|
+
} else {
|
|
8448
|
+
const sp = slugPath(p);
|
|
8449
|
+
paths.set(p, sp);
|
|
8450
|
+
used.add(sp);
|
|
8451
|
+
warnings.push(`OKF export: ${okfPath} not restorable (unsafe or collides); using ${sp}`);
|
|
8452
|
+
}
|
|
8453
|
+
}
|
|
8454
|
+
for (const p of native) {
|
|
8455
|
+
const sp = slugPath(p);
|
|
8456
|
+
paths.set(p, sp);
|
|
8457
|
+
used.add(sp);
|
|
8458
|
+
}
|
|
8459
|
+
return { paths, warnings };
|
|
8460
|
+
}
|
|
8461
|
+
|
|
8462
|
+
// src/export/okf/bundle.ts
|
|
8463
|
+
var IGNORED_DIR_ENTRIES = /* @__PURE__ */ new Set([".DS_Store", "Thumbs.db"]);
|
|
8464
|
+
function makeResolver(pages, paths) {
|
|
8465
|
+
const map = new Map(pages.map((p) => [p.slug, { path: paths.get(p), title: p.title }]));
|
|
8466
|
+
return (slug) => map.get(slug) ?? null;
|
|
8467
|
+
}
|
|
8468
|
+
async function writeConfined(out, rel, content) {
|
|
8469
|
+
const abs = await confineUnderRoot(rel, out, { mustExist: false });
|
|
8470
|
+
await atomicWrite(abs, content);
|
|
8471
|
+
return abs;
|
|
8472
|
+
}
|
|
8473
|
+
async function lexists(p) {
|
|
8474
|
+
try {
|
|
8475
|
+
await lstat4(p);
|
|
8476
|
+
return true;
|
|
8477
|
+
} catch {
|
|
8478
|
+
return false;
|
|
8479
|
+
}
|
|
8480
|
+
}
|
|
8481
|
+
async function resolveRealTarget(p) {
|
|
8482
|
+
let cur = path53.resolve(p);
|
|
8483
|
+
const suffix = [];
|
|
8484
|
+
for (; ; ) {
|
|
8485
|
+
const real = await safeRealpath(cur);
|
|
8486
|
+
if (real) return suffix.length ? path53.join(real, ...suffix.reverse()) : real;
|
|
8487
|
+
const parent = path53.dirname(cur);
|
|
8488
|
+
if (parent === cur) return path53.resolve(p);
|
|
8489
|
+
suffix.push(path53.basename(cur));
|
|
8490
|
+
cur = parent;
|
|
8491
|
+
}
|
|
8492
|
+
}
|
|
8493
|
+
async function assertSafeBundleTarget(out, root) {
|
|
8494
|
+
const realRoot = await safeRealpath(root) ?? path53.resolve(root);
|
|
8495
|
+
const target = await resolveRealTarget(out);
|
|
8496
|
+
if (target === path53.parse(target).root) throw new Error("OKF export: refusing to export to the filesystem root");
|
|
8497
|
+
if (target === realRoot) throw new Error("OKF export: refusing to export to the project root");
|
|
8498
|
+
if (isInsideDir(target, path53.join(realRoot, ".git"))) throw new Error("OKF export: refusing to export inside .git");
|
|
8499
|
+
}
|
|
8500
|
+
async function assertNoTopLevelGit(realOut) {
|
|
8501
|
+
if (await lexists(path53.join(realOut, ".git"))) throw new Error("OKF export: refusing to export to a directory containing .git");
|
|
8502
|
+
}
|
|
8503
|
+
async function isRecognizedBundle(realOut) {
|
|
8504
|
+
const idx = path53.join(realOut, "index.md");
|
|
8505
|
+
try {
|
|
8506
|
+
if (!(await lstat4(idx)).isFile()) return false;
|
|
8507
|
+
} catch {
|
|
8508
|
+
return false;
|
|
8509
|
+
}
|
|
8510
|
+
const { meta } = parseFrontmatterStatus(await readFile29(idx, "utf-8"));
|
|
8511
|
+
return typeof meta.okf_version !== "undefined";
|
|
8512
|
+
}
|
|
8513
|
+
async function assertNoNestedGit(realOut) {
|
|
8514
|
+
for (const e of await readdir16(realOut, { withFileTypes: true })) {
|
|
8515
|
+
if (e.name === ".git") throw new Error(`OKF export: refusing to clear ${realOut} \u2014 it contains a nested .git`);
|
|
8516
|
+
if (e.isDirectory()) await assertNoNestedGit(path53.join(realOut, e.name));
|
|
8517
|
+
}
|
|
8518
|
+
}
|
|
8519
|
+
async function clearContents(realOut) {
|
|
8520
|
+
for (const name of await readdir16(realOut)) await rm(path53.join(realOut, name), { recursive: true, force: true });
|
|
8521
|
+
}
|
|
8522
|
+
async function buildLog(root, pageCount) {
|
|
8523
|
+
const fallback = [{ date: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10), action: "Export", text: `${pageCount} doc(s)` }];
|
|
8524
|
+
const llmwikiLog = await readFile29(path53.join(root, "log.md"), "utf-8").catch(() => null);
|
|
8525
|
+
const parsed = llmwikiLog ? parseLlmwikiLog(llmwikiLog) : [];
|
|
8526
|
+
return buildOkfLog(parsed.length ? parsed : fallback);
|
|
8527
|
+
}
|
|
8528
|
+
async function copyResolvedReferences(refs, realOut) {
|
|
8529
|
+
const written = [];
|
|
8530
|
+
for (const { srcAbs, destName } of refs.values()) {
|
|
8531
|
+
const dest = await confineUnderRoot(path53.join("references", destName), realOut, { mustExist: false });
|
|
8532
|
+
await mkdir9(path53.dirname(dest), { recursive: true });
|
|
8533
|
+
await copyFile2(srcAbs, dest);
|
|
8534
|
+
written.push(dest);
|
|
8535
|
+
}
|
|
8536
|
+
return written;
|
|
8537
|
+
}
|
|
8538
|
+
function reportSkippedReferences(pages, refs, onWarn) {
|
|
8539
|
+
const skipped = collectReferenceFiles(pages).filter((f) => !refs.has(f));
|
|
8540
|
+
if (skipped.length === 0) return;
|
|
8541
|
+
onWarn(`OKF export: ${skipped.length} cited source(s) not bundled (missing or outside sources/)`);
|
|
8542
|
+
}
|
|
8543
|
+
async function buildOkfBundle(root, pages, out, onWarn = (m) => status("!", warn(m))) {
|
|
8544
|
+
await assertSafeBundleTarget(out, root);
|
|
8545
|
+
await mkdir9(out, { recursive: true });
|
|
8546
|
+
const realOut = await safeRealpath(out) ?? path53.normalize(out);
|
|
8547
|
+
await assertNoTopLevelGit(realOut);
|
|
8548
|
+
const empty = (await readdir16(realOut)).filter((e) => !IGNORED_DIR_ENTRIES.has(e)).length === 0;
|
|
8549
|
+
const bundle = empty ? false : await isRecognizedBundle(realOut);
|
|
8550
|
+
if (!empty && !bundle) throw new Error(`OKF export: ${out} is not empty and not an OKF bundle; export into a fresh directory or an existing OKF bundle.`);
|
|
8551
|
+
if (bundle) {
|
|
8552
|
+
await assertNoNestedGit(realOut);
|
|
8553
|
+
await clearContents(realOut);
|
|
8554
|
+
}
|
|
8555
|
+
const { paths, warnings: pathWarnings } = resolveOutputPaths(pages, realOut);
|
|
8556
|
+
const resolve = makeResolver(pages, paths);
|
|
8557
|
+
const refs = await resolveReferences(root, pages);
|
|
8558
|
+
const refName = (file) => refs.get(file)?.destName ?? null;
|
|
8559
|
+
const written = [];
|
|
8560
|
+
written.push(await writeConfined(realOut, "index.md", buildOkfIndex(pages, paths)));
|
|
8561
|
+
for (const p of pages) {
|
|
8562
|
+
const docRel = paths.get(p);
|
|
8563
|
+
written.push(await writeConfined(realOut, docRel, renderOkfDoc(p, resolve, refName)));
|
|
8564
|
+
}
|
|
8565
|
+
written.push(...await copyResolvedReferences(refs, realOut));
|
|
8566
|
+
written.push(await writeConfined(realOut, "log.md", await buildLog(root, pages.length)));
|
|
8567
|
+
reportSkippedReferences(pages, refs, onWarn);
|
|
8568
|
+
for (const w of pathWarnings) onWarn(w);
|
|
8569
|
+
return written;
|
|
8570
|
+
}
|
|
8571
|
+
|
|
8572
|
+
// src/export/okf/run.ts
|
|
8573
|
+
async function runOkfExport(root, opts = {}) {
|
|
8574
|
+
const outDir = opts.out ?? path54.join(root, EXPORT_DIR, "okf");
|
|
8575
|
+
const pages = await collectExportPages(root);
|
|
8576
|
+
const warnings = [];
|
|
8577
|
+
const writtenPaths = await buildOkfBundle(root, pages, outDir, (m) => warnings.push(m));
|
|
8578
|
+
return { outDir, writtenPaths, warnings };
|
|
8579
|
+
}
|
|
8580
|
+
|
|
7830
8581
|
// src/export/types.ts
|
|
7831
8582
|
var MARP_SOURCES = ["concepts", "queries", "all"];
|
|
7832
8583
|
var EXPORT_TARGETS = [
|
|
8584
|
+
"llms-txt",
|
|
8585
|
+
"llms-full-txt",
|
|
8586
|
+
"json",
|
|
8587
|
+
"json-ld",
|
|
8588
|
+
"graphml",
|
|
8589
|
+
"marp",
|
|
8590
|
+
"okf"
|
|
8591
|
+
];
|
|
8592
|
+
var DEFAULT_EXPORT_TARGETS = [
|
|
7833
8593
|
"llms-txt",
|
|
7834
8594
|
"llms-full-txt",
|
|
7835
8595
|
"json",
|
|
@@ -7840,18 +8600,20 @@ var EXPORT_TARGETS = [
|
|
|
7840
8600
|
|
|
7841
8601
|
// src/commands/export.ts
|
|
7842
8602
|
var require2 = createRequire(import.meta.url);
|
|
7843
|
-
var EXPORT_DIR = "dist/exports";
|
|
7844
8603
|
var TARGET_FILENAMES = {
|
|
7845
8604
|
"llms-txt": "llms.txt",
|
|
7846
8605
|
"llms-full-txt": "llms-full.txt",
|
|
7847
8606
|
json: "wiki.json",
|
|
7848
8607
|
"json-ld": "wiki.jsonld",
|
|
7849
8608
|
graphml: "wiki.graphml",
|
|
7850
|
-
marp: "wiki.md"
|
|
8609
|
+
marp: "wiki.md",
|
|
8610
|
+
// okf is a directory target; this entry satisfies the exhaustive Record type
|
|
8611
|
+
// but is never accessed (the OKF branch continues before reaching it).
|
|
8612
|
+
okf: "okf"
|
|
7851
8613
|
};
|
|
7852
8614
|
function resolveProjectTitle(root) {
|
|
7853
8615
|
try {
|
|
7854
|
-
const pkg = require2(
|
|
8616
|
+
const pkg = require2(path55.join(root, "package.json"));
|
|
7855
8617
|
return typeof pkg.name === "string" ? pkg.name : "Knowledge Wiki";
|
|
7856
8618
|
} catch {
|
|
7857
8619
|
return "Knowledge Wiki";
|
|
@@ -7887,6 +8649,8 @@ function buildContent(inputs) {
|
|
|
7887
8649
|
return buildGraphml(pages);
|
|
7888
8650
|
case "marp":
|
|
7889
8651
|
return buildMarp(pages, projectTitle, marpSource);
|
|
8652
|
+
case "okf":
|
|
8653
|
+
throw new Error("buildContent called for okf \u2014 this is a programming error");
|
|
7890
8654
|
}
|
|
7891
8655
|
}
|
|
7892
8656
|
function computeReportedPageCount(pages, targets, marpSource) {
|
|
@@ -7904,8 +8668,15 @@ async function runExport(root, options = {}) {
|
|
|
7904
8668
|
const marpSource = resolveMarpSource(options.source);
|
|
7905
8669
|
const written = [];
|
|
7906
8670
|
for (const target of targets) {
|
|
8671
|
+
if (target === "okf") {
|
|
8672
|
+
const { outDir, writtenPaths, warnings } = await runOkfExport(root, { out: options.out });
|
|
8673
|
+
written.push(...writtenPaths);
|
|
8674
|
+
for (const w of warnings) status("!", warn(w));
|
|
8675
|
+
status("+", success(`Exported okf bundle \u2192 ${source(outDir)}`));
|
|
8676
|
+
continue;
|
|
8677
|
+
}
|
|
7907
8678
|
const content = buildContent({ target, pages, projectTitle, marpSource, projectId });
|
|
7908
|
-
const outPath =
|
|
8679
|
+
const outPath = path55.join(root, EXPORT_DIR, TARGET_FILENAMES[target]);
|
|
7909
8680
|
await atomicWrite(outPath, content);
|
|
7910
8681
|
written.push(outPath);
|
|
7911
8682
|
status("+", success(`Exported ${target} \u2192 ${source(outPath)}`));
|
|
@@ -7913,7 +8684,7 @@ async function runExport(root, options = {}) {
|
|
|
7913
8684
|
return { written, pageCount: computeReportedPageCount(pages, targets, marpSource) };
|
|
7914
8685
|
}
|
|
7915
8686
|
function resolveTargets(rawTarget) {
|
|
7916
|
-
if (!rawTarget) return [...
|
|
8687
|
+
if (!rawTarget) return [...DEFAULT_EXPORT_TARGETS];
|
|
7917
8688
|
if (!isValidTarget(rawTarget)) {
|
|
7918
8689
|
throw new Error(
|
|
7919
8690
|
`Unknown export target "${rawTarget}". Valid targets: ${EXPORT_TARGETS.join(", ")}`
|
|
@@ -7930,19 +8701,342 @@ async function exportCommand(root, options) {
|
|
|
7930
8701
|
);
|
|
7931
8702
|
}
|
|
7932
8703
|
|
|
8704
|
+
// src/import/run.ts
|
|
8705
|
+
import path59 from "path";
|
|
8706
|
+
|
|
8707
|
+
// src/import/okf-import.ts
|
|
8708
|
+
import path58 from "path";
|
|
8709
|
+
|
|
8710
|
+
// src/import/okf-read.ts
|
|
8711
|
+
import { readdir as readdir17, readFile as readFile30, stat as stat2 } from "fs/promises";
|
|
8712
|
+
import path56 from "path";
|
|
8713
|
+
|
|
8714
|
+
// src/import/okf-limits.ts
|
|
8715
|
+
var DEFAULT_OKF_LIMITS = {
|
|
8716
|
+
maxFiles: 5e3,
|
|
8717
|
+
maxDocBytes: 2e6,
|
|
8718
|
+
maxTotalBytes: 1e8,
|
|
8719
|
+
// Total directory entries visited during the walk — bounds deep-empty-dir /
|
|
8720
|
+
// many-non-`.md` trees that wouldn't otherwise count toward maxFiles.
|
|
8721
|
+
maxEntries: 1e5
|
|
8722
|
+
};
|
|
8723
|
+
|
|
8724
|
+
// src/import/okf-read.ts
|
|
8725
|
+
var RESERVED = /* @__PURE__ */ new Set(["index.md", "log.md"]);
|
|
8726
|
+
async function listMarkdown(root, dir, limits, acc = [], visited = { n: 0 }) {
|
|
8727
|
+
for (const entry of await readdir17(dir, { withFileTypes: true })) {
|
|
8728
|
+
if (++visited.n > limits.maxEntries) throw new Error(`OKF import: bundle exceeds max entry count (> ${limits.maxEntries})`);
|
|
8729
|
+
const abs = path56.join(dir, entry.name);
|
|
8730
|
+
if (entry.isDirectory()) await listMarkdown(root, abs, limits, acc, visited);
|
|
8731
|
+
else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
8732
|
+
acc.push(path56.relative(root, abs).split(path56.sep).join("/"));
|
|
8733
|
+
if (acc.length > limits.maxFiles) throw new Error(`OKF import: bundle exceeds max file count (> ${limits.maxFiles})`);
|
|
8734
|
+
}
|
|
8735
|
+
}
|
|
8736
|
+
return acc;
|
|
8737
|
+
}
|
|
8738
|
+
async function confinedInside(realRoot, rel) {
|
|
8739
|
+
const real = await safeRealpath(path56.join(realRoot, rel));
|
|
8740
|
+
return real && isInsideDir(real, realRoot) ? real : null;
|
|
8741
|
+
}
|
|
8742
|
+
async function readOkfBundle(bundleDir, overrides = {}, onWarn = (m) => status("!", warn(m))) {
|
|
8743
|
+
const limits = { ...DEFAULT_OKF_LIMITS, ...overrides };
|
|
8744
|
+
const realRoot = await safeRealpath(bundleDir);
|
|
8745
|
+
if (!realRoot) throw new Error(`OKF import: bundle not found: ${bundleDir}`);
|
|
8746
|
+
const rels = (await listMarkdown(realRoot, realRoot, limits)).sort();
|
|
8747
|
+
const docs = [];
|
|
8748
|
+
let total = 0;
|
|
8749
|
+
for (const rel of rels) {
|
|
8750
|
+
if (RESERVED.has(rel)) continue;
|
|
8751
|
+
const real = await confinedInside(realRoot, rel);
|
|
8752
|
+
if (!real) {
|
|
8753
|
+
onWarn(`OKF import: skipped path escaping bundle: ${rel}`);
|
|
8754
|
+
continue;
|
|
8755
|
+
}
|
|
8756
|
+
const size = (await stat2(real)).size;
|
|
8757
|
+
total += size;
|
|
8758
|
+
if (size > limits.maxDocBytes || total > limits.maxTotalBytes) throw new Error(`OKF import: bundle exceeds size limit at ${rel}`);
|
|
8759
|
+
const parsed = parseFrontmatterStatus(await readFile30(real, "utf-8"));
|
|
8760
|
+
const type = parsed.meta.type;
|
|
8761
|
+
if (parsed.malformedFrontmatter || typeof type !== "string" || type.trim() === "") {
|
|
8762
|
+
onWarn(`OKF import: skipped doc without a valid type: ${rel}`);
|
|
8763
|
+
continue;
|
|
8764
|
+
}
|
|
8765
|
+
docs.push({ relPath: rel, meta: parsed.meta, body: parsed.body });
|
|
8766
|
+
}
|
|
8767
|
+
return docs;
|
|
8768
|
+
}
|
|
8769
|
+
|
|
8770
|
+
// src/import/okf-map.ts
|
|
8771
|
+
var KNOWN_KINDS = new Set(PAGE_KINDS);
|
|
8772
|
+
function slugFromRelPath(relPath) {
|
|
8773
|
+
const noExt = relPath.replace(/\.md$/i, "");
|
|
8774
|
+
const stripped = noExt.replace(/^(concepts|queries)\//, "");
|
|
8775
|
+
return slugify(stripped.replace(/[/\\]+/g, "-"));
|
|
8776
|
+
}
|
|
8777
|
+
function humanize(slug) {
|
|
8778
|
+
return slug.replace(/[-_]+/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
|
|
8779
|
+
}
|
|
8780
|
+
function resolveTargetDir(meta, relPath) {
|
|
8781
|
+
const x = meta["x-llmwiki"];
|
|
8782
|
+
if (x?.pageDirectory === "concepts" || x?.pageDirectory === "queries") return x.pageDirectory;
|
|
8783
|
+
return relPath.startsWith("queries/") ? "queries" : "concepts";
|
|
8784
|
+
}
|
|
8785
|
+
function asStringArray(v) {
|
|
8786
|
+
return Array.isArray(v) ? v.filter((s) => typeof s === "string") : [];
|
|
8787
|
+
}
|
|
8788
|
+
function pickString(v, fallback) {
|
|
8789
|
+
return typeof v === "string" && v.trim() ? v : fallback;
|
|
8790
|
+
}
|
|
8791
|
+
function applyXLlmwiki(fields, x) {
|
|
8792
|
+
if (typeof x.confidence === "number") fields.confidence = x.confidence;
|
|
8793
|
+
}
|
|
8794
|
+
function baseFields(meta, ctx, slug) {
|
|
8795
|
+
const x = meta["x-llmwiki"] ?? {};
|
|
8796
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8797
|
+
return {
|
|
8798
|
+
title: pickString(meta.title, humanize(slug)),
|
|
8799
|
+
summary: typeof meta.description === "string" ? meta.description : "",
|
|
8800
|
+
// Strip any PRIOR `okf:` origin tokens before stamping the current bundle's, so a
|
|
8801
|
+
// repeated export->import->export->import cycle keeps exactly one origin token.
|
|
8802
|
+
sources: Array.from(/* @__PURE__ */ new Set([
|
|
8803
|
+
...asStringArray(x.sources).filter((s) => !s.startsWith("okf:")),
|
|
8804
|
+
`okf:${ctx.bundleId}`
|
|
8805
|
+
])),
|
|
8806
|
+
kind: KNOWN_KINDS.has(meta.type) ? meta.type : "concept",
|
|
8807
|
+
// Lossy across an OKF round-trip: `createdAt` is reset to now (OKF carries only
|
|
8808
|
+
// `timestamp`, mapped to `updatedAt`); `modelId`/`promptVersion` are llmwiki-internal
|
|
8809
|
+
// lineage with no OKF representation, so they are not preserved on re-import.
|
|
8810
|
+
createdAt: now,
|
|
8811
|
+
updatedAt: typeof meta.timestamp === "string" ? meta.timestamp : now,
|
|
8812
|
+
provenanceState: "imported"
|
|
8813
|
+
};
|
|
8814
|
+
}
|
|
8815
|
+
function buildXokf(meta, okfPath) {
|
|
8816
|
+
const rawType = typeof meta.type === "string" ? meta.type : "concept";
|
|
8817
|
+
const known = KNOWN_KINDS.has(rawType);
|
|
8818
|
+
return { ...known ? {} : { type: rawType }, okfPath, originalFrontmatter: meta };
|
|
8819
|
+
}
|
|
8820
|
+
function buildPageFields(doc, ctx, slug) {
|
|
8821
|
+
const meta = doc.meta;
|
|
8822
|
+
const fields = baseFields(meta, ctx, slug);
|
|
8823
|
+
if (Array.isArray(meta.tags)) fields.tags = asStringArray(meta.tags);
|
|
8824
|
+
applyXLlmwiki(fields, meta["x-llmwiki"] ?? {});
|
|
8825
|
+
fields["x-okf"] = buildXokf(meta, doc.relPath);
|
|
8826
|
+
return fields;
|
|
8827
|
+
}
|
|
8828
|
+
function okfDocToPage(doc, ctx) {
|
|
8829
|
+
const slug = slugFromRelPath(doc.relPath);
|
|
8830
|
+
const fields = buildPageFields(doc, ctx, slug);
|
|
8831
|
+
const isNative = doc.meta["x-llmwiki"] !== void 0;
|
|
8832
|
+
const resolveLink = (linkPath) => {
|
|
8833
|
+
const linkSlug = slugFromRelPath(linkPath);
|
|
8834
|
+
const title = ctx.titleOf(linkSlug);
|
|
8835
|
+
return title !== null ? { slug: linkSlug, title } : null;
|
|
8836
|
+
};
|
|
8837
|
+
const body = isNative ? okfLinksToWikilinks(canonicalBody(doc.body), resolveLink) : doc.body;
|
|
8838
|
+
const pageBody = `${buildFrontmatter(fields)}
|
|
8839
|
+
${body}`;
|
|
8840
|
+
return {
|
|
8841
|
+
slug,
|
|
8842
|
+
title: fields.title,
|
|
8843
|
+
summary: fields.summary,
|
|
8844
|
+
sources: fields.sources,
|
|
8845
|
+
targetDirectory: resolveTargetDir(doc.meta, doc.relPath),
|
|
8846
|
+
okfPath: doc.relPath,
|
|
8847
|
+
body: pageBody
|
|
8848
|
+
};
|
|
8849
|
+
}
|
|
8850
|
+
|
|
8851
|
+
// src/import/okf-collision.ts
|
|
8852
|
+
import { access } from "fs/promises";
|
|
8853
|
+
import path57 from "path";
|
|
8854
|
+
async function fileExists(p) {
|
|
8855
|
+
try {
|
|
8856
|
+
await access(p);
|
|
8857
|
+
return true;
|
|
8858
|
+
} catch {
|
|
8859
|
+
return false;
|
|
8860
|
+
}
|
|
8861
|
+
}
|
|
8862
|
+
async function filterCollisions(root, pages) {
|
|
8863
|
+
const pendingSlugs = new Set((await listCandidates(root)).map((c) => c.slug));
|
|
8864
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
8865
|
+
const kept = [];
|
|
8866
|
+
const skipped = [];
|
|
8867
|
+
for (const page of pages) {
|
|
8868
|
+
let reason = null;
|
|
8869
|
+
if (claimed.has(page.slug)) reason = "duplicate-in-bundle";
|
|
8870
|
+
else if (pendingSlugs.has(page.slug)) reason = "pending-candidate";
|
|
8871
|
+
else if (await fileExists(path57.join(root, CONCEPTS_DIR, `${page.slug}.md`)) || await fileExists(path57.join(root, QUERIES_DIR, `${page.slug}.md`))) reason = "live-page";
|
|
8872
|
+
if (reason) {
|
|
8873
|
+
skipped.push({ slug: page.slug, okfPath: page.okfPath, reason });
|
|
8874
|
+
} else {
|
|
8875
|
+
claimed.add(page.slug);
|
|
8876
|
+
kept.push(page);
|
|
8877
|
+
}
|
|
8878
|
+
}
|
|
8879
|
+
return { kept, skipped };
|
|
8880
|
+
}
|
|
8881
|
+
|
|
8882
|
+
// src/import/okf-import.ts
|
|
8883
|
+
function bundleIdFor(bundleDir) {
|
|
8884
|
+
return slugify(path58.basename(path58.resolve(bundleDir))) || "bundle";
|
|
8885
|
+
}
|
|
8886
|
+
function titleResolver(docs) {
|
|
8887
|
+
const map = /* @__PURE__ */ new Map();
|
|
8888
|
+
for (const d of docs) {
|
|
8889
|
+
const slug = slugFromRelPath(d.relPath);
|
|
8890
|
+
if (typeof d.meta.title === "string") map.set(slug, d.meta.title);
|
|
8891
|
+
}
|
|
8892
|
+
return (slug) => map.get(slug) ?? null;
|
|
8893
|
+
}
|
|
8894
|
+
async function importOkfBundle(bundleDir, root, overrides = {}, onWarn) {
|
|
8895
|
+
const docs = await readOkfBundle(bundleDir, overrides, onWarn);
|
|
8896
|
+
const ctx = { bundleId: bundleIdFor(bundleDir), titleOf: titleResolver(docs) };
|
|
8897
|
+
const mapped = docs.map((d) => okfDocToPage(d, ctx));
|
|
8898
|
+
const { kept, skipped } = await filterCollisions(root, mapped);
|
|
8899
|
+
return { pages: kept, skipped };
|
|
8900
|
+
}
|
|
8901
|
+
|
|
8902
|
+
// src/commands/import-core.ts
|
|
8903
|
+
function isWritable(page) {
|
|
8904
|
+
return page.slug.length > 0 && validateWikiPage(page.body);
|
|
8905
|
+
}
|
|
8906
|
+
|
|
8907
|
+
// src/import/okf-refresh.ts
|
|
8908
|
+
async function safelyUpdateEmbeddings2(root, slugs) {
|
|
8909
|
+
try {
|
|
8910
|
+
await updateEmbeddings(root, slugs);
|
|
8911
|
+
} catch (err) {
|
|
8912
|
+
status("!", warn(`Embeddings update skipped: ${err instanceof Error ? err.message : err}`));
|
|
8913
|
+
}
|
|
8914
|
+
}
|
|
8915
|
+
async function refreshAfterImport(root, slugs) {
|
|
8916
|
+
await resolveLinks(root, slugs, slugs);
|
|
8917
|
+
await generateIndex(root);
|
|
8918
|
+
await generateMOC(root);
|
|
8919
|
+
await safelyUpdateEmbeddings2(root, slugs);
|
|
8920
|
+
}
|
|
8921
|
+
|
|
8922
|
+
// src/import/run-errors.ts
|
|
8923
|
+
var LockUnavailableError = class extends Error {
|
|
8924
|
+
constructor(message = "Could not acquire lock. Try again later.") {
|
|
8925
|
+
super(message);
|
|
8926
|
+
this.name = "LockUnavailableError";
|
|
8927
|
+
}
|
|
8928
|
+
};
|
|
8929
|
+
var QueueFullError = class extends Error {
|
|
8930
|
+
constructor(message = "Review queue would exceed the cap; approve/reject pending candidates first.") {
|
|
8931
|
+
super(message);
|
|
8932
|
+
this.name = "QueueFullError";
|
|
8933
|
+
}
|
|
8934
|
+
};
|
|
8935
|
+
|
|
8936
|
+
// src/import/run.ts
|
|
8937
|
+
var toImported = (p) => ({ slug: p.slug, okfPath: p.okfPath, targetDirectory: p.targetDirectory });
|
|
8938
|
+
function partition(pages, collisionSkips) {
|
|
8939
|
+
const valid = [];
|
|
8940
|
+
const invalid = [];
|
|
8941
|
+
for (const p of pages) {
|
|
8942
|
+
if (isWritable(p)) valid.push(p);
|
|
8943
|
+
else invalid.push({ slug: p.slug, okfPath: p.okfPath, reason: "invalid-page" });
|
|
8944
|
+
}
|
|
8945
|
+
return { valid, skipped: [...collisionSkips, ...invalid] };
|
|
8946
|
+
}
|
|
8947
|
+
async function stageAll(root, valid) {
|
|
8948
|
+
for (const page of valid) {
|
|
8949
|
+
await writeCandidate(root, {
|
|
8950
|
+
title: page.title,
|
|
8951
|
+
slug: page.slug,
|
|
8952
|
+
summary: page.summary,
|
|
8953
|
+
sources: page.sources,
|
|
8954
|
+
body: page.body,
|
|
8955
|
+
reviewMode: "imported",
|
|
8956
|
+
heldReasons: [{ code: "imported-okf" }],
|
|
8957
|
+
targetDirectory: page.targetDirectory,
|
|
8958
|
+
okfPath: page.okfPath
|
|
8959
|
+
});
|
|
8960
|
+
}
|
|
8961
|
+
}
|
|
8962
|
+
async function writeAll(root, valid) {
|
|
8963
|
+
const written = [];
|
|
8964
|
+
try {
|
|
8965
|
+
for (const page of valid) {
|
|
8966
|
+
const dir = page.targetDirectory === "queries" ? QUERIES_DIR : CONCEPTS_DIR;
|
|
8967
|
+
await atomicWrite(path59.join(root, dir, `${page.slug}.md`), page.body);
|
|
8968
|
+
written.push(page.slug);
|
|
8969
|
+
}
|
|
8970
|
+
} finally {
|
|
8971
|
+
if (written.length) await refreshAfterImport(root, written);
|
|
8972
|
+
}
|
|
8973
|
+
}
|
|
8974
|
+
async function runOkfImport(root, dir, opts = {}) {
|
|
8975
|
+
const warnings = [];
|
|
8976
|
+
if (opts.dryRun) {
|
|
8977
|
+
const { pages, skipped } = await importOkfBundle(dir, root, {}, (m) => warnings.push(m));
|
|
8978
|
+
const { valid, skipped: allSkipped } = partition(pages, skipped);
|
|
8979
|
+
return { mode: "dry-run", pages: valid.map(toImported), skipped: allSkipped, warnings, nextAction: "Re-run without dryRun to apply." };
|
|
8980
|
+
}
|
|
8981
|
+
const locked = await withQuiet(() => acquireLock(root));
|
|
8982
|
+
if (!locked) throw new LockUnavailableError();
|
|
8983
|
+
try {
|
|
8984
|
+
const { pages, skipped } = await importOkfBundle(dir, root, {}, (m) => warnings.push(m));
|
|
8985
|
+
const { valid, skipped: allSkipped } = partition(pages, skipped);
|
|
8986
|
+
if (opts.maxNewCandidates !== void 0 && !opts.trusted) {
|
|
8987
|
+
const pending = (await listCandidates(root)).length;
|
|
8988
|
+
if (pending + valid.length > opts.maxNewCandidates) throw new QueueFullError();
|
|
8989
|
+
}
|
|
8990
|
+
if (opts.trusted) await writeAll(root, valid);
|
|
8991
|
+
else await stageAll(root, valid);
|
|
8992
|
+
return {
|
|
8993
|
+
mode: opts.trusted ? "written" : "staged",
|
|
8994
|
+
pages: valid.map(toImported),
|
|
8995
|
+
skipped: allSkipped,
|
|
8996
|
+
warnings,
|
|
8997
|
+
nextAction: opts.trusted ? `${valid.length} page(s) written live.` : "Review and approve before it goes live: llmwiki review list."
|
|
8998
|
+
};
|
|
8999
|
+
} finally {
|
|
9000
|
+
await releaseLock(root);
|
|
9001
|
+
}
|
|
9002
|
+
}
|
|
9003
|
+
|
|
9004
|
+
// src/commands/import.ts
|
|
9005
|
+
function printReport(report) {
|
|
9006
|
+
for (const w of report.warnings) status("!", warn(w));
|
|
9007
|
+
for (const p of report.pages) status("+", `${p.slug} (${p.targetDirectory}) \u2190 ${p.okfPath}`);
|
|
9008
|
+
for (const s of report.skipped) status("!", warn(`skip ${s.okfPath} \u2014 ${s.reason}`));
|
|
9009
|
+
const verb = report.mode === "written" ? "wrote" : report.mode === "dry-run" ? "would import" : "staged";
|
|
9010
|
+
status("+", success(`OKF import: ${verb} ${report.pages.length} page(s); skipped ${report.skipped.length}.`));
|
|
9011
|
+
}
|
|
9012
|
+
async function importCommand(root, options) {
|
|
9013
|
+
if (!options.okf) throw new Error("import: --okf <dir> is required");
|
|
9014
|
+
try {
|
|
9015
|
+
const report = await runOkfImport(root, options.okf, { trusted: options.trusted, dryRun: options.dryRun });
|
|
9016
|
+
printReport(report);
|
|
9017
|
+
} catch (err) {
|
|
9018
|
+
if (err instanceof LockUnavailableError) {
|
|
9019
|
+
status("!", error(err.message));
|
|
9020
|
+
process.exitCode = 1;
|
|
9021
|
+
return;
|
|
9022
|
+
}
|
|
9023
|
+
throw err;
|
|
9024
|
+
}
|
|
9025
|
+
}
|
|
9026
|
+
|
|
7933
9027
|
// src/commands/schema.ts
|
|
7934
|
-
import { existsSync as
|
|
7935
|
-
import { mkdir as
|
|
7936
|
-
import
|
|
9028
|
+
import { existsSync as existsSync18 } from "fs";
|
|
9029
|
+
import { mkdir as mkdir10, writeFile as writeFile5 } from "fs/promises";
|
|
9030
|
+
import path60 from "path";
|
|
7937
9031
|
async function schemaInitCommand() {
|
|
7938
9032
|
const root = process.cwd();
|
|
7939
9033
|
const defaults = buildDefaultSchema();
|
|
7940
9034
|
const targetPath = defaultSchemaInitPath(root);
|
|
7941
|
-
if (
|
|
9035
|
+
if (existsSync18(targetPath)) {
|
|
7942
9036
|
status("!", warn(`Schema file already exists at ${targetPath}`));
|
|
7943
9037
|
return;
|
|
7944
9038
|
}
|
|
7945
|
-
await
|
|
9039
|
+
await mkdir10(path60.dirname(targetPath), { recursive: true });
|
|
7946
9040
|
const serializable = {
|
|
7947
9041
|
version: defaults.version,
|
|
7948
9042
|
defaultKind: defaults.defaultKind,
|
|
@@ -7961,6 +9055,12 @@ async function schemaShowCommand() {
|
|
|
7961
9055
|
}
|
|
7962
9056
|
|
|
7963
9057
|
// src/commands/review-list.ts
|
|
9058
|
+
function formatMode(candidate) {
|
|
9059
|
+
return candidate.reviewMode;
|
|
9060
|
+
}
|
|
9061
|
+
function formatReasons(candidate) {
|
|
9062
|
+
return candidate.heldReasons.map((r) => r.code).join(", ");
|
|
9063
|
+
}
|
|
7964
9064
|
async function reviewListCommand() {
|
|
7965
9065
|
header("Pending review candidates");
|
|
7966
9066
|
const candidates = await listCandidates(process.cwd());
|
|
@@ -7970,7 +9070,8 @@ async function reviewListCommand() {
|
|
|
7970
9070
|
}
|
|
7971
9071
|
for (const candidate of candidates) {
|
|
7972
9072
|
const sources = candidate.sources.join(", ");
|
|
7973
|
-
const
|
|
9073
|
+
const mode = `${formatMode(candidate)}: ${formatReasons(candidate)}`;
|
|
9074
|
+
const meta = dim(`${candidate.generatedAt} | ${mode} | sources: ${sources}`);
|
|
7974
9075
|
status("?", `${info(candidate.id)} \u2192 ${candidate.slug} ${meta}`);
|
|
7975
9076
|
}
|
|
7976
9077
|
status(
|
|
@@ -7980,6 +9081,31 @@ async function reviewListCommand() {
|
|
|
7980
9081
|
}
|
|
7981
9082
|
|
|
7982
9083
|
// src/commands/review-show.ts
|
|
9084
|
+
function candidateReasons(candidate) {
|
|
9085
|
+
return candidate.heldReasons.map((reason) => reason.detail ? `${reason.code} (${reason.detail})` : reason.code);
|
|
9086
|
+
}
|
|
9087
|
+
function printReviewMetadata(candidate) {
|
|
9088
|
+
status("i", dim(`review: ${candidate.reviewMode}`));
|
|
9089
|
+
status("i", dim(`reasons: ${candidateReasons(candidate).join(", ")}`));
|
|
9090
|
+
if (candidate.okfPath) {
|
|
9091
|
+
status("i", dim(`okfPath: ${candidate.okfPath}`));
|
|
9092
|
+
}
|
|
9093
|
+
if (candidate.confidence !== void 0) {
|
|
9094
|
+
status("i", dim(`confidence: ${candidate.confidence}`));
|
|
9095
|
+
}
|
|
9096
|
+
if (candidate.contradicted !== void 0) {
|
|
9097
|
+
status("i", dim(`contradicted: ${candidate.contradicted}`));
|
|
9098
|
+
}
|
|
9099
|
+
}
|
|
9100
|
+
function printImportedWarning(candidate) {
|
|
9101
|
+
if (candidate.reviewMode !== "imported") return;
|
|
9102
|
+
status(
|
|
9103
|
+
"!",
|
|
9104
|
+
warn(
|
|
9105
|
+
"Imported from an external OKF bundle \u2014 treat the body as UNTRUSTED content (possible prompt injection); provenance is unverified. Review the full body before approving."
|
|
9106
|
+
)
|
|
9107
|
+
);
|
|
9108
|
+
}
|
|
7983
9109
|
async function reviewShowCommand(id) {
|
|
7984
9110
|
const candidate = await loadCandidateOrFail(process.cwd(), id);
|
|
7985
9111
|
if (!candidate) return;
|
|
@@ -7989,6 +9115,8 @@ async function reviewShowCommand(id) {
|
|
|
7989
9115
|
status("i", dim(`summary: ${candidate.summary}`));
|
|
7990
9116
|
status("i", dim(`sources: ${candidate.sources.join(", ")}`));
|
|
7991
9117
|
status("i", dim(`generated: ${candidate.generatedAt}`));
|
|
9118
|
+
printReviewMetadata(candidate);
|
|
9119
|
+
printImportedWarning(candidate);
|
|
7992
9120
|
console.log();
|
|
7993
9121
|
console.log(candidate.body);
|
|
7994
9122
|
if (candidate.schemaViolations && candidate.schemaViolations.length > 0) {
|
|
@@ -8008,7 +9136,7 @@ async function reviewShowCommand(id) {
|
|
|
8008
9136
|
}
|
|
8009
9137
|
|
|
8010
9138
|
// src/commands/review-approve.ts
|
|
8011
|
-
import
|
|
9139
|
+
import path61 from "path";
|
|
8012
9140
|
|
|
8013
9141
|
// src/commands/review-helpers.ts
|
|
8014
9142
|
async function runReviewUnderLock(id, underLock) {
|
|
@@ -8040,7 +9168,8 @@ async function approveUnderLock(root, id) {
|
|
|
8040
9168
|
process.exitCode = 1;
|
|
8041
9169
|
return;
|
|
8042
9170
|
}
|
|
8043
|
-
const
|
|
9171
|
+
const dir = candidate.targetDirectory === "queries" ? QUERIES_DIR : CONCEPTS_DIR;
|
|
9172
|
+
const pagePath = path61.join(root, dir, `${candidate.slug}.md`);
|
|
8044
9173
|
await atomicWrite(pagePath, candidate.body);
|
|
8045
9174
|
status("+", success(`Approved \u2192 ${source(pagePath)}`));
|
|
8046
9175
|
await persistCandidateSourceStates(root, candidate);
|
|
@@ -8051,28 +9180,28 @@ async function approveUnderLock(root, id) {
|
|
|
8051
9180
|
async function persistCandidateSourceStates(root, candidate) {
|
|
8052
9181
|
const states = candidate.sourceStates;
|
|
8053
9182
|
if (!states) return;
|
|
8054
|
-
const
|
|
8055
|
-
|
|
8056
|
-
if (otherSources.has(sourceFile)) continue;
|
|
8057
|
-
await updateSourceState(root, sourceFile, entry);
|
|
9183
|
+
for (const [sourceFile, candidateEntry] of Object.entries(states)) {
|
|
9184
|
+
await addApprovedSlugToSourceState(root, sourceFile, candidate.slug, candidateEntry.hash);
|
|
8058
9185
|
}
|
|
8059
9186
|
}
|
|
8060
|
-
async function
|
|
8061
|
-
const
|
|
8062
|
-
const
|
|
8063
|
-
|
|
8064
|
-
|
|
8065
|
-
|
|
8066
|
-
|
|
8067
|
-
|
|
9187
|
+
async function addApprovedSlugToSourceState(root, sourceFile, approvedSlug, sourceHash) {
|
|
9188
|
+
const currentState = await readState(root);
|
|
9189
|
+
const existing = currentState.sources[sourceFile];
|
|
9190
|
+
const concepts = existing?.concepts ?? [];
|
|
9191
|
+
const merged = Array.from(/* @__PURE__ */ new Set([...concepts, approvedSlug]));
|
|
9192
|
+
await updateSourceState(root, sourceFile, {
|
|
9193
|
+
hash: sourceHash,
|
|
9194
|
+
concepts: merged,
|
|
9195
|
+
compiledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9196
|
+
});
|
|
8068
9197
|
}
|
|
8069
9198
|
async function refreshWikiAfterApproval(root, slug) {
|
|
8070
9199
|
await resolveLinks(root, [slug], [slug]);
|
|
8071
9200
|
await generateIndex(root);
|
|
8072
9201
|
await generateMOC(root);
|
|
8073
|
-
await
|
|
9202
|
+
await safelyUpdateEmbeddings3(root, [slug]);
|
|
8074
9203
|
}
|
|
8075
|
-
async function
|
|
9204
|
+
async function safelyUpdateEmbeddings3(root, slugs) {
|
|
8076
9205
|
try {
|
|
8077
9206
|
await updateEmbeddings(root, slugs);
|
|
8078
9207
|
} catch (err) {
|
|
@@ -8096,32 +9225,32 @@ async function rejectUnderLock(root, id) {
|
|
|
8096
9225
|
}
|
|
8097
9226
|
|
|
8098
9227
|
// src/commands/rules.ts
|
|
8099
|
-
import { existsSync as
|
|
8100
|
-
import
|
|
9228
|
+
import { existsSync as existsSync20 } from "fs";
|
|
9229
|
+
import path65 from "path";
|
|
8101
9230
|
|
|
8102
9231
|
// src/compiler/rule-extractor.ts
|
|
8103
|
-
import { readFile as
|
|
8104
|
-
import
|
|
9232
|
+
import { readFile as readFile32 } from "fs/promises";
|
|
9233
|
+
import path64 from "path";
|
|
8105
9234
|
|
|
8106
9235
|
// src/compiler/rule-state.ts
|
|
8107
|
-
import { readFile as
|
|
8108
|
-
import { existsSync as
|
|
8109
|
-
import
|
|
9236
|
+
import { readFile as readFile31, writeFile as writeFile6, rename as rename4, mkdir as mkdir11 } from "fs/promises";
|
|
9237
|
+
import { existsSync as existsSync19 } from "fs";
|
|
9238
|
+
import path62 from "path";
|
|
8110
9239
|
function emptyRuleState() {
|
|
8111
9240
|
return { version: 1, indexHash: "", sources: {} };
|
|
8112
9241
|
}
|
|
8113
9242
|
async function readRuleState(root) {
|
|
8114
|
-
const filePath =
|
|
8115
|
-
if (!
|
|
9243
|
+
const filePath = path62.join(root, RULE_STATE_FILE);
|
|
9244
|
+
if (!existsSync19(filePath)) return emptyRuleState();
|
|
8116
9245
|
try {
|
|
8117
|
-
return JSON.parse(await
|
|
9246
|
+
return JSON.parse(await readFile31(filePath, "utf-8"));
|
|
8118
9247
|
} catch {
|
|
8119
9248
|
return emptyRuleState();
|
|
8120
9249
|
}
|
|
8121
9250
|
}
|
|
8122
9251
|
async function writeRuleState(root, state) {
|
|
8123
|
-
await
|
|
8124
|
-
const filePath =
|
|
9252
|
+
await mkdir11(path62.join(root, LLMWIKI_DIR), { recursive: true });
|
|
9253
|
+
const filePath = path62.join(root, RULE_STATE_FILE);
|
|
8125
9254
|
const tmpPath = `${filePath}.tmp`;
|
|
8126
9255
|
await writeFile6(tmpPath, JSON.stringify(state, null, 2), "utf-8");
|
|
8127
9256
|
await rename4(tmpPath, filePath);
|
|
@@ -8255,15 +9384,15 @@ function parseRules(toolOutput) {
|
|
|
8255
9384
|
}
|
|
8256
9385
|
|
|
8257
9386
|
// src/compiler/rule-candidates.ts
|
|
8258
|
-
import
|
|
8259
|
-
import { createHash as
|
|
9387
|
+
import path63 from "path";
|
|
9388
|
+
import { createHash as createHash8 } from "crypto";
|
|
8260
9389
|
var CONFIDENCE_VALUES = ["low", "medium", "high"];
|
|
8261
9390
|
var STATUS_VALUES = ["proposed", "approved", "rejected"];
|
|
8262
9391
|
function ruleCandidatePath(root, id) {
|
|
8263
|
-
return
|
|
9392
|
+
return path63.join(root, RULE_CANDIDATES_DIR, `${id}${CANDIDATE_JSON_EXT}`);
|
|
8264
9393
|
}
|
|
8265
9394
|
function ruleArchivePath(root, id) {
|
|
8266
|
-
return
|
|
9395
|
+
return path63.join(root, RULE_CANDIDATES_ARCHIVE_DIR, `${id}${CANDIDATE_JSON_EXT}`);
|
|
8267
9396
|
}
|
|
8268
9397
|
var CATEGORY_CAP = 64;
|
|
8269
9398
|
var TITLE_CAP = 256;
|
|
@@ -8278,7 +9407,7 @@ function sanitizeRuleCategory(raw) {
|
|
|
8278
9407
|
}
|
|
8279
9408
|
function buildRuleSlug(title, contentSignature) {
|
|
8280
9409
|
const base = slugify(title);
|
|
8281
|
-
const hash =
|
|
9410
|
+
const hash = createHash8("sha256").update(contentSignature).digest("hex").slice(0, 8);
|
|
8282
9411
|
return base ? `${base}-${hash}` : `rule-${hash}`;
|
|
8283
9412
|
}
|
|
8284
9413
|
function validateRuleCandidate(c) {
|
|
@@ -8441,7 +9570,7 @@ async function readRuleCandidate(root, fileId) {
|
|
|
8441
9570
|
}
|
|
8442
9571
|
}
|
|
8443
9572
|
async function listRuleCandidates(root) {
|
|
8444
|
-
const dir =
|
|
9573
|
+
const dir = path63.join(root, RULE_CANDIDATES_DIR);
|
|
8445
9574
|
const fileIds = await listCandidateFileIds(dir);
|
|
8446
9575
|
const candidates = [];
|
|
8447
9576
|
for (const fileId of fileIds) {
|
|
@@ -8471,7 +9600,7 @@ async function archiveRuleCandidate(root, fileId) {
|
|
|
8471
9600
|
}
|
|
8472
9601
|
|
|
8473
9602
|
// src/compiler/rule-extractor.ts
|
|
8474
|
-
import { createHash as
|
|
9603
|
+
import { createHash as createHash9 } from "crypto";
|
|
8475
9604
|
var PROVENANCE_SOURCE = "llm-wiki-compiler";
|
|
8476
9605
|
function isUrl2(value) {
|
|
8477
9606
|
return typeof value === "string" && /^https?:\/\//i.test(value);
|
|
@@ -8528,9 +9657,9 @@ ${rule.description}`;
|
|
|
8528
9657
|
);
|
|
8529
9658
|
}
|
|
8530
9659
|
async function extractForSource2(root, sourceFile, provenance, createdAt) {
|
|
8531
|
-
const sourcePath =
|
|
8532
|
-
const raw = await
|
|
8533
|
-
const hash =
|
|
9660
|
+
const sourcePath = path64.join(root, SOURCES_DIR, sourceFile);
|
|
9661
|
+
const raw = await readFile32(sourcePath, "utf-8");
|
|
9662
|
+
const hash = createHash9("sha256").update(raw).digest("hex");
|
|
8534
9663
|
const { meta } = parseFrontmatter(raw);
|
|
8535
9664
|
const numbered = budgetAndNumberSource(sourceFile, raw);
|
|
8536
9665
|
const maxLine = numbered.split("\n").length;
|
|
@@ -8600,7 +9729,7 @@ function buildRuleCandidatesJson(candidates) {
|
|
|
8600
9729
|
var RULE_EXPORT_PATH = "dist/exports/rule-candidates.json";
|
|
8601
9730
|
async function rulesExtractCommand() {
|
|
8602
9731
|
const root = process.cwd();
|
|
8603
|
-
if (!
|
|
9732
|
+
if (!existsSync20(path65.join(root, SOURCES_DIR))) {
|
|
8604
9733
|
status("!", warn("No sources found. Run `llmwiki ingest <url>` first."));
|
|
8605
9734
|
return;
|
|
8606
9735
|
}
|
|
@@ -8663,7 +9792,7 @@ async function rulesExportCommand(options = {}) {
|
|
|
8663
9792
|
const root = process.cwd();
|
|
8664
9793
|
const scope = resolveScope(options.scope);
|
|
8665
9794
|
const candidates = await collectRuleCandidatesForExport(root, scope);
|
|
8666
|
-
const outPath =
|
|
9795
|
+
const outPath = path65.join(root, RULE_EXPORT_PATH);
|
|
8667
9796
|
await atomicWrite(outPath, buildRuleCandidatesJson(candidates));
|
|
8668
9797
|
status(
|
|
8669
9798
|
"+",
|
|
@@ -8748,8 +9877,8 @@ async function runRulesAction(work) {
|
|
|
8748
9877
|
}
|
|
8749
9878
|
|
|
8750
9879
|
// src/project/state.ts
|
|
8751
|
-
import { stat as
|
|
8752
|
-
import
|
|
9880
|
+
import { stat as stat3, readdir as readdir18, readFile as readFile33 } from "fs/promises";
|
|
9881
|
+
import path66 from "path";
|
|
8753
9882
|
var MARKDOWN_EXT = ".md";
|
|
8754
9883
|
async function collectProjectState(root) {
|
|
8755
9884
|
const rootReadable = await isDirectory(root);
|
|
@@ -8785,31 +9914,31 @@ function brokenProjectState(root) {
|
|
|
8785
9914
|
}
|
|
8786
9915
|
async function collectDirPresence(root) {
|
|
8787
9916
|
const [hasSourcesDir, hasWikiDir, hasInternalDir] = await Promise.all([
|
|
8788
|
-
isDirectory(
|
|
8789
|
-
isDirectory(
|
|
8790
|
-
isDirectory(
|
|
9917
|
+
isDirectory(path66.join(root, SOURCES_DIR)),
|
|
9918
|
+
isDirectory(path66.join(root, "wiki")),
|
|
9919
|
+
isDirectory(path66.join(root, LLMWIKI_DIR))
|
|
8791
9920
|
]);
|
|
8792
9921
|
return { hasSourcesDir, hasWikiDir, hasInternalDir };
|
|
8793
9922
|
}
|
|
8794
9923
|
async function collectPageCounts(root, dirs) {
|
|
8795
9924
|
const [sourceCount, conceptCount, queryCount, pendingCandidates, hasIndex] = await Promise.all([
|
|
8796
|
-
dirs.hasSourcesDir ? countMarkdownFiles(
|
|
8797
|
-
dirs.hasWikiDir ? countMarkdownFiles(
|
|
8798
|
-
dirs.hasWikiDir ? countMarkdownFiles(
|
|
9925
|
+
dirs.hasSourcesDir ? countMarkdownFiles(path66.join(root, SOURCES_DIR)) : 0,
|
|
9926
|
+
dirs.hasWikiDir ? countMarkdownFiles(path66.join(root, CONCEPTS_DIR)) : 0,
|
|
9927
|
+
dirs.hasWikiDir ? countMarkdownFiles(path66.join(root, QUERIES_DIR)) : 0,
|
|
8799
9928
|
dirs.hasInternalDir ? safeCountCandidates(root) : 0,
|
|
8800
|
-
dirs.hasWikiDir ? isFile(
|
|
9929
|
+
dirs.hasWikiDir ? isFile(path66.join(root, INDEX_FILE)) : false
|
|
8801
9930
|
]);
|
|
8802
9931
|
return { sourceCount, conceptCount, queryCount, pendingCandidates, hasIndex };
|
|
8803
9932
|
}
|
|
8804
9933
|
async function collectMtimes(root, dirs) {
|
|
8805
9934
|
const [latestWikiMtimeMs, latestSourceMtimeMs] = await Promise.all([
|
|
8806
|
-
dirs.hasWikiDir ? safeMtime(
|
|
8807
|
-
dirs.hasSourcesDir ? safeMtime(
|
|
9935
|
+
dirs.hasWikiDir ? safeMtime(path66.join(root, "wiki")) : Promise.resolve(null),
|
|
9936
|
+
dirs.hasSourcesDir ? safeMtime(path66.join(root, SOURCES_DIR)) : Promise.resolve(null)
|
|
8808
9937
|
]);
|
|
8809
9938
|
return { latestWikiMtimeMs, latestSourceMtimeMs };
|
|
8810
9939
|
}
|
|
8811
9940
|
async function collectLintCacheStatus(root) {
|
|
8812
|
-
const cachePath =
|
|
9941
|
+
const cachePath = path66.join(root, LAST_LINT_FILE);
|
|
8813
9942
|
const exists = await isFile(cachePath);
|
|
8814
9943
|
if (!exists) return { present: false, entry: null };
|
|
8815
9944
|
const entry = await readLintCacheEntry(cachePath);
|
|
@@ -8818,7 +9947,7 @@ async function collectLintCacheStatus(root) {
|
|
|
8818
9947
|
async function readLintCacheEntry(cachePath) {
|
|
8819
9948
|
let raw;
|
|
8820
9949
|
try {
|
|
8821
|
-
raw = await
|
|
9950
|
+
raw = await readFile33(cachePath, "utf-8");
|
|
8822
9951
|
} catch {
|
|
8823
9952
|
return null;
|
|
8824
9953
|
}
|
|
@@ -8931,7 +10060,7 @@ function isLintCacheStale(entry, latestWikiMtimeMs) {
|
|
|
8931
10060
|
}
|
|
8932
10061
|
async function isDirectory(target) {
|
|
8933
10062
|
try {
|
|
8934
|
-
const stats = await
|
|
10063
|
+
const stats = await stat3(target);
|
|
8935
10064
|
return stats.isDirectory();
|
|
8936
10065
|
} catch {
|
|
8937
10066
|
return false;
|
|
@@ -8939,7 +10068,7 @@ async function isDirectory(target) {
|
|
|
8939
10068
|
}
|
|
8940
10069
|
async function isFile(target) {
|
|
8941
10070
|
try {
|
|
8942
|
-
const stats = await
|
|
10071
|
+
const stats = await stat3(target);
|
|
8943
10072
|
return stats.isFile();
|
|
8944
10073
|
} catch {
|
|
8945
10074
|
return false;
|
|
@@ -8947,7 +10076,7 @@ async function isFile(target) {
|
|
|
8947
10076
|
}
|
|
8948
10077
|
async function safeMtime(target) {
|
|
8949
10078
|
try {
|
|
8950
|
-
const stats = await
|
|
10079
|
+
const stats = await stat3(target);
|
|
8951
10080
|
return stats.mtimeMs;
|
|
8952
10081
|
} catch {
|
|
8953
10082
|
return null;
|
|
@@ -8955,7 +10084,7 @@ async function safeMtime(target) {
|
|
|
8955
10084
|
}
|
|
8956
10085
|
async function countMarkdownFiles(dir) {
|
|
8957
10086
|
try {
|
|
8958
|
-
const entries = await
|
|
10087
|
+
const entries = await readdir18(dir, { withFileTypes: true });
|
|
8959
10088
|
let count = 0;
|
|
8960
10089
|
for (const entry of entries) {
|
|
8961
10090
|
if (entry.isFile() && entry.name.endsWith(MARKDOWN_EXT)) count += 1;
|
|
@@ -9171,8 +10300,8 @@ function plural(count) {
|
|
|
9171
10300
|
}
|
|
9172
10301
|
|
|
9173
10302
|
// src/compiler/refresh-plan.ts
|
|
9174
|
-
import { readdir as
|
|
9175
|
-
import
|
|
10303
|
+
import { readdir as readdir19 } from "fs/promises";
|
|
10304
|
+
import path67 from "path";
|
|
9176
10305
|
function ownersOf2(slug, snapshot) {
|
|
9177
10306
|
return Object.entries(snapshot.sources).filter(([, s]) => s.concepts.includes(slug)).map(([file, s]) => ({
|
|
9178
10307
|
file,
|
|
@@ -9223,7 +10352,7 @@ async function resolveStaleRefresh(root) {
|
|
|
9223
10352
|
}
|
|
9224
10353
|
const snapshot = await buildFreshnessSnapshot(root, classified);
|
|
9225
10354
|
const state = classified.state;
|
|
9226
|
-
const pages = await scanWikiPages(
|
|
10355
|
+
const pages = await scanWikiPages(path67.join(root, CONCEPTS_DIR));
|
|
9227
10356
|
const c = classifyPages(pages, snapshot);
|
|
9228
10357
|
return {
|
|
9229
10358
|
stateStatus: "ok",
|
|
@@ -9250,7 +10379,7 @@ function knownAffectedFor(state, changed, deleted) {
|
|
|
9250
10379
|
async function listNewSkipped(root, state) {
|
|
9251
10380
|
let files;
|
|
9252
10381
|
try {
|
|
9253
|
-
files = (await
|
|
10382
|
+
files = (await readdir19(path67.join(root, SOURCES_DIR))).filter((f) => f.endsWith(".md"));
|
|
9254
10383
|
} catch {
|
|
9255
10384
|
return [];
|
|
9256
10385
|
}
|
|
@@ -9293,7 +10422,7 @@ async function warnOnReviewBypass(root) {
|
|
|
9293
10422
|
const pending = await countCandidates(root);
|
|
9294
10423
|
if (pending > 0) {
|
|
9295
10424
|
status("!", warn(
|
|
9296
|
-
|
|
10425
|
+
`${pending} pending review candidate(s) exist \u2014 refresh respects the project review policy; pages tripping policy are held for review, not written directly.`
|
|
9297
10426
|
));
|
|
9298
10427
|
}
|
|
9299
10428
|
}
|
|
@@ -9303,12 +10432,24 @@ function reportCompileOutcome(result, plan) {
|
|
|
9303
10432
|
reportNewSkipped(plan.newSkipped);
|
|
9304
10433
|
return 1;
|
|
9305
10434
|
}
|
|
9306
|
-
|
|
9307
|
-
|
|
9308
|
-
|
|
10435
|
+
const heldCount = (result.review?.held.length ?? 0) + (result.review?.forced.length ?? 0);
|
|
10436
|
+
const liveRefreshed = plan.recompiledPages.length - heldCount;
|
|
10437
|
+
const orphanedCount = plan.computedOrphanedPages.length;
|
|
10438
|
+
status("\u2713", success(buildRefreshSummary(liveRefreshed, heldCount, orphanedCount)));
|
|
9309
10439
|
reportNewSkipped(plan.newSkipped);
|
|
9310
10440
|
return 0;
|
|
9311
10441
|
}
|
|
10442
|
+
function buildRefreshSummary(liveRefreshed, heldCount, orphanedCount) {
|
|
10443
|
+
const parts = [];
|
|
10444
|
+
if (liveRefreshed > 0 || heldCount === 0) {
|
|
10445
|
+
parts.push(`Refreshed ${liveRefreshed} page(s)`);
|
|
10446
|
+
}
|
|
10447
|
+
if (heldCount > 0) {
|
|
10448
|
+
parts.push(`held ${heldCount} for review \u2014 run \`llmwiki review list\``);
|
|
10449
|
+
}
|
|
10450
|
+
parts.push(`cleaned up ${orphanedCount} orphaned page(s)`);
|
|
10451
|
+
return parts.join("; ");
|
|
10452
|
+
}
|
|
9312
10453
|
function maybeEnsureProvider(plan, ensureProvider) {
|
|
9313
10454
|
const needsLLM = plan.changedOwners.length > 0 || plan.knownAffected.length > 0;
|
|
9314
10455
|
if (needsLLM) ensureProvider?.();
|
|
@@ -9354,7 +10495,7 @@ function planDetailRows(plan) {
|
|
|
9354
10495
|
}
|
|
9355
10496
|
|
|
9356
10497
|
// src/commands/quickstart.ts
|
|
9357
|
-
import
|
|
10498
|
+
import path68 from "path";
|
|
9358
10499
|
var QUICKSTART_JSON_VERSION = 1;
|
|
9359
10500
|
var VIEW_OPEN_ARGS_KEY = "view\0--open";
|
|
9360
10501
|
var NOOP_RESTORE = () => {
|
|
@@ -9421,7 +10562,7 @@ async function runIngestStep(source2) {
|
|
|
9421
10562
|
status("*", info(`Ingesting ${source2}`));
|
|
9422
10563
|
try {
|
|
9423
10564
|
const result = await ingestSource(process.cwd(), source2);
|
|
9424
|
-
const relPath =
|
|
10565
|
+
const relPath = path68.join(SOURCES_DIR, result.filename);
|
|
9425
10566
|
status("+", success(`Ingested \u2192 ${relPath}`));
|
|
9426
10567
|
return buildIngestSuccess(result, relPath);
|
|
9427
10568
|
} catch (err) {
|
|
@@ -9670,7 +10811,7 @@ function appendNextLines(lines, next) {
|
|
|
9670
10811
|
}
|
|
9671
10812
|
|
|
9672
10813
|
// src/commands/context.ts
|
|
9673
|
-
import
|
|
10814
|
+
import path69 from "path";
|
|
9674
10815
|
|
|
9675
10816
|
// src/context/ranking.ts
|
|
9676
10817
|
var WEIGHT_TITLE_MATCH = 0.5;
|
|
@@ -10494,7 +11635,7 @@ function appendPrimaryPages(lines, primary) {
|
|
|
10494
11635
|
for (const page of primary) appendPrimaryPage(lines, page);
|
|
10495
11636
|
}
|
|
10496
11637
|
function appendPrimaryPage(lines, page) {
|
|
10497
|
-
const pageFile =
|
|
11638
|
+
const pageFile = path69.join("wiki", page.pageDirectory, `${slugFromId(page.id)}.md`);
|
|
10498
11639
|
lines.push(`### ${page.title} (\`${pageFile}\`)`);
|
|
10499
11640
|
lines.push("");
|
|
10500
11641
|
lines.push(`Why included: ${page.reasons.join(", ") || "(no signals)"}`);
|
|
@@ -10555,8 +11696,8 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
|
|
|
10555
11696
|
import { z as z2 } from "zod";
|
|
10556
11697
|
|
|
10557
11698
|
// src/status/collect.ts
|
|
10558
|
-
import
|
|
10559
|
-
import { readdir as
|
|
11699
|
+
import path70 from "path";
|
|
11700
|
+
import { readdir as readdir20 } from "fs/promises";
|
|
10560
11701
|
var MAX_STATUS_LIST = 100;
|
|
10561
11702
|
function classifyConceptPages(scanned, snapshot) {
|
|
10562
11703
|
const stalePages = [];
|
|
@@ -10577,7 +11718,7 @@ function lastCompileTime(sources) {
|
|
|
10577
11718
|
}
|
|
10578
11719
|
async function listSourceFilesOnDisk(root) {
|
|
10579
11720
|
try {
|
|
10580
|
-
const entries = await
|
|
11721
|
+
const entries = await readdir20(path70.join(root, SOURCES_DIR));
|
|
10581
11722
|
return entries.filter((f) => f.endsWith(".md"));
|
|
10582
11723
|
} catch {
|
|
10583
11724
|
return [];
|
|
@@ -10605,9 +11746,9 @@ async function collectStatus(root) {
|
|
|
10605
11746
|
const classified = await readStateClassified(root);
|
|
10606
11747
|
const snapshot = await buildFreshnessSnapshot(root, classified);
|
|
10607
11748
|
const [conceptSummaries, queries, scannedConcepts, pendingCandidates, sourceFilesOnDisk] = await Promise.all([
|
|
10608
|
-
collectPageSummaries(
|
|
10609
|
-
collectPageSummaries(
|
|
10610
|
-
scanWikiPages(
|
|
11749
|
+
collectPageSummaries(path70.join(root, CONCEPTS_DIR)),
|
|
11750
|
+
collectPageSummaries(path70.join(root, QUERIES_DIR)),
|
|
11751
|
+
scanWikiPages(path70.join(root, CONCEPTS_DIR)),
|
|
10611
11752
|
countCandidates(root),
|
|
10612
11753
|
listSourceFilesOnDisk(root)
|
|
10613
11754
|
]);
|
|
@@ -10629,11 +11770,11 @@ async function collectStatus(root) {
|
|
|
10629
11770
|
}
|
|
10630
11771
|
|
|
10631
11772
|
// src/pages/read.ts
|
|
10632
|
-
import
|
|
11773
|
+
import path71 from "path";
|
|
10633
11774
|
var PAGE_DIRS2 = [CONCEPTS_DIR, QUERIES_DIR];
|
|
10634
11775
|
async function readPageRecord2(root, slug) {
|
|
10635
11776
|
for (const dir of PAGE_DIRS2) {
|
|
10636
|
-
const content = await safeReadFile(
|
|
11777
|
+
const content = await safeReadFile(path71.join(root, dir, `${slug}.md`));
|
|
10637
11778
|
if (!content) continue;
|
|
10638
11779
|
const { meta, body } = parseFrontmatter(content);
|
|
10639
11780
|
if (meta.orphaned) continue;
|
|
@@ -10648,7 +11789,7 @@ async function readPageRecord2(root, slug) {
|
|
|
10648
11789
|
}
|
|
10649
11790
|
|
|
10650
11791
|
// src/search/retrieval.ts
|
|
10651
|
-
import
|
|
11792
|
+
import path72 from "path";
|
|
10652
11793
|
function dedupePreservingOrder(slugs) {
|
|
10653
11794
|
const seen = /* @__PURE__ */ new Set();
|
|
10654
11795
|
const out = [];
|
|
@@ -10670,7 +11811,7 @@ async function pickSearchSlugs(root, question) {
|
|
|
10670
11811
|
if (candidates.length > 0) return candidates.map((c) => c.slug);
|
|
10671
11812
|
} catch {
|
|
10672
11813
|
}
|
|
10673
|
-
const indexContent = await safeReadFile(
|
|
11814
|
+
const indexContent = await safeReadFile(path72.join(root, INDEX_FILE));
|
|
10674
11815
|
const { pages } = await selectPages(question, indexContent);
|
|
10675
11816
|
return pages;
|
|
10676
11817
|
}
|
|
@@ -10683,13 +11824,21 @@ async function loadPageRecords(root, slugs) {
|
|
|
10683
11824
|
return records;
|
|
10684
11825
|
}
|
|
10685
11826
|
|
|
10686
|
-
// src/mcp/
|
|
11827
|
+
// src/mcp/result.ts
|
|
10687
11828
|
function jsonResult(payload) {
|
|
10688
11829
|
return {
|
|
10689
11830
|
content: [{ type: "text", text: JSON.stringify(payload, null, 2) }],
|
|
10690
11831
|
structuredContent: { result: payload }
|
|
10691
11832
|
};
|
|
10692
11833
|
}
|
|
11834
|
+
function errorResult(message) {
|
|
11835
|
+
return {
|
|
11836
|
+
content: [{ type: "text", text: message }],
|
|
11837
|
+
isError: true
|
|
11838
|
+
};
|
|
11839
|
+
}
|
|
11840
|
+
|
|
11841
|
+
// src/mcp/tools.ts
|
|
10693
11842
|
function registerWikiTools(server, root) {
|
|
10694
11843
|
registerIngestTool(server, root);
|
|
10695
11844
|
registerCompileTool(server, root);
|
|
@@ -10871,9 +12020,62 @@ function registerEvalTool(server, root) {
|
|
|
10871
12020
|
);
|
|
10872
12021
|
}
|
|
10873
12022
|
|
|
12023
|
+
// src/mcp/okf-tools.ts
|
|
12024
|
+
import { z as z3 } from "zod";
|
|
12025
|
+
var MAX_MCP_PENDING_CANDIDATES = 200;
|
|
12026
|
+
function registerOkfTools(server, root, maxPending = MAX_MCP_PENDING_CANDIDATES) {
|
|
12027
|
+
server.registerTool(
|
|
12028
|
+
"export_okf",
|
|
12029
|
+
{
|
|
12030
|
+
title: "Export OKF bundle",
|
|
12031
|
+
description: "Export the wiki as an Open Knowledge Format (OKF) v0.1 bundle. `out` defaults to dist/exports/okf and must stay inside the project.",
|
|
12032
|
+
inputSchema: { out: z3.string().optional().describe("Output dir for the bundle (must resolve inside the project root)") }
|
|
12033
|
+
},
|
|
12034
|
+
// withQuiet: the export core can print skipped-reference warnings via output.status;
|
|
12035
|
+
// on the MCP stdio transport those would land on the JSON-RPC stream and corrupt it.
|
|
12036
|
+
async ({ out }) => withQuiet(async () => {
|
|
12037
|
+
try {
|
|
12038
|
+
const dest = await confineUnderRoot(out ?? "dist/exports/okf", root, { mustExist: false });
|
|
12039
|
+
const report = await runOkfExport(root, { out: dest });
|
|
12040
|
+
return jsonResult({ outDir: report.outDir, fileCount: report.writtenPaths.length, files: report.writtenPaths.slice(0, 20), warnings: report.warnings });
|
|
12041
|
+
} catch (err) {
|
|
12042
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
12043
|
+
}
|
|
12044
|
+
})
|
|
12045
|
+
);
|
|
12046
|
+
server.registerTool(
|
|
12047
|
+
"import_okf",
|
|
12048
|
+
{
|
|
12049
|
+
title: "Import OKF bundle (staged for review)",
|
|
12050
|
+
description: "Import an OKF bundle from a path INSIDE the project as review candidates. Always staging-only \u2014 imported knowledge requires human review/approval before it enters the wiki. Use dryRun to preview.",
|
|
12051
|
+
inputSchema: {
|
|
12052
|
+
dir: z3.string().describe("Bundle directory (must resolve inside the project root)"),
|
|
12053
|
+
dryRun: z3.boolean().optional().describe("Preview only \u2014 stage nothing")
|
|
12054
|
+
}
|
|
12055
|
+
},
|
|
12056
|
+
// withQuiet: same stdio-stream guard as export_okf (the import core may emit warnings).
|
|
12057
|
+
async ({ dir, dryRun }) => withQuiet(async () => {
|
|
12058
|
+
try {
|
|
12059
|
+
const bundle = await confineUnderRoot(dir, root, { mustExist: true });
|
|
12060
|
+
const report = await runOkfImport(root, bundle, { trusted: false, dryRun, maxNewCandidates: maxPending });
|
|
12061
|
+
return jsonResult({
|
|
12062
|
+
mode: report.mode,
|
|
12063
|
+
imported: report.pages.length,
|
|
12064
|
+
pages: report.pages,
|
|
12065
|
+
skipped: report.skipped,
|
|
12066
|
+
warnings: report.warnings,
|
|
12067
|
+
nextAction: report.mode === "dry-run" ? "Preview only \u2014 re-run without dryRun to stage these as review candidates." : "Imported docs are STAGED as review candidates \u2014 approve them via `llmwiki review` (human gate) before they enter the wiki."
|
|
12068
|
+
});
|
|
12069
|
+
} catch (err) {
|
|
12070
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
12071
|
+
}
|
|
12072
|
+
})
|
|
12073
|
+
);
|
|
12074
|
+
}
|
|
12075
|
+
|
|
10874
12076
|
// src/mcp/resources.ts
|
|
10875
|
-
import
|
|
10876
|
-
import { readdir as
|
|
12077
|
+
import path73 from "path";
|
|
12078
|
+
import { readdir as readdir21 } from "fs/promises";
|
|
10877
12079
|
import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
10878
12080
|
function jsonContent(uri, payload) {
|
|
10879
12081
|
return {
|
|
@@ -10908,7 +12110,7 @@ function registerIndexResource(server, root) {
|
|
|
10908
12110
|
mimeType: "text/markdown"
|
|
10909
12111
|
},
|
|
10910
12112
|
async (uri) => {
|
|
10911
|
-
const content = await safeReadFile(
|
|
12113
|
+
const content = await safeReadFile(path73.join(root, INDEX_FILE));
|
|
10912
12114
|
return { contents: [markdownContent(uri, content)] };
|
|
10913
12115
|
}
|
|
10914
12116
|
);
|
|
@@ -10975,23 +12177,23 @@ function registerQueryResource(server, root) {
|
|
|
10975
12177
|
);
|
|
10976
12178
|
}
|
|
10977
12179
|
async function listSources(root) {
|
|
10978
|
-
const sourcesPath =
|
|
12180
|
+
const sourcesPath = path73.join(root, SOURCES_DIR);
|
|
10979
12181
|
let files;
|
|
10980
12182
|
try {
|
|
10981
|
-
files = await
|
|
12183
|
+
files = await readdir21(sourcesPath);
|
|
10982
12184
|
} catch {
|
|
10983
12185
|
return [];
|
|
10984
12186
|
}
|
|
10985
12187
|
const records = [];
|
|
10986
12188
|
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
10987
|
-
const content = await safeReadFile(
|
|
12189
|
+
const content = await safeReadFile(path73.join(sourcesPath, file));
|
|
10988
12190
|
const { meta } = parseFrontmatter(content);
|
|
10989
12191
|
records.push({ filename: file, ...meta });
|
|
10990
12192
|
}
|
|
10991
12193
|
return records;
|
|
10992
12194
|
}
|
|
10993
12195
|
async function loadPageWithMeta(root, dir, slug) {
|
|
10994
|
-
const filePath =
|
|
12196
|
+
const filePath = path73.join(root, dir, `${slug}.md`);
|
|
10995
12197
|
const content = await safeReadFile(filePath);
|
|
10996
12198
|
if (!content) {
|
|
10997
12199
|
throw new Error(`Page not found: ${dir}/${slug}.md`);
|
|
@@ -11030,10 +12232,10 @@ function registerEvalHistoryResource(server, root) {
|
|
|
11030
12232
|
);
|
|
11031
12233
|
}
|
|
11032
12234
|
async function listPagesUnder(root, dir, scheme) {
|
|
11033
|
-
const pagesPath =
|
|
12235
|
+
const pagesPath = path73.join(root, dir);
|
|
11034
12236
|
let files;
|
|
11035
12237
|
try {
|
|
11036
|
-
files = await
|
|
12238
|
+
files = await readdir21(pagesPath);
|
|
11037
12239
|
} catch {
|
|
11038
12240
|
return { resources: [] };
|
|
11039
12241
|
}
|
|
@@ -11051,6 +12253,7 @@ async function startMCPServer(options) {
|
|
|
11051
12253
|
instructions: "llmwiki is a knowledge compiler. Use ingest_source to add raw sources, compile_wiki to run the LLM pipeline, query_wiki for grounded answers, search_pages to retrieve relevant pages, and run_eval to score wiki quality. read_page, lint_wiki, wiki_status, and run_eval (fast suite, record: false) work without an API key and do not mutate state."
|
|
11052
12254
|
});
|
|
11053
12255
|
registerWikiTools(server, root);
|
|
12256
|
+
registerOkfTools(server, root);
|
|
11054
12257
|
registerWikiResources(server, root);
|
|
11055
12258
|
const transport = new StdioServerTransport();
|
|
11056
12259
|
await server.connect(transport);
|
|
@@ -11248,7 +12451,7 @@ program.command("export").description("Export wiki content to portable formats (
|
|
|
11248
12451
|
).option(
|
|
11249
12452
|
"--project-id <id>",
|
|
11250
12453
|
"Bridge identifier embedded in the JSON export envelope. Must match /^[a-z0-9][a-z0-9-]{0,62}$/."
|
|
11251
|
-
).action(async (options) => {
|
|
12454
|
+
).option("--out <dir>", "Output directory for directory-style targets (e.g. okf)").action(async (options) => {
|
|
11252
12455
|
try {
|
|
11253
12456
|
await exportCommand(process.cwd(), options);
|
|
11254
12457
|
} catch (err) {
|
|
@@ -11256,6 +12459,17 @@ program.command("export").description("Export wiki content to portable formats (
|
|
|
11256
12459
|
process.exit(1);
|
|
11257
12460
|
}
|
|
11258
12461
|
});
|
|
12462
|
+
program.command("import").description("Import an OKF bundle as review candidates (default) or live pages (--trusted)").requiredOption("--okf <dir>", "Path to the OKF bundle directory to import").option(
|
|
12463
|
+
"--trusted",
|
|
12464
|
+
"Write mapped pages directly into wiki/ instead of staging for review (you vouch for the bundle's contents and its self-declared provenance)"
|
|
12465
|
+
).option("--dry-run", "Report what would be imported (and skipped) without writing anything").action(async (options) => {
|
|
12466
|
+
try {
|
|
12467
|
+
await importCommand(process.cwd(), options);
|
|
12468
|
+
} catch (err) {
|
|
12469
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
12470
|
+
process.exit(1);
|
|
12471
|
+
}
|
|
12472
|
+
});
|
|
11259
12473
|
program.command("next").description("Show the recommended next action for this llmwiki project (read-only)").option("--json", "Emit a stable JSON envelope for agent consumption").action(
|
|
11260
12474
|
async (options) => runExitCodeCommand(() => nextCommand({ json: options.json }))
|
|
11261
12475
|
);
|