llm-wiki-compiler 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +15 -20
- package/dist/cli.js +413 -182
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +48 -1
- package/dist/index.js +743 -66
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -219,6 +219,7 @@ var OLLAMA_DEFAULT_HOST = "http://localhost:11434/v1";
|
|
|
219
219
|
var COPILOT_BASE_URL = "https://api.githubcopilot.com";
|
|
220
220
|
var OPENAI_DEFAULT_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
221
221
|
var OLLAMA_DEFAULT_TIMEOUT_MS = 30 * 60 * 1e3;
|
|
222
|
+
var EXPORT_DIR = "dist/exports";
|
|
222
223
|
var SOURCES_DIR = "sources";
|
|
223
224
|
var CONCEPTS_DIR = "wiki/concepts";
|
|
224
225
|
var QUERIES_DIR = "wiki/queries";
|
|
@@ -281,6 +282,25 @@ async function confinedRegularFile(dir, name) {
|
|
|
281
282
|
if (real === null || !isInsideDir(real, dir)) return null;
|
|
282
283
|
return real;
|
|
283
284
|
}
|
|
285
|
+
async function confineUnderRoot(target, root, opts) {
|
|
286
|
+
const realRoot = await safeRealpath(root) ?? path2.resolve(root);
|
|
287
|
+
const abs = path2.normalize(path2.resolve(realRoot, target));
|
|
288
|
+
if (!isInsideDir(abs, realRoot)) throw new Error(`path escapes project root: ${target}`);
|
|
289
|
+
if (opts.mustExist) {
|
|
290
|
+
const real = await safeRealpath(abs);
|
|
291
|
+
if (real === null || !isInsideDir(real, realRoot)) throw new Error(`path escapes project root: ${target}`);
|
|
292
|
+
return real;
|
|
293
|
+
}
|
|
294
|
+
for (let cur = abs; ; cur = path2.dirname(cur)) {
|
|
295
|
+
const real = await safeRealpath(cur);
|
|
296
|
+
if (real !== null) {
|
|
297
|
+
if (!isInsideDir(real, realRoot)) throw new Error(`path escapes project root: ${target}`);
|
|
298
|
+
break;
|
|
299
|
+
}
|
|
300
|
+
if (path2.dirname(cur) === cur) break;
|
|
301
|
+
}
|
|
302
|
+
return abs;
|
|
303
|
+
}
|
|
284
304
|
|
|
285
305
|
// src/viewer/path-safety.ts
|
|
286
306
|
import path3 from "path";
|
|
@@ -429,6 +449,9 @@ function source(text) {
|
|
|
429
449
|
}
|
|
430
450
|
var quietMode = false;
|
|
431
451
|
var quietScope = new AsyncLocalStorage();
|
|
452
|
+
function withQuiet(fn) {
|
|
453
|
+
return quietScope.run(true, fn);
|
|
454
|
+
}
|
|
432
455
|
function isQuiet() {
|
|
433
456
|
return quietScope.getStore() ?? quietMode;
|
|
434
457
|
}
|
|
@@ -7728,7 +7751,7 @@ async function evalJudgementsCommand(options = {}) {
|
|
|
7728
7751
|
}
|
|
7729
7752
|
|
|
7730
7753
|
// src/commands/export.ts
|
|
7731
|
-
import
|
|
7754
|
+
import path55 from "path";
|
|
7732
7755
|
import { createRequire } from "module";
|
|
7733
7756
|
|
|
7734
7757
|
// src/export/collect.ts
|
|
@@ -8198,9 +8221,12 @@ ${pageToSlide(p)}`);
|
|
|
8198
8221
|
return [frontmatter, titleSlide, ...slides, ""].join("\n\n");
|
|
8199
8222
|
}
|
|
8200
8223
|
|
|
8224
|
+
// src/export/okf/run.ts
|
|
8225
|
+
import path54 from "path";
|
|
8226
|
+
|
|
8201
8227
|
// src/export/okf/bundle.ts
|
|
8202
|
-
import { mkdir as mkdir9, copyFile as copyFile2, rm, readFile as readFile29 } from "fs/promises";
|
|
8203
|
-
import
|
|
8228
|
+
import { mkdir as mkdir9, copyFile as copyFile2, rm, readFile as readFile29, readdir as readdir16, lstat as lstat4 } from "fs/promises";
|
|
8229
|
+
import path53 from "path";
|
|
8204
8230
|
|
|
8205
8231
|
// src/export/okf/mapping.ts
|
|
8206
8232
|
import { createHash as createHash7 } from "crypto";
|
|
@@ -8268,7 +8294,7 @@ function mapPageToOkfFrontmatter(page) {
|
|
|
8268
8294
|
return fm;
|
|
8269
8295
|
}
|
|
8270
8296
|
var WIKILINK = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
|
|
8271
|
-
var OKF_LINK = /\[([^\]]+)\]\(\/(
|
|
8297
|
+
var OKF_LINK = /\[([^\]]+)\]\(\/([^)]+?)\.md\)/g;
|
|
8272
8298
|
var FENCE = /(```[\s\S]*?```|~~~[\s\S]*?~~~)/g;
|
|
8273
8299
|
function wikilinksToOkf(body, resolve) {
|
|
8274
8300
|
return body.split(FENCE).map(
|
|
@@ -8276,14 +8302,15 @@ function wikilinksToOkf(body, resolve) {
|
|
|
8276
8302
|
const slug = slugify(rawSlug);
|
|
8277
8303
|
const target = resolve(slug);
|
|
8278
8304
|
if (!target) return match;
|
|
8279
|
-
return `[${disp ?? target.title}](/${target.
|
|
8305
|
+
return `[${disp ?? target.title}](/${target.path})`;
|
|
8280
8306
|
})
|
|
8281
8307
|
).join("");
|
|
8282
8308
|
}
|
|
8283
|
-
function okfLinksToWikilinks(body,
|
|
8284
|
-
return body.replace(OKF_LINK, (
|
|
8285
|
-
const
|
|
8286
|
-
|
|
8309
|
+
function okfLinksToWikilinks(body, resolveLink) {
|
|
8310
|
+
return body.replace(OKF_LINK, (match, text, linkPath) => {
|
|
8311
|
+
const r = resolveLink(linkPath);
|
|
8312
|
+
if (!r) return match;
|
|
8313
|
+
return r.title === text ? `[[${r.slug}]]` : `[[${r.slug}|${text}]]`;
|
|
8287
8314
|
});
|
|
8288
8315
|
}
|
|
8289
8316
|
|
|
@@ -8315,8 +8342,8 @@ ${rewritten}${citations}`;
|
|
|
8315
8342
|
}
|
|
8316
8343
|
|
|
8317
8344
|
// src/export/okf/index-log.ts
|
|
8318
|
-
function buildOkfIndex(pages) {
|
|
8319
|
-
const entry = (p) => `* [${p.title}](/${
|
|
8345
|
+
function buildOkfIndex(pages, paths) {
|
|
8346
|
+
const entry = (p) => `* [${p.title}](/${paths.get(p)}) - ${p.summary}`;
|
|
8320
8347
|
const concepts = pages.filter((p) => p.pageDirectory === "concepts").map(entry);
|
|
8321
8348
|
const queries = pages.filter((p) => p.pageDirectory === "queries").map(entry);
|
|
8322
8349
|
const sections = [];
|
|
@@ -8379,71 +8406,178 @@ async function resolveReferences(root, pages) {
|
|
|
8379
8406
|
return resolved;
|
|
8380
8407
|
}
|
|
8381
8408
|
|
|
8409
|
+
// src/export/okf/output-paths.ts
|
|
8410
|
+
import path52 from "path";
|
|
8411
|
+
function slugPath(p) {
|
|
8412
|
+
return `${p.pageDirectory}/${p.slug}.md`;
|
|
8413
|
+
}
|
|
8414
|
+
function isSafeOkfPath(okfPath, realOut) {
|
|
8415
|
+
if (typeof okfPath !== "string" || okfPath.length === 0) return false;
|
|
8416
|
+
if (path52.isAbsolute(okfPath)) return false;
|
|
8417
|
+
if (!/^[A-Za-z0-9._/-]+$/.test(okfPath)) return false;
|
|
8418
|
+
const segments = okfPath.split(/[/\\]+/);
|
|
8419
|
+
if (segments.includes("..") || segments.includes(".")) return false;
|
|
8420
|
+
if (!okfPath.endsWith(".md")) return false;
|
|
8421
|
+
if (okfPath === "index.md" || okfPath === "log.md") return false;
|
|
8422
|
+
if (okfPath === "references" || okfPath.startsWith("references/")) return false;
|
|
8423
|
+
return isInsideDir(path52.normalize(path52.join(realOut, okfPath)), realOut);
|
|
8424
|
+
}
|
|
8425
|
+
function compareForeign(a, b) {
|
|
8426
|
+
const ap = a.xOkf.okfPath, bp = b.xOkf.okfPath;
|
|
8427
|
+
if (ap !== bp) return ap < bp ? -1 : 1;
|
|
8428
|
+
if (a.pageDirectory !== b.pageDirectory) return a.pageDirectory < b.pageDirectory ? -1 : 1;
|
|
8429
|
+
if (a.slug !== b.slug) return a.slug < b.slug ? -1 : 1;
|
|
8430
|
+
return 0;
|
|
8431
|
+
}
|
|
8432
|
+
function resolveOutputPaths(pages, realOut) {
|
|
8433
|
+
const owner = /* @__PURE__ */ new Map();
|
|
8434
|
+
for (const p of pages) owner.set(slugPath(p), p);
|
|
8435
|
+
const paths = /* @__PURE__ */ new Map();
|
|
8436
|
+
const warnings = [];
|
|
8437
|
+
const used = /* @__PURE__ */ new Set();
|
|
8438
|
+
const foreign = pages.filter((p) => typeof p.xOkf?.okfPath === "string").sort(compareForeign);
|
|
8439
|
+
const native = pages.filter((p) => typeof p.xOkf?.okfPath !== "string");
|
|
8440
|
+
for (const p of foreign) {
|
|
8441
|
+
const okfPath = p.xOkf.okfPath;
|
|
8442
|
+
const ownerOfOkf = owner.get(okfPath);
|
|
8443
|
+
const usable = isSafeOkfPath(okfPath, realOut) && !used.has(okfPath) && (ownerOfOkf === void 0 || ownerOfOkf === p);
|
|
8444
|
+
if (usable) {
|
|
8445
|
+
paths.set(p, okfPath);
|
|
8446
|
+
used.add(okfPath);
|
|
8447
|
+
} else {
|
|
8448
|
+
const sp = slugPath(p);
|
|
8449
|
+
paths.set(p, sp);
|
|
8450
|
+
used.add(sp);
|
|
8451
|
+
warnings.push(`OKF export: ${okfPath} not restorable (unsafe or collides); using ${sp}`);
|
|
8452
|
+
}
|
|
8453
|
+
}
|
|
8454
|
+
for (const p of native) {
|
|
8455
|
+
const sp = slugPath(p);
|
|
8456
|
+
paths.set(p, sp);
|
|
8457
|
+
used.add(sp);
|
|
8458
|
+
}
|
|
8459
|
+
return { paths, warnings };
|
|
8460
|
+
}
|
|
8461
|
+
|
|
8382
8462
|
// src/export/okf/bundle.ts
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
);
|
|
8463
|
+
var IGNORED_DIR_ENTRIES = /* @__PURE__ */ new Set([".DS_Store", "Thumbs.db"]);
|
|
8464
|
+
function makeResolver(pages, paths) {
|
|
8465
|
+
const map = new Map(pages.map((p) => [p.slug, { path: paths.get(p), title: p.title }]));
|
|
8387
8466
|
return (slug) => map.get(slug) ?? null;
|
|
8388
8467
|
}
|
|
8389
8468
|
async function writeConfined(out, rel, content) {
|
|
8390
|
-
const
|
|
8391
|
-
|
|
8392
|
-
|
|
8393
|
-
return normalized;
|
|
8469
|
+
const abs = await confineUnderRoot(rel, out, { mustExist: false });
|
|
8470
|
+
await atomicWrite(abs, content);
|
|
8471
|
+
return abs;
|
|
8394
8472
|
}
|
|
8395
|
-
async function
|
|
8396
|
-
|
|
8397
|
-
await
|
|
8473
|
+
async function lexists(p) {
|
|
8474
|
+
try {
|
|
8475
|
+
await lstat4(p);
|
|
8476
|
+
return true;
|
|
8477
|
+
} catch {
|
|
8478
|
+
return false;
|
|
8479
|
+
}
|
|
8480
|
+
}
|
|
8481
|
+
async function resolveRealTarget(p) {
|
|
8482
|
+
let cur = path53.resolve(p);
|
|
8483
|
+
const suffix = [];
|
|
8484
|
+
for (; ; ) {
|
|
8485
|
+
const real = await safeRealpath(cur);
|
|
8486
|
+
if (real) return suffix.length ? path53.join(real, ...suffix.reverse()) : real;
|
|
8487
|
+
const parent = path53.dirname(cur);
|
|
8488
|
+
if (parent === cur) return path53.resolve(p);
|
|
8489
|
+
suffix.push(path53.basename(cur));
|
|
8490
|
+
cur = parent;
|
|
8398
8491
|
}
|
|
8399
8492
|
}
|
|
8493
|
+
async function assertSafeBundleTarget(out, root) {
|
|
8494
|
+
const realRoot = await safeRealpath(root) ?? path53.resolve(root);
|
|
8495
|
+
const target = await resolveRealTarget(out);
|
|
8496
|
+
if (target === path53.parse(target).root) throw new Error("OKF export: refusing to export to the filesystem root");
|
|
8497
|
+
if (target === realRoot) throw new Error("OKF export: refusing to export to the project root");
|
|
8498
|
+
if (isInsideDir(target, path53.join(realRoot, ".git"))) throw new Error("OKF export: refusing to export inside .git");
|
|
8499
|
+
}
|
|
8500
|
+
async function assertNoTopLevelGit(realOut) {
|
|
8501
|
+
if (await lexists(path53.join(realOut, ".git"))) throw new Error("OKF export: refusing to export to a directory containing .git");
|
|
8502
|
+
}
|
|
8503
|
+
async function isRecognizedBundle(realOut) {
|
|
8504
|
+
const idx = path53.join(realOut, "index.md");
|
|
8505
|
+
try {
|
|
8506
|
+
if (!(await lstat4(idx)).isFile()) return false;
|
|
8507
|
+
} catch {
|
|
8508
|
+
return false;
|
|
8509
|
+
}
|
|
8510
|
+
const { meta } = parseFrontmatterStatus(await readFile29(idx, "utf-8"));
|
|
8511
|
+
return typeof meta.okf_version !== "undefined";
|
|
8512
|
+
}
|
|
8513
|
+
async function assertNoNestedGit(realOut) {
|
|
8514
|
+
for (const e of await readdir16(realOut, { withFileTypes: true })) {
|
|
8515
|
+
if (e.name === ".git") throw new Error(`OKF export: refusing to clear ${realOut} \u2014 it contains a nested .git`);
|
|
8516
|
+
if (e.isDirectory()) await assertNoNestedGit(path53.join(realOut, e.name));
|
|
8517
|
+
}
|
|
8518
|
+
}
|
|
8519
|
+
async function clearContents(realOut) {
|
|
8520
|
+
for (const name of await readdir16(realOut)) await rm(path53.join(realOut, name), { recursive: true, force: true });
|
|
8521
|
+
}
|
|
8400
8522
|
async function buildLog(root, pageCount) {
|
|
8401
8523
|
const fallback = [{ date: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10), action: "Export", text: `${pageCount} doc(s)` }];
|
|
8402
|
-
const llmwikiLog = await readFile29(
|
|
8524
|
+
const llmwikiLog = await readFile29(path53.join(root, "log.md"), "utf-8").catch(() => null);
|
|
8403
8525
|
const parsed = llmwikiLog ? parseLlmwikiLog(llmwikiLog) : [];
|
|
8404
8526
|
return buildOkfLog(parsed.length ? parsed : fallback);
|
|
8405
8527
|
}
|
|
8406
8528
|
async function copyResolvedReferences(refs, realOut) {
|
|
8407
8529
|
const written = [];
|
|
8408
8530
|
for (const { srcAbs, destName } of refs.values()) {
|
|
8409
|
-
const dest =
|
|
8410
|
-
|
|
8411
|
-
await mkdir9(path52.dirname(dest), { recursive: true });
|
|
8531
|
+
const dest = await confineUnderRoot(path53.join("references", destName), realOut, { mustExist: false });
|
|
8532
|
+
await mkdir9(path53.dirname(dest), { recursive: true });
|
|
8412
8533
|
await copyFile2(srcAbs, dest);
|
|
8413
8534
|
written.push(dest);
|
|
8414
8535
|
}
|
|
8415
8536
|
return written;
|
|
8416
8537
|
}
|
|
8417
|
-
function reportSkippedReferences(pages, refs) {
|
|
8538
|
+
function reportSkippedReferences(pages, refs, onWarn) {
|
|
8418
8539
|
const skipped = collectReferenceFiles(pages).filter((f) => !refs.has(f));
|
|
8419
8540
|
if (skipped.length === 0) return;
|
|
8420
|
-
|
|
8421
|
-
"!",
|
|
8422
|
-
warn(`OKF export: ${skipped.length} cited source(s) not bundled (missing or outside sources/)`)
|
|
8423
|
-
);
|
|
8541
|
+
onWarn(`OKF export: ${skipped.length} cited source(s) not bundled (missing or outside sources/)`);
|
|
8424
8542
|
}
|
|
8425
|
-
async function buildOkfBundle(root, pages, out) {
|
|
8543
|
+
async function buildOkfBundle(root, pages, out, onWarn = (m) => status("!", warn(m))) {
|
|
8544
|
+
await assertSafeBundleTarget(out, root);
|
|
8426
8545
|
await mkdir9(out, { recursive: true });
|
|
8427
|
-
const realOut = await safeRealpath(out) ??
|
|
8428
|
-
await
|
|
8429
|
-
const
|
|
8546
|
+
const realOut = await safeRealpath(out) ?? path53.normalize(out);
|
|
8547
|
+
await assertNoTopLevelGit(realOut);
|
|
8548
|
+
const empty = (await readdir16(realOut)).filter((e) => !IGNORED_DIR_ENTRIES.has(e)).length === 0;
|
|
8549
|
+
const bundle = empty ? false : await isRecognizedBundle(realOut);
|
|
8550
|
+
if (!empty && !bundle) throw new Error(`OKF export: ${out} is not empty and not an OKF bundle; export into a fresh directory or an existing OKF bundle.`);
|
|
8551
|
+
if (bundle) {
|
|
8552
|
+
await assertNoNestedGit(realOut);
|
|
8553
|
+
await clearContents(realOut);
|
|
8554
|
+
}
|
|
8555
|
+
const { paths, warnings: pathWarnings } = resolveOutputPaths(pages, realOut);
|
|
8556
|
+
const resolve = makeResolver(pages, paths);
|
|
8430
8557
|
const refs = await resolveReferences(root, pages);
|
|
8431
8558
|
const refName = (file) => refs.get(file)?.destName ?? null;
|
|
8432
8559
|
const written = [];
|
|
8433
|
-
written.push(await writeConfined(realOut, "index.md", buildOkfIndex(pages)));
|
|
8560
|
+
written.push(await writeConfined(realOut, "index.md", buildOkfIndex(pages, paths)));
|
|
8434
8561
|
for (const p of pages) {
|
|
8435
|
-
const docRel =
|
|
8436
|
-
const docAbs = path52.normalize(path52.join(realOut, docRel));
|
|
8437
|
-
const pageDir = path52.join(realOut, p.pageDirectory);
|
|
8438
|
-
if (!isInsideDir(docAbs, pageDir)) throw new Error(`OKF page slug escapes its directory: ${p.slug}`);
|
|
8562
|
+
const docRel = paths.get(p);
|
|
8439
8563
|
written.push(await writeConfined(realOut, docRel, renderOkfDoc(p, resolve, refName)));
|
|
8440
8564
|
}
|
|
8441
8565
|
written.push(...await copyResolvedReferences(refs, realOut));
|
|
8442
8566
|
written.push(await writeConfined(realOut, "log.md", await buildLog(root, pages.length)));
|
|
8443
|
-
reportSkippedReferences(pages, refs);
|
|
8567
|
+
reportSkippedReferences(pages, refs, onWarn);
|
|
8568
|
+
for (const w of pathWarnings) onWarn(w);
|
|
8444
8569
|
return written;
|
|
8445
8570
|
}
|
|
8446
8571
|
|
|
8572
|
+
// src/export/okf/run.ts
|
|
8573
|
+
async function runOkfExport(root, opts = {}) {
|
|
8574
|
+
const outDir = opts.out ?? path54.join(root, EXPORT_DIR, "okf");
|
|
8575
|
+
const pages = await collectExportPages(root);
|
|
8576
|
+
const warnings = [];
|
|
8577
|
+
const writtenPaths = await buildOkfBundle(root, pages, outDir, (m) => warnings.push(m));
|
|
8578
|
+
return { outDir, writtenPaths, warnings };
|
|
8579
|
+
}
|
|
8580
|
+
|
|
8447
8581
|
// src/export/types.ts
|
|
8448
8582
|
var MARP_SOURCES = ["concepts", "queries", "all"];
|
|
8449
8583
|
var EXPORT_TARGETS = [
|
|
@@ -8466,7 +8600,6 @@ var DEFAULT_EXPORT_TARGETS = [
|
|
|
8466
8600
|
|
|
8467
8601
|
// src/commands/export.ts
|
|
8468
8602
|
var require2 = createRequire(import.meta.url);
|
|
8469
|
-
var EXPORT_DIR = "dist/exports";
|
|
8470
8603
|
var TARGET_FILENAMES = {
|
|
8471
8604
|
"llms-txt": "llms.txt",
|
|
8472
8605
|
"llms-full-txt": "llms-full.txt",
|
|
@@ -8480,7 +8613,7 @@ var TARGET_FILENAMES = {
|
|
|
8480
8613
|
};
|
|
8481
8614
|
function resolveProjectTitle(root) {
|
|
8482
8615
|
try {
|
|
8483
|
-
const pkg = require2(
|
|
8616
|
+
const pkg = require2(path55.join(root, "package.json"));
|
|
8484
8617
|
return typeof pkg.name === "string" ? pkg.name : "Knowledge Wiki";
|
|
8485
8618
|
} catch {
|
|
8486
8619
|
return "Knowledge Wiki";
|
|
@@ -8536,14 +8669,14 @@ async function runExport(root, options = {}) {
|
|
|
8536
8669
|
const written = [];
|
|
8537
8670
|
for (const target of targets) {
|
|
8538
8671
|
if (target === "okf") {
|
|
8539
|
-
const outDir =
|
|
8540
|
-
const writtenPaths = await buildOkfBundle(root, pages, outDir);
|
|
8672
|
+
const { outDir, writtenPaths, warnings } = await runOkfExport(root, { out: options.out });
|
|
8541
8673
|
written.push(...writtenPaths);
|
|
8674
|
+
for (const w of warnings) status("!", warn(w));
|
|
8542
8675
|
status("+", success(`Exported okf bundle \u2192 ${source(outDir)}`));
|
|
8543
8676
|
continue;
|
|
8544
8677
|
}
|
|
8545
8678
|
const content = buildContent({ target, pages, projectTitle, marpSource, projectId });
|
|
8546
|
-
const outPath =
|
|
8679
|
+
const outPath = path55.join(root, EXPORT_DIR, TARGET_FILENAMES[target]);
|
|
8547
8680
|
await atomicWrite(outPath, content);
|
|
8548
8681
|
written.push(outPath);
|
|
8549
8682
|
status("+", success(`Exported ${target} \u2192 ${source(outPath)}`));
|
|
@@ -8568,15 +8701,15 @@ async function exportCommand(root, options) {
|
|
|
8568
8701
|
);
|
|
8569
8702
|
}
|
|
8570
8703
|
|
|
8571
|
-
// src/
|
|
8572
|
-
import
|
|
8704
|
+
// src/import/run.ts
|
|
8705
|
+
import path59 from "path";
|
|
8573
8706
|
|
|
8574
8707
|
// src/import/okf-import.ts
|
|
8575
|
-
import
|
|
8708
|
+
import path58 from "path";
|
|
8576
8709
|
|
|
8577
8710
|
// src/import/okf-read.ts
|
|
8578
|
-
import { readdir as
|
|
8579
|
-
import
|
|
8711
|
+
import { readdir as readdir17, readFile as readFile30, stat as stat2 } from "fs/promises";
|
|
8712
|
+
import path56 from "path";
|
|
8580
8713
|
|
|
8581
8714
|
// src/import/okf-limits.ts
|
|
8582
8715
|
var DEFAULT_OKF_LIMITS = {
|
|
@@ -8591,22 +8724,22 @@ var DEFAULT_OKF_LIMITS = {
|
|
|
8591
8724
|
// src/import/okf-read.ts
|
|
8592
8725
|
var RESERVED = /* @__PURE__ */ new Set(["index.md", "log.md"]);
|
|
8593
8726
|
async function listMarkdown(root, dir, limits, acc = [], visited = { n: 0 }) {
|
|
8594
|
-
for (const entry of await
|
|
8727
|
+
for (const entry of await readdir17(dir, { withFileTypes: true })) {
|
|
8595
8728
|
if (++visited.n > limits.maxEntries) throw new Error(`OKF import: bundle exceeds max entry count (> ${limits.maxEntries})`);
|
|
8596
|
-
const abs =
|
|
8729
|
+
const abs = path56.join(dir, entry.name);
|
|
8597
8730
|
if (entry.isDirectory()) await listMarkdown(root, abs, limits, acc, visited);
|
|
8598
8731
|
else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
8599
|
-
acc.push(
|
|
8732
|
+
acc.push(path56.relative(root, abs).split(path56.sep).join("/"));
|
|
8600
8733
|
if (acc.length > limits.maxFiles) throw new Error(`OKF import: bundle exceeds max file count (> ${limits.maxFiles})`);
|
|
8601
8734
|
}
|
|
8602
8735
|
}
|
|
8603
8736
|
return acc;
|
|
8604
8737
|
}
|
|
8605
8738
|
async function confinedInside(realRoot, rel) {
|
|
8606
|
-
const real = await safeRealpath(
|
|
8739
|
+
const real = await safeRealpath(path56.join(realRoot, rel));
|
|
8607
8740
|
return real && isInsideDir(real, realRoot) ? real : null;
|
|
8608
8741
|
}
|
|
8609
|
-
async function readOkfBundle(bundleDir, overrides = {}) {
|
|
8742
|
+
async function readOkfBundle(bundleDir, overrides = {}, onWarn = (m) => status("!", warn(m))) {
|
|
8610
8743
|
const limits = { ...DEFAULT_OKF_LIMITS, ...overrides };
|
|
8611
8744
|
const realRoot = await safeRealpath(bundleDir);
|
|
8612
8745
|
if (!realRoot) throw new Error(`OKF import: bundle not found: ${bundleDir}`);
|
|
@@ -8617,7 +8750,7 @@ async function readOkfBundle(bundleDir, overrides = {}) {
|
|
|
8617
8750
|
if (RESERVED.has(rel)) continue;
|
|
8618
8751
|
const real = await confinedInside(realRoot, rel);
|
|
8619
8752
|
if (!real) {
|
|
8620
|
-
|
|
8753
|
+
onWarn(`OKF import: skipped path escaping bundle: ${rel}`);
|
|
8621
8754
|
continue;
|
|
8622
8755
|
}
|
|
8623
8756
|
const size = (await stat2(real)).size;
|
|
@@ -8626,7 +8759,7 @@ async function readOkfBundle(bundleDir, overrides = {}) {
|
|
|
8626
8759
|
const parsed = parseFrontmatterStatus(await readFile30(real, "utf-8"));
|
|
8627
8760
|
const type = parsed.meta.type;
|
|
8628
8761
|
if (parsed.malformedFrontmatter || typeof type !== "string" || type.trim() === "") {
|
|
8629
|
-
|
|
8762
|
+
onWarn(`OKF import: skipped doc without a valid type: ${rel}`);
|
|
8630
8763
|
continue;
|
|
8631
8764
|
}
|
|
8632
8765
|
docs.push({ relPath: rel, meta: parsed.meta, body: parsed.body });
|
|
@@ -8696,7 +8829,12 @@ function okfDocToPage(doc, ctx) {
|
|
|
8696
8829
|
const slug = slugFromRelPath(doc.relPath);
|
|
8697
8830
|
const fields = buildPageFields(doc, ctx, slug);
|
|
8698
8831
|
const isNative = doc.meta["x-llmwiki"] !== void 0;
|
|
8699
|
-
const
|
|
8832
|
+
const resolveLink = (linkPath) => {
|
|
8833
|
+
const linkSlug = slugFromRelPath(linkPath);
|
|
8834
|
+
const title = ctx.titleOf(linkSlug);
|
|
8835
|
+
return title !== null ? { slug: linkSlug, title } : null;
|
|
8836
|
+
};
|
|
8837
|
+
const body = isNative ? okfLinksToWikilinks(canonicalBody(doc.body), resolveLink) : doc.body;
|
|
8700
8838
|
const pageBody = `${buildFrontmatter(fields)}
|
|
8701
8839
|
${body}`;
|
|
8702
8840
|
return {
|
|
@@ -8712,7 +8850,7 @@ ${body}`;
|
|
|
8712
8850
|
|
|
8713
8851
|
// src/import/okf-collision.ts
|
|
8714
8852
|
import { access } from "fs/promises";
|
|
8715
|
-
import
|
|
8853
|
+
import path57 from "path";
|
|
8716
8854
|
async function fileExists(p) {
|
|
8717
8855
|
try {
|
|
8718
8856
|
await access(p);
|
|
@@ -8730,9 +8868,8 @@ async function filterCollisions(root, pages) {
|
|
|
8730
8868
|
let reason = null;
|
|
8731
8869
|
if (claimed.has(page.slug)) reason = "duplicate-in-bundle";
|
|
8732
8870
|
else if (pendingSlugs.has(page.slug)) reason = "pending-candidate";
|
|
8733
|
-
else if (await fileExists(
|
|
8871
|
+
else if (await fileExists(path57.join(root, CONCEPTS_DIR, `${page.slug}.md`)) || await fileExists(path57.join(root, QUERIES_DIR, `${page.slug}.md`))) reason = "live-page";
|
|
8734
8872
|
if (reason) {
|
|
8735
|
-
status("!", warn(`OKF import: skipped ${page.okfPath} (slug "${page.slug}" collides: ${reason})`));
|
|
8736
8873
|
skipped.push({ slug: page.slug, okfPath: page.okfPath, reason });
|
|
8737
8874
|
} else {
|
|
8738
8875
|
claimed.add(page.slug);
|
|
@@ -8744,7 +8881,7 @@ async function filterCollisions(root, pages) {
|
|
|
8744
8881
|
|
|
8745
8882
|
// src/import/okf-import.ts
|
|
8746
8883
|
function bundleIdFor(bundleDir) {
|
|
8747
|
-
return slugify(
|
|
8884
|
+
return slugify(path58.basename(path58.resolve(bundleDir))) || "bundle";
|
|
8748
8885
|
}
|
|
8749
8886
|
function titleResolver(docs) {
|
|
8750
8887
|
const map = /* @__PURE__ */ new Map();
|
|
@@ -8754,14 +8891,19 @@ function titleResolver(docs) {
|
|
|
8754
8891
|
}
|
|
8755
8892
|
return (slug) => map.get(slug) ?? null;
|
|
8756
8893
|
}
|
|
8757
|
-
async function importOkfBundle(bundleDir, root, overrides = {}) {
|
|
8758
|
-
const docs = await readOkfBundle(bundleDir, overrides);
|
|
8894
|
+
async function importOkfBundle(bundleDir, root, overrides = {}, onWarn) {
|
|
8895
|
+
const docs = await readOkfBundle(bundleDir, overrides, onWarn);
|
|
8759
8896
|
const ctx = { bundleId: bundleIdFor(bundleDir), titleOf: titleResolver(docs) };
|
|
8760
8897
|
const mapped = docs.map((d) => okfDocToPage(d, ctx));
|
|
8761
8898
|
const { kept, skipped } = await filterCollisions(root, mapped);
|
|
8762
8899
|
return { pages: kept, skipped };
|
|
8763
8900
|
}
|
|
8764
8901
|
|
|
8902
|
+
// src/commands/import-core.ts
|
|
8903
|
+
function isWritable(page) {
|
|
8904
|
+
return page.slug.length > 0 && validateWikiPage(page.body);
|
|
8905
|
+
}
|
|
8906
|
+
|
|
8765
8907
|
// src/import/okf-refresh.ts
|
|
8766
8908
|
async function safelyUpdateEmbeddings2(root, slugs) {
|
|
8767
8909
|
try {
|
|
@@ -8777,88 +8919,115 @@ async function refreshAfterImport(root, slugs) {
|
|
|
8777
8919
|
await safelyUpdateEmbeddings2(root, slugs);
|
|
8778
8920
|
}
|
|
8779
8921
|
|
|
8780
|
-
// src/
|
|
8781
|
-
|
|
8782
|
-
|
|
8783
|
-
|
|
8784
|
-
|
|
8785
|
-
|
|
8786
|
-
|
|
8787
|
-
|
|
8788
|
-
|
|
8789
|
-
|
|
8790
|
-
|
|
8791
|
-
|
|
8792
|
-
|
|
8793
|
-
|
|
8794
|
-
|
|
8795
|
-
|
|
8796
|
-
|
|
8797
|
-
|
|
8798
|
-
|
|
8799
|
-
|
|
8800
|
-
|
|
8922
|
+
// src/import/run-errors.ts
|
|
8923
|
+
var LockUnavailableError = class extends Error {
|
|
8924
|
+
constructor(message = "Could not acquire lock. Try again later.") {
|
|
8925
|
+
super(message);
|
|
8926
|
+
this.name = "LockUnavailableError";
|
|
8927
|
+
}
|
|
8928
|
+
};
|
|
8929
|
+
var QueueFullError = class extends Error {
|
|
8930
|
+
constructor(message = "Review queue would exceed the cap; approve/reject pending candidates first.") {
|
|
8931
|
+
super(message);
|
|
8932
|
+
this.name = "QueueFullError";
|
|
8933
|
+
}
|
|
8934
|
+
};
|
|
8935
|
+
|
|
8936
|
+
// src/import/run.ts
|
|
8937
|
+
var toImported = (p) => ({ slug: p.slug, okfPath: p.okfPath, targetDirectory: p.targetDirectory });
|
|
8938
|
+
function partition(pages, collisionSkips) {
|
|
8939
|
+
const valid = [];
|
|
8940
|
+
const invalid = [];
|
|
8941
|
+
for (const p of pages) {
|
|
8942
|
+
if (isWritable(p)) valid.push(p);
|
|
8943
|
+
else invalid.push({ slug: p.slug, okfPath: p.okfPath, reason: "invalid-page" });
|
|
8944
|
+
}
|
|
8945
|
+
return { valid, skipped: [...collisionSkips, ...invalid] };
|
|
8946
|
+
}
|
|
8947
|
+
async function stageAll(root, valid) {
|
|
8948
|
+
for (const page of valid) {
|
|
8949
|
+
await writeCandidate(root, {
|
|
8950
|
+
title: page.title,
|
|
8951
|
+
slug: page.slug,
|
|
8952
|
+
summary: page.summary,
|
|
8953
|
+
sources: page.sources,
|
|
8954
|
+
body: page.body,
|
|
8955
|
+
reviewMode: "imported",
|
|
8956
|
+
heldReasons: [{ code: "imported-okf" }],
|
|
8957
|
+
targetDirectory: page.targetDirectory,
|
|
8958
|
+
okfPath: page.okfPath
|
|
8959
|
+
});
|
|
8960
|
+
}
|
|
8801
8961
|
}
|
|
8802
|
-
async function
|
|
8962
|
+
async function writeAll(root, valid) {
|
|
8803
8963
|
const written = [];
|
|
8804
8964
|
try {
|
|
8805
|
-
for (const page of
|
|
8965
|
+
for (const page of valid) {
|
|
8806
8966
|
const dir = page.targetDirectory === "queries" ? QUERIES_DIR : CONCEPTS_DIR;
|
|
8807
|
-
await atomicWrite(
|
|
8967
|
+
await atomicWrite(path59.join(root, dir, `${page.slug}.md`), page.body);
|
|
8808
8968
|
written.push(page.slug);
|
|
8809
8969
|
}
|
|
8810
8970
|
} finally {
|
|
8811
8971
|
if (written.length) await refreshAfterImport(root, written);
|
|
8812
8972
|
}
|
|
8813
8973
|
}
|
|
8814
|
-
function
|
|
8815
|
-
|
|
8816
|
-
|
|
8817
|
-
|
|
8818
|
-
}
|
|
8819
|
-
|
|
8820
|
-
|
|
8974
|
+
async function runOkfImport(root, dir, opts = {}) {
|
|
8975
|
+
const warnings = [];
|
|
8976
|
+
if (opts.dryRun) {
|
|
8977
|
+
const { pages, skipped } = await importOkfBundle(dir, root, {}, (m) => warnings.push(m));
|
|
8978
|
+
const { valid, skipped: allSkipped } = partition(pages, skipped);
|
|
8979
|
+
return { mode: "dry-run", pages: valid.map(toImported), skipped: allSkipped, warnings, nextAction: "Re-run without dryRun to apply." };
|
|
8980
|
+
}
|
|
8981
|
+
const locked = await withQuiet(() => acquireLock(root));
|
|
8982
|
+
if (!locked) throw new LockUnavailableError();
|
|
8983
|
+
try {
|
|
8984
|
+
const { pages, skipped } = await importOkfBundle(dir, root, {}, (m) => warnings.push(m));
|
|
8985
|
+
const { valid, skipped: allSkipped } = partition(pages, skipped);
|
|
8986
|
+
if (opts.maxNewCandidates !== void 0 && !opts.trusted) {
|
|
8987
|
+
const pending = (await listCandidates(root)).length;
|
|
8988
|
+
if (pending + valid.length > opts.maxNewCandidates) throw new QueueFullError();
|
|
8989
|
+
}
|
|
8990
|
+
if (opts.trusted) await writeAll(root, valid);
|
|
8991
|
+
else await stageAll(root, valid);
|
|
8992
|
+
return {
|
|
8993
|
+
mode: opts.trusted ? "written" : "staged",
|
|
8994
|
+
pages: valid.map(toImported),
|
|
8995
|
+
skipped: allSkipped,
|
|
8996
|
+
warnings,
|
|
8997
|
+
nextAction: opts.trusted ? `${valid.length} page(s) written live.` : "Review and approve before it goes live: llmwiki review list."
|
|
8998
|
+
};
|
|
8999
|
+
} finally {
|
|
9000
|
+
await releaseLock(root);
|
|
8821
9001
|
}
|
|
8822
|
-
for (const s of skipped) status("!", warn(`skip ${s.okfPath} \u2014 ${s.reason}`));
|
|
8823
9002
|
}
|
|
8824
|
-
|
|
8825
|
-
|
|
8826
|
-
|
|
8827
|
-
const
|
|
8828
|
-
|
|
8829
|
-
|
|
9003
|
+
|
|
9004
|
+
// src/commands/import.ts
|
|
9005
|
+
function printReport(report) {
|
|
9006
|
+
for (const w of report.warnings) status("!", warn(w));
|
|
9007
|
+
for (const p of report.pages) status("+", `${p.slug} (${p.targetDirectory}) \u2190 ${p.okfPath}`);
|
|
9008
|
+
for (const s of report.skipped) status("!", warn(`skip ${s.okfPath} \u2014 ${s.reason}`));
|
|
9009
|
+
const verb = report.mode === "written" ? "wrote" : report.mode === "dry-run" ? "would import" : "staged";
|
|
9010
|
+
status("+", success(`OKF import: ${verb} ${report.pages.length} page(s); skipped ${report.skipped.length}.`));
|
|
8830
9011
|
}
|
|
8831
9012
|
async function importCommand(root, options) {
|
|
8832
9013
|
if (!options.okf) throw new Error("import: --okf <dir> is required");
|
|
8833
|
-
if (options.dryRun) {
|
|
8834
|
-
await previewImport(root, options.okf);
|
|
8835
|
-
return;
|
|
8836
|
-
}
|
|
8837
|
-
const locked = await acquireLock(root);
|
|
8838
|
-
if (!locked) {
|
|
8839
|
-
status("!", error("Could not acquire lock. Try again later."));
|
|
8840
|
-
process.exitCode = 1;
|
|
8841
|
-
return;
|
|
8842
|
-
}
|
|
8843
9014
|
try {
|
|
8844
|
-
const
|
|
8845
|
-
|
|
8846
|
-
|
|
8847
|
-
|
|
8848
|
-
|
|
8849
|
-
|
|
8850
|
-
|
|
8851
|
-
|
|
8852
|
-
|
|
8853
|
-
} finally {
|
|
8854
|
-
await releaseLock(root);
|
|
9015
|
+
const report = await runOkfImport(root, options.okf, { trusted: options.trusted, dryRun: options.dryRun });
|
|
9016
|
+
printReport(report);
|
|
9017
|
+
} catch (err) {
|
|
9018
|
+
if (err instanceof LockUnavailableError) {
|
|
9019
|
+
status("!", error(err.message));
|
|
9020
|
+
process.exitCode = 1;
|
|
9021
|
+
return;
|
|
9022
|
+
}
|
|
9023
|
+
throw err;
|
|
8855
9024
|
}
|
|
8856
9025
|
}
|
|
8857
9026
|
|
|
8858
9027
|
// src/commands/schema.ts
|
|
8859
9028
|
import { existsSync as existsSync18 } from "fs";
|
|
8860
9029
|
import { mkdir as mkdir10, writeFile as writeFile5 } from "fs/promises";
|
|
8861
|
-
import
|
|
9030
|
+
import path60 from "path";
|
|
8862
9031
|
async function schemaInitCommand() {
|
|
8863
9032
|
const root = process.cwd();
|
|
8864
9033
|
const defaults = buildDefaultSchema();
|
|
@@ -8867,7 +9036,7 @@ async function schemaInitCommand() {
|
|
|
8867
9036
|
status("!", warn(`Schema file already exists at ${targetPath}`));
|
|
8868
9037
|
return;
|
|
8869
9038
|
}
|
|
8870
|
-
await mkdir10(
|
|
9039
|
+
await mkdir10(path60.dirname(targetPath), { recursive: true });
|
|
8871
9040
|
const serializable = {
|
|
8872
9041
|
version: defaults.version,
|
|
8873
9042
|
defaultKind: defaults.defaultKind,
|
|
@@ -8967,7 +9136,7 @@ async function reviewShowCommand(id) {
|
|
|
8967
9136
|
}
|
|
8968
9137
|
|
|
8969
9138
|
// src/commands/review-approve.ts
|
|
8970
|
-
import
|
|
9139
|
+
import path61 from "path";
|
|
8971
9140
|
|
|
8972
9141
|
// src/commands/review-helpers.ts
|
|
8973
9142
|
async function runReviewUnderLock(id, underLock) {
|
|
@@ -9000,7 +9169,7 @@ async function approveUnderLock(root, id) {
|
|
|
9000
9169
|
return;
|
|
9001
9170
|
}
|
|
9002
9171
|
const dir = candidate.targetDirectory === "queries" ? QUERIES_DIR : CONCEPTS_DIR;
|
|
9003
|
-
const pagePath =
|
|
9172
|
+
const pagePath = path61.join(root, dir, `${candidate.slug}.md`);
|
|
9004
9173
|
await atomicWrite(pagePath, candidate.body);
|
|
9005
9174
|
status("+", success(`Approved \u2192 ${source(pagePath)}`));
|
|
9006
9175
|
await persistCandidateSourceStates(root, candidate);
|
|
@@ -9057,21 +9226,21 @@ async function rejectUnderLock(root, id) {
|
|
|
9057
9226
|
|
|
9058
9227
|
// src/commands/rules.ts
|
|
9059
9228
|
import { existsSync as existsSync20 } from "fs";
|
|
9060
|
-
import
|
|
9229
|
+
import path65 from "path";
|
|
9061
9230
|
|
|
9062
9231
|
// src/compiler/rule-extractor.ts
|
|
9063
9232
|
import { readFile as readFile32 } from "fs/promises";
|
|
9064
|
-
import
|
|
9233
|
+
import path64 from "path";
|
|
9065
9234
|
|
|
9066
9235
|
// src/compiler/rule-state.ts
|
|
9067
9236
|
import { readFile as readFile31, writeFile as writeFile6, rename as rename4, mkdir as mkdir11 } from "fs/promises";
|
|
9068
9237
|
import { existsSync as existsSync19 } from "fs";
|
|
9069
|
-
import
|
|
9238
|
+
import path62 from "path";
|
|
9070
9239
|
function emptyRuleState() {
|
|
9071
9240
|
return { version: 1, indexHash: "", sources: {} };
|
|
9072
9241
|
}
|
|
9073
9242
|
async function readRuleState(root) {
|
|
9074
|
-
const filePath =
|
|
9243
|
+
const filePath = path62.join(root, RULE_STATE_FILE);
|
|
9075
9244
|
if (!existsSync19(filePath)) return emptyRuleState();
|
|
9076
9245
|
try {
|
|
9077
9246
|
return JSON.parse(await readFile31(filePath, "utf-8"));
|
|
@@ -9080,8 +9249,8 @@ async function readRuleState(root) {
|
|
|
9080
9249
|
}
|
|
9081
9250
|
}
|
|
9082
9251
|
async function writeRuleState(root, state) {
|
|
9083
|
-
await mkdir11(
|
|
9084
|
-
const filePath =
|
|
9252
|
+
await mkdir11(path62.join(root, LLMWIKI_DIR), { recursive: true });
|
|
9253
|
+
const filePath = path62.join(root, RULE_STATE_FILE);
|
|
9085
9254
|
const tmpPath = `${filePath}.tmp`;
|
|
9086
9255
|
await writeFile6(tmpPath, JSON.stringify(state, null, 2), "utf-8");
|
|
9087
9256
|
await rename4(tmpPath, filePath);
|
|
@@ -9215,15 +9384,15 @@ function parseRules(toolOutput) {
|
|
|
9215
9384
|
}
|
|
9216
9385
|
|
|
9217
9386
|
// src/compiler/rule-candidates.ts
|
|
9218
|
-
import
|
|
9387
|
+
import path63 from "path";
|
|
9219
9388
|
import { createHash as createHash8 } from "crypto";
|
|
9220
9389
|
var CONFIDENCE_VALUES = ["low", "medium", "high"];
|
|
9221
9390
|
var STATUS_VALUES = ["proposed", "approved", "rejected"];
|
|
9222
9391
|
function ruleCandidatePath(root, id) {
|
|
9223
|
-
return
|
|
9392
|
+
return path63.join(root, RULE_CANDIDATES_DIR, `${id}${CANDIDATE_JSON_EXT}`);
|
|
9224
9393
|
}
|
|
9225
9394
|
function ruleArchivePath(root, id) {
|
|
9226
|
-
return
|
|
9395
|
+
return path63.join(root, RULE_CANDIDATES_ARCHIVE_DIR, `${id}${CANDIDATE_JSON_EXT}`);
|
|
9227
9396
|
}
|
|
9228
9397
|
var CATEGORY_CAP = 64;
|
|
9229
9398
|
var TITLE_CAP = 256;
|
|
@@ -9401,7 +9570,7 @@ async function readRuleCandidate(root, fileId) {
|
|
|
9401
9570
|
}
|
|
9402
9571
|
}
|
|
9403
9572
|
async function listRuleCandidates(root) {
|
|
9404
|
-
const dir =
|
|
9573
|
+
const dir = path63.join(root, RULE_CANDIDATES_DIR);
|
|
9405
9574
|
const fileIds = await listCandidateFileIds(dir);
|
|
9406
9575
|
const candidates = [];
|
|
9407
9576
|
for (const fileId of fileIds) {
|
|
@@ -9488,7 +9657,7 @@ ${rule.description}`;
|
|
|
9488
9657
|
);
|
|
9489
9658
|
}
|
|
9490
9659
|
async function extractForSource2(root, sourceFile, provenance, createdAt) {
|
|
9491
|
-
const sourcePath =
|
|
9660
|
+
const sourcePath = path64.join(root, SOURCES_DIR, sourceFile);
|
|
9492
9661
|
const raw = await readFile32(sourcePath, "utf-8");
|
|
9493
9662
|
const hash = createHash9("sha256").update(raw).digest("hex");
|
|
9494
9663
|
const { meta } = parseFrontmatter(raw);
|
|
@@ -9560,7 +9729,7 @@ function buildRuleCandidatesJson(candidates) {
|
|
|
9560
9729
|
var RULE_EXPORT_PATH = "dist/exports/rule-candidates.json";
|
|
9561
9730
|
async function rulesExtractCommand() {
|
|
9562
9731
|
const root = process.cwd();
|
|
9563
|
-
if (!existsSync20(
|
|
9732
|
+
if (!existsSync20(path65.join(root, SOURCES_DIR))) {
|
|
9564
9733
|
status("!", warn("No sources found. Run `llmwiki ingest <url>` first."));
|
|
9565
9734
|
return;
|
|
9566
9735
|
}
|
|
@@ -9623,7 +9792,7 @@ async function rulesExportCommand(options = {}) {
|
|
|
9623
9792
|
const root = process.cwd();
|
|
9624
9793
|
const scope = resolveScope(options.scope);
|
|
9625
9794
|
const candidates = await collectRuleCandidatesForExport(root, scope);
|
|
9626
|
-
const outPath =
|
|
9795
|
+
const outPath = path65.join(root, RULE_EXPORT_PATH);
|
|
9627
9796
|
await atomicWrite(outPath, buildRuleCandidatesJson(candidates));
|
|
9628
9797
|
status(
|
|
9629
9798
|
"+",
|
|
@@ -9708,8 +9877,8 @@ async function runRulesAction(work) {
|
|
|
9708
9877
|
}
|
|
9709
9878
|
|
|
9710
9879
|
// src/project/state.ts
|
|
9711
|
-
import { stat as stat3, readdir as
|
|
9712
|
-
import
|
|
9880
|
+
import { stat as stat3, readdir as readdir18, readFile as readFile33 } from "fs/promises";
|
|
9881
|
+
import path66 from "path";
|
|
9713
9882
|
var MARKDOWN_EXT = ".md";
|
|
9714
9883
|
async function collectProjectState(root) {
|
|
9715
9884
|
const rootReadable = await isDirectory(root);
|
|
@@ -9745,31 +9914,31 @@ function brokenProjectState(root) {
|
|
|
9745
9914
|
}
|
|
9746
9915
|
async function collectDirPresence(root) {
|
|
9747
9916
|
const [hasSourcesDir, hasWikiDir, hasInternalDir] = await Promise.all([
|
|
9748
|
-
isDirectory(
|
|
9749
|
-
isDirectory(
|
|
9750
|
-
isDirectory(
|
|
9917
|
+
isDirectory(path66.join(root, SOURCES_DIR)),
|
|
9918
|
+
isDirectory(path66.join(root, "wiki")),
|
|
9919
|
+
isDirectory(path66.join(root, LLMWIKI_DIR))
|
|
9751
9920
|
]);
|
|
9752
9921
|
return { hasSourcesDir, hasWikiDir, hasInternalDir };
|
|
9753
9922
|
}
|
|
9754
9923
|
async function collectPageCounts(root, dirs) {
|
|
9755
9924
|
const [sourceCount, conceptCount, queryCount, pendingCandidates, hasIndex] = await Promise.all([
|
|
9756
|
-
dirs.hasSourcesDir ? countMarkdownFiles(
|
|
9757
|
-
dirs.hasWikiDir ? countMarkdownFiles(
|
|
9758
|
-
dirs.hasWikiDir ? countMarkdownFiles(
|
|
9925
|
+
dirs.hasSourcesDir ? countMarkdownFiles(path66.join(root, SOURCES_DIR)) : 0,
|
|
9926
|
+
dirs.hasWikiDir ? countMarkdownFiles(path66.join(root, CONCEPTS_DIR)) : 0,
|
|
9927
|
+
dirs.hasWikiDir ? countMarkdownFiles(path66.join(root, QUERIES_DIR)) : 0,
|
|
9759
9928
|
dirs.hasInternalDir ? safeCountCandidates(root) : 0,
|
|
9760
|
-
dirs.hasWikiDir ? isFile(
|
|
9929
|
+
dirs.hasWikiDir ? isFile(path66.join(root, INDEX_FILE)) : false
|
|
9761
9930
|
]);
|
|
9762
9931
|
return { sourceCount, conceptCount, queryCount, pendingCandidates, hasIndex };
|
|
9763
9932
|
}
|
|
9764
9933
|
async function collectMtimes(root, dirs) {
|
|
9765
9934
|
const [latestWikiMtimeMs, latestSourceMtimeMs] = await Promise.all([
|
|
9766
|
-
dirs.hasWikiDir ? safeMtime(
|
|
9767
|
-
dirs.hasSourcesDir ? safeMtime(
|
|
9935
|
+
dirs.hasWikiDir ? safeMtime(path66.join(root, "wiki")) : Promise.resolve(null),
|
|
9936
|
+
dirs.hasSourcesDir ? safeMtime(path66.join(root, SOURCES_DIR)) : Promise.resolve(null)
|
|
9768
9937
|
]);
|
|
9769
9938
|
return { latestWikiMtimeMs, latestSourceMtimeMs };
|
|
9770
9939
|
}
|
|
9771
9940
|
async function collectLintCacheStatus(root) {
|
|
9772
|
-
const cachePath =
|
|
9941
|
+
const cachePath = path66.join(root, LAST_LINT_FILE);
|
|
9773
9942
|
const exists = await isFile(cachePath);
|
|
9774
9943
|
if (!exists) return { present: false, entry: null };
|
|
9775
9944
|
const entry = await readLintCacheEntry(cachePath);
|
|
@@ -9915,7 +10084,7 @@ async function safeMtime(target) {
|
|
|
9915
10084
|
}
|
|
9916
10085
|
async function countMarkdownFiles(dir) {
|
|
9917
10086
|
try {
|
|
9918
|
-
const entries = await
|
|
10087
|
+
const entries = await readdir18(dir, { withFileTypes: true });
|
|
9919
10088
|
let count = 0;
|
|
9920
10089
|
for (const entry of entries) {
|
|
9921
10090
|
if (entry.isFile() && entry.name.endsWith(MARKDOWN_EXT)) count += 1;
|
|
@@ -10131,8 +10300,8 @@ function plural(count) {
|
|
|
10131
10300
|
}
|
|
10132
10301
|
|
|
10133
10302
|
// src/compiler/refresh-plan.ts
|
|
10134
|
-
import { readdir as
|
|
10135
|
-
import
|
|
10303
|
+
import { readdir as readdir19 } from "fs/promises";
|
|
10304
|
+
import path67 from "path";
|
|
10136
10305
|
function ownersOf2(slug, snapshot) {
|
|
10137
10306
|
return Object.entries(snapshot.sources).filter(([, s]) => s.concepts.includes(slug)).map(([file, s]) => ({
|
|
10138
10307
|
file,
|
|
@@ -10183,7 +10352,7 @@ async function resolveStaleRefresh(root) {
|
|
|
10183
10352
|
}
|
|
10184
10353
|
const snapshot = await buildFreshnessSnapshot(root, classified);
|
|
10185
10354
|
const state = classified.state;
|
|
10186
|
-
const pages = await scanWikiPages(
|
|
10355
|
+
const pages = await scanWikiPages(path67.join(root, CONCEPTS_DIR));
|
|
10187
10356
|
const c = classifyPages(pages, snapshot);
|
|
10188
10357
|
return {
|
|
10189
10358
|
stateStatus: "ok",
|
|
@@ -10210,7 +10379,7 @@ function knownAffectedFor(state, changed, deleted) {
|
|
|
10210
10379
|
async function listNewSkipped(root, state) {
|
|
10211
10380
|
let files;
|
|
10212
10381
|
try {
|
|
10213
|
-
files = (await
|
|
10382
|
+
files = (await readdir19(path67.join(root, SOURCES_DIR))).filter((f) => f.endsWith(".md"));
|
|
10214
10383
|
} catch {
|
|
10215
10384
|
return [];
|
|
10216
10385
|
}
|
|
@@ -10326,7 +10495,7 @@ function planDetailRows(plan) {
|
|
|
10326
10495
|
}
|
|
10327
10496
|
|
|
10328
10497
|
// src/commands/quickstart.ts
|
|
10329
|
-
import
|
|
10498
|
+
import path68 from "path";
|
|
10330
10499
|
var QUICKSTART_JSON_VERSION = 1;
|
|
10331
10500
|
var VIEW_OPEN_ARGS_KEY = "view\0--open";
|
|
10332
10501
|
var NOOP_RESTORE = () => {
|
|
@@ -10393,7 +10562,7 @@ async function runIngestStep(source2) {
|
|
|
10393
10562
|
status("*", info(`Ingesting ${source2}`));
|
|
10394
10563
|
try {
|
|
10395
10564
|
const result = await ingestSource(process.cwd(), source2);
|
|
10396
|
-
const relPath =
|
|
10565
|
+
const relPath = path68.join(SOURCES_DIR, result.filename);
|
|
10397
10566
|
status("+", success(`Ingested \u2192 ${relPath}`));
|
|
10398
10567
|
return buildIngestSuccess(result, relPath);
|
|
10399
10568
|
} catch (err) {
|
|
@@ -10642,7 +10811,7 @@ function appendNextLines(lines, next) {
|
|
|
10642
10811
|
}
|
|
10643
10812
|
|
|
10644
10813
|
// src/commands/context.ts
|
|
10645
|
-
import
|
|
10814
|
+
import path69 from "path";
|
|
10646
10815
|
|
|
10647
10816
|
// src/context/ranking.ts
|
|
10648
10817
|
var WEIGHT_TITLE_MATCH = 0.5;
|
|
@@ -11466,7 +11635,7 @@ function appendPrimaryPages(lines, primary) {
|
|
|
11466
11635
|
for (const page of primary) appendPrimaryPage(lines, page);
|
|
11467
11636
|
}
|
|
11468
11637
|
function appendPrimaryPage(lines, page) {
|
|
11469
|
-
const pageFile =
|
|
11638
|
+
const pageFile = path69.join("wiki", page.pageDirectory, `${slugFromId(page.id)}.md`);
|
|
11470
11639
|
lines.push(`### ${page.title} (\`${pageFile}\`)`);
|
|
11471
11640
|
lines.push("");
|
|
11472
11641
|
lines.push(`Why included: ${page.reasons.join(", ") || "(no signals)"}`);
|
|
@@ -11527,8 +11696,8 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
|
|
|
11527
11696
|
import { z as z2 } from "zod";
|
|
11528
11697
|
|
|
11529
11698
|
// src/status/collect.ts
|
|
11530
|
-
import
|
|
11531
|
-
import { readdir as
|
|
11699
|
+
import path70 from "path";
|
|
11700
|
+
import { readdir as readdir20 } from "fs/promises";
|
|
11532
11701
|
var MAX_STATUS_LIST = 100;
|
|
11533
11702
|
function classifyConceptPages(scanned, snapshot) {
|
|
11534
11703
|
const stalePages = [];
|
|
@@ -11549,7 +11718,7 @@ function lastCompileTime(sources) {
|
|
|
11549
11718
|
}
|
|
11550
11719
|
async function listSourceFilesOnDisk(root) {
|
|
11551
11720
|
try {
|
|
11552
|
-
const entries = await
|
|
11721
|
+
const entries = await readdir20(path70.join(root, SOURCES_DIR));
|
|
11553
11722
|
return entries.filter((f) => f.endsWith(".md"));
|
|
11554
11723
|
} catch {
|
|
11555
11724
|
return [];
|
|
@@ -11577,9 +11746,9 @@ async function collectStatus(root) {
|
|
|
11577
11746
|
const classified = await readStateClassified(root);
|
|
11578
11747
|
const snapshot = await buildFreshnessSnapshot(root, classified);
|
|
11579
11748
|
const [conceptSummaries, queries, scannedConcepts, pendingCandidates, sourceFilesOnDisk] = await Promise.all([
|
|
11580
|
-
collectPageSummaries(
|
|
11581
|
-
collectPageSummaries(
|
|
11582
|
-
scanWikiPages(
|
|
11749
|
+
collectPageSummaries(path70.join(root, CONCEPTS_DIR)),
|
|
11750
|
+
collectPageSummaries(path70.join(root, QUERIES_DIR)),
|
|
11751
|
+
scanWikiPages(path70.join(root, CONCEPTS_DIR)),
|
|
11583
11752
|
countCandidates(root),
|
|
11584
11753
|
listSourceFilesOnDisk(root)
|
|
11585
11754
|
]);
|
|
@@ -11601,11 +11770,11 @@ async function collectStatus(root) {
|
|
|
11601
11770
|
}
|
|
11602
11771
|
|
|
11603
11772
|
// src/pages/read.ts
|
|
11604
|
-
import
|
|
11773
|
+
import path71 from "path";
|
|
11605
11774
|
var PAGE_DIRS2 = [CONCEPTS_DIR, QUERIES_DIR];
|
|
11606
11775
|
async function readPageRecord2(root, slug) {
|
|
11607
11776
|
for (const dir of PAGE_DIRS2) {
|
|
11608
|
-
const content = await safeReadFile(
|
|
11777
|
+
const content = await safeReadFile(path71.join(root, dir, `${slug}.md`));
|
|
11609
11778
|
if (!content) continue;
|
|
11610
11779
|
const { meta, body } = parseFrontmatter(content);
|
|
11611
11780
|
if (meta.orphaned) continue;
|
|
@@ -11620,7 +11789,7 @@ async function readPageRecord2(root, slug) {
|
|
|
11620
11789
|
}
|
|
11621
11790
|
|
|
11622
11791
|
// src/search/retrieval.ts
|
|
11623
|
-
import
|
|
11792
|
+
import path72 from "path";
|
|
11624
11793
|
function dedupePreservingOrder(slugs) {
|
|
11625
11794
|
const seen = /* @__PURE__ */ new Set();
|
|
11626
11795
|
const out = [];
|
|
@@ -11642,7 +11811,7 @@ async function pickSearchSlugs(root, question) {
|
|
|
11642
11811
|
if (candidates.length > 0) return candidates.map((c) => c.slug);
|
|
11643
11812
|
} catch {
|
|
11644
11813
|
}
|
|
11645
|
-
const indexContent = await safeReadFile(
|
|
11814
|
+
const indexContent = await safeReadFile(path72.join(root, INDEX_FILE));
|
|
11646
11815
|
const { pages } = await selectPages(question, indexContent);
|
|
11647
11816
|
return pages;
|
|
11648
11817
|
}
|
|
@@ -11655,13 +11824,21 @@ async function loadPageRecords(root, slugs) {
|
|
|
11655
11824
|
return records;
|
|
11656
11825
|
}
|
|
11657
11826
|
|
|
11658
|
-
// src/mcp/
|
|
11827
|
+
// src/mcp/result.ts
|
|
11659
11828
|
function jsonResult(payload) {
|
|
11660
11829
|
return {
|
|
11661
11830
|
content: [{ type: "text", text: JSON.stringify(payload, null, 2) }],
|
|
11662
11831
|
structuredContent: { result: payload }
|
|
11663
11832
|
};
|
|
11664
11833
|
}
|
|
11834
|
+
function errorResult(message) {
|
|
11835
|
+
return {
|
|
11836
|
+
content: [{ type: "text", text: message }],
|
|
11837
|
+
isError: true
|
|
11838
|
+
};
|
|
11839
|
+
}
|
|
11840
|
+
|
|
11841
|
+
// src/mcp/tools.ts
|
|
11665
11842
|
function registerWikiTools(server, root) {
|
|
11666
11843
|
registerIngestTool(server, root);
|
|
11667
11844
|
registerCompileTool(server, root);
|
|
@@ -11843,9 +12020,62 @@ function registerEvalTool(server, root) {
|
|
|
11843
12020
|
);
|
|
11844
12021
|
}
|
|
11845
12022
|
|
|
12023
|
+
// src/mcp/okf-tools.ts
|
|
12024
|
+
import { z as z3 } from "zod";
|
|
12025
|
+
var MAX_MCP_PENDING_CANDIDATES = 200;
|
|
12026
|
+
function registerOkfTools(server, root, maxPending = MAX_MCP_PENDING_CANDIDATES) {
|
|
12027
|
+
server.registerTool(
|
|
12028
|
+
"export_okf",
|
|
12029
|
+
{
|
|
12030
|
+
title: "Export OKF bundle",
|
|
12031
|
+
description: "Export the wiki as an Open Knowledge Format (OKF) v0.1 bundle. `out` defaults to dist/exports/okf and must stay inside the project.",
|
|
12032
|
+
inputSchema: { out: z3.string().optional().describe("Output dir for the bundle (must resolve inside the project root)") }
|
|
12033
|
+
},
|
|
12034
|
+
// withQuiet: the export core can print skipped-reference warnings via output.status;
|
|
12035
|
+
// on the MCP stdio transport those would land on the JSON-RPC stream and corrupt it.
|
|
12036
|
+
async ({ out }) => withQuiet(async () => {
|
|
12037
|
+
try {
|
|
12038
|
+
const dest = await confineUnderRoot(out ?? "dist/exports/okf", root, { mustExist: false });
|
|
12039
|
+
const report = await runOkfExport(root, { out: dest });
|
|
12040
|
+
return jsonResult({ outDir: report.outDir, fileCount: report.writtenPaths.length, files: report.writtenPaths.slice(0, 20), warnings: report.warnings });
|
|
12041
|
+
} catch (err) {
|
|
12042
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
12043
|
+
}
|
|
12044
|
+
})
|
|
12045
|
+
);
|
|
12046
|
+
server.registerTool(
|
|
12047
|
+
"import_okf",
|
|
12048
|
+
{
|
|
12049
|
+
title: "Import OKF bundle (staged for review)",
|
|
12050
|
+
description: "Import an OKF bundle from a path INSIDE the project as review candidates. Always staging-only \u2014 imported knowledge requires human review/approval before it enters the wiki. Use dryRun to preview.",
|
|
12051
|
+
inputSchema: {
|
|
12052
|
+
dir: z3.string().describe("Bundle directory (must resolve inside the project root)"),
|
|
12053
|
+
dryRun: z3.boolean().optional().describe("Preview only \u2014 stage nothing")
|
|
12054
|
+
}
|
|
12055
|
+
},
|
|
12056
|
+
// withQuiet: same stdio-stream guard as export_okf (the import core may emit warnings).
|
|
12057
|
+
async ({ dir, dryRun }) => withQuiet(async () => {
|
|
12058
|
+
try {
|
|
12059
|
+
const bundle = await confineUnderRoot(dir, root, { mustExist: true });
|
|
12060
|
+
const report = await runOkfImport(root, bundle, { trusted: false, dryRun, maxNewCandidates: maxPending });
|
|
12061
|
+
return jsonResult({
|
|
12062
|
+
mode: report.mode,
|
|
12063
|
+
imported: report.pages.length,
|
|
12064
|
+
pages: report.pages,
|
|
12065
|
+
skipped: report.skipped,
|
|
12066
|
+
warnings: report.warnings,
|
|
12067
|
+
nextAction: report.mode === "dry-run" ? "Preview only \u2014 re-run without dryRun to stage these as review candidates." : "Imported docs are STAGED as review candidates \u2014 approve them via `llmwiki review` (human gate) before they enter the wiki."
|
|
12068
|
+
});
|
|
12069
|
+
} catch (err) {
|
|
12070
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
12071
|
+
}
|
|
12072
|
+
})
|
|
12073
|
+
);
|
|
12074
|
+
}
|
|
12075
|
+
|
|
11846
12076
|
// src/mcp/resources.ts
|
|
11847
|
-
import
|
|
11848
|
-
import { readdir as
|
|
12077
|
+
import path73 from "path";
|
|
12078
|
+
import { readdir as readdir21 } from "fs/promises";
|
|
11849
12079
|
import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
11850
12080
|
function jsonContent(uri, payload) {
|
|
11851
12081
|
return {
|
|
@@ -11880,7 +12110,7 @@ function registerIndexResource(server, root) {
|
|
|
11880
12110
|
mimeType: "text/markdown"
|
|
11881
12111
|
},
|
|
11882
12112
|
async (uri) => {
|
|
11883
|
-
const content = await safeReadFile(
|
|
12113
|
+
const content = await safeReadFile(path73.join(root, INDEX_FILE));
|
|
11884
12114
|
return { contents: [markdownContent(uri, content)] };
|
|
11885
12115
|
}
|
|
11886
12116
|
);
|
|
@@ -11947,23 +12177,23 @@ function registerQueryResource(server, root) {
|
|
|
11947
12177
|
);
|
|
11948
12178
|
}
|
|
11949
12179
|
async function listSources(root) {
|
|
11950
|
-
const sourcesPath =
|
|
12180
|
+
const sourcesPath = path73.join(root, SOURCES_DIR);
|
|
11951
12181
|
let files;
|
|
11952
12182
|
try {
|
|
11953
|
-
files = await
|
|
12183
|
+
files = await readdir21(sourcesPath);
|
|
11954
12184
|
} catch {
|
|
11955
12185
|
return [];
|
|
11956
12186
|
}
|
|
11957
12187
|
const records = [];
|
|
11958
12188
|
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
11959
|
-
const content = await safeReadFile(
|
|
12189
|
+
const content = await safeReadFile(path73.join(sourcesPath, file));
|
|
11960
12190
|
const { meta } = parseFrontmatter(content);
|
|
11961
12191
|
records.push({ filename: file, ...meta });
|
|
11962
12192
|
}
|
|
11963
12193
|
return records;
|
|
11964
12194
|
}
|
|
11965
12195
|
async function loadPageWithMeta(root, dir, slug) {
|
|
11966
|
-
const filePath =
|
|
12196
|
+
const filePath = path73.join(root, dir, `${slug}.md`);
|
|
11967
12197
|
const content = await safeReadFile(filePath);
|
|
11968
12198
|
if (!content) {
|
|
11969
12199
|
throw new Error(`Page not found: ${dir}/${slug}.md`);
|
|
@@ -12002,10 +12232,10 @@ function registerEvalHistoryResource(server, root) {
|
|
|
12002
12232
|
);
|
|
12003
12233
|
}
|
|
12004
12234
|
async function listPagesUnder(root, dir, scheme) {
|
|
12005
|
-
const pagesPath =
|
|
12235
|
+
const pagesPath = path73.join(root, dir);
|
|
12006
12236
|
let files;
|
|
12007
12237
|
try {
|
|
12008
|
-
files = await
|
|
12238
|
+
files = await readdir21(pagesPath);
|
|
12009
12239
|
} catch {
|
|
12010
12240
|
return { resources: [] };
|
|
12011
12241
|
}
|
|
@@ -12023,6 +12253,7 @@ async function startMCPServer(options) {
|
|
|
12023
12253
|
instructions: "llmwiki is a knowledge compiler. Use ingest_source to add raw sources, compile_wiki to run the LLM pipeline, query_wiki for grounded answers, search_pages to retrieve relevant pages, and run_eval to score wiki quality. read_page, lint_wiki, wiki_status, and run_eval (fast suite, record: false) work without an API key and do not mutate state."
|
|
12024
12254
|
});
|
|
12025
12255
|
registerWikiTools(server, root);
|
|
12256
|
+
registerOkfTools(server, root);
|
|
12026
12257
|
registerWikiResources(server, root);
|
|
12027
12258
|
const transport = new StdioServerTransport();
|
|
12028
12259
|
await server.connect(transport);
|