llm-wiki-compiler 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +251 -0
- package/README.md +71 -9
- package/dist/cli.js +3072 -387
- package/dist/cli.js.map +1 -1
- package/dist/viewer/assets/THIRD_PARTY_NOTICES.txt +22 -0
- package/dist/viewer/assets/d3.min.js +2 -0
- package/dist/viewer/assets/index.html +4 -2
- package/dist/viewer/assets/viewer-graph.js +369 -0
- package/dist/viewer/assets/viewer-rail.js +115 -39
- package/dist/viewer/assets/viewer-search.js +48 -16
- package/dist/viewer/assets/viewer-sidebar.js +105 -35
- package/dist/viewer/assets/viewer.css +68 -1
- package/dist/viewer/assets/viewer.js +232 -115
- package/package.json +6 -3
package/dist/cli.js
CHANGED
|
@@ -14,7 +14,8 @@ import { writeFile, rename, readFile, mkdir } from "fs/promises";
|
|
|
14
14
|
import path from "path";
|
|
15
15
|
import yaml from "js-yaml";
|
|
16
16
|
var CITATION_MARKER_PATTERN = /\^\[([^\]]+)\]/g;
|
|
17
|
-
var SPAN_SUFFIX_PATTERN = /^(?<file>[^:#]+)(?:(?::(?<colonStart>\d+)(
|
|
17
|
+
var SPAN_SUFFIX_PATTERN = /^(?<file>[^:#]+)(?:(?::(?<colonStart>\d+)(?:[,-]\s*(?<colonEnd>\d+))?)|(?:#L(?<hashStart>\d+)(?:-L(?<hashEnd>\d+))?))?$/;
|
|
18
|
+
var COLON_MULTILINE_PATTERN = /^(?<file>[^:#]+):(?<lines>\d+(?:,\s*\d+)+)$/;
|
|
18
19
|
var MIN_LINE_NUMBER = 1;
|
|
19
20
|
var VALID_PROVENANCE_STATES = /* @__PURE__ */ new Set([
|
|
20
21
|
"extracted",
|
|
@@ -71,13 +72,31 @@ function extractClaimCitations(body) {
|
|
|
71
72
|
}
|
|
72
73
|
return citations;
|
|
73
74
|
}
|
|
75
|
+
function splitCitationMarker(inner) {
|
|
76
|
+
return inner.split(/,(?!\s*\d+\s*(?:,|$))/);
|
|
77
|
+
}
|
|
74
78
|
function parseCitationEntries(inner) {
|
|
75
79
|
const spans = [];
|
|
76
|
-
for (const part of inner
|
|
80
|
+
for (const part of splitCitationMarker(inner)) {
|
|
77
81
|
const trimmed = part.trim();
|
|
78
82
|
if (trimmed.length === 0) continue;
|
|
79
|
-
|
|
80
|
-
|
|
83
|
+
spans.push(...parseSpanEntries(trimmed));
|
|
84
|
+
}
|
|
85
|
+
return spans;
|
|
86
|
+
}
|
|
87
|
+
function parseSpanEntries(entry) {
|
|
88
|
+
const multi = COLON_MULTILINE_PATTERN.exec(entry);
|
|
89
|
+
if (multi?.groups) return parseCommaLines(multi.groups.file, multi.groups.lines);
|
|
90
|
+
const single = parseSpanEntry(entry);
|
|
91
|
+
return single !== void 0 ? [single] : [];
|
|
92
|
+
}
|
|
93
|
+
function parseCommaLines(file, linesStr) {
|
|
94
|
+
const spans = [];
|
|
95
|
+
for (const token of linesStr.split(/,\s*/)) {
|
|
96
|
+
const lineNum = Number(token);
|
|
97
|
+
if (isValidLineRange(lineNum, lineNum)) {
|
|
98
|
+
spans.push({ file, lines: { start: lineNum, end: lineNum } });
|
|
99
|
+
}
|
|
81
100
|
}
|
|
82
101
|
return spans;
|
|
83
102
|
}
|
|
@@ -284,10 +303,16 @@ function error(text) {
|
|
|
284
303
|
function source(text) {
|
|
285
304
|
return `${CYAN}${text}${RESET}`;
|
|
286
305
|
}
|
|
306
|
+
var quietMode = false;
|
|
307
|
+
function setQuiet(quiet) {
|
|
308
|
+
quietMode = quiet;
|
|
309
|
+
}
|
|
287
310
|
function status(icon, message) {
|
|
311
|
+
if (quietMode) return;
|
|
288
312
|
console.log(`${icon} ${message}`);
|
|
289
313
|
}
|
|
290
314
|
function header(title) {
|
|
315
|
+
if (quietMode) return;
|
|
291
316
|
console.log(`
|
|
292
317
|
${BOLD}${title}${RESET}`);
|
|
293
318
|
console.log(dim("\u2500".repeat(Math.min(title.length + 4, 60))));
|
|
@@ -458,7 +483,8 @@ var AnthropicProvider = class {
|
|
|
458
483
|
max_tokens: maxTokens,
|
|
459
484
|
system,
|
|
460
485
|
messages,
|
|
461
|
-
tools: anthropicTools
|
|
486
|
+
tools: anthropicTools,
|
|
487
|
+
tool_choice: { type: "any" }
|
|
462
488
|
});
|
|
463
489
|
const toolBlock = response.content.find((block) => block.type === "tool_use");
|
|
464
490
|
if (toolBlock?.type === "tool_use") {
|
|
@@ -702,8 +728,8 @@ function parseVtt(raw, filePath) {
|
|
|
702
728
|
const lines = raw.split("\n");
|
|
703
729
|
const output = [];
|
|
704
730
|
let inCue = false;
|
|
705
|
-
for (const
|
|
706
|
-
const trimmed =
|
|
731
|
+
for (const line2 of lines) {
|
|
732
|
+
const trimmed = line2.trim();
|
|
707
733
|
if (trimmed === "WEBVTT" || trimmed === "") {
|
|
708
734
|
inCue = false;
|
|
709
735
|
continue;
|
|
@@ -723,8 +749,8 @@ function parseVtt(raw, filePath) {
|
|
|
723
749
|
function parseSrt(raw, filePath) {
|
|
724
750
|
const lines = raw.split("\n");
|
|
725
751
|
const output = [];
|
|
726
|
-
for (const
|
|
727
|
-
const trimmed =
|
|
752
|
+
for (const line2 of lines) {
|
|
753
|
+
const trimmed = line2.trim();
|
|
728
754
|
if (trimmed === "" || SRT_SEQUENCE_PATTERN.test(trimmed)) {
|
|
729
755
|
continue;
|
|
730
756
|
}
|
|
@@ -939,9 +965,9 @@ function titleFromFirstUserMessage(turns) {
|
|
|
939
965
|
const firstUser = turns.find((t) => t.role === "user" && t.content.trim().length > 0);
|
|
940
966
|
return resolveSessionTitle(void 0, firstUser?.content, "Claude Session");
|
|
941
967
|
}
|
|
942
|
-
function parseLine(
|
|
968
|
+
function parseLine(line2) {
|
|
943
969
|
try {
|
|
944
|
-
return JSON.parse(
|
|
970
|
+
return JSON.parse(line2);
|
|
945
971
|
} catch {
|
|
946
972
|
return null;
|
|
947
973
|
}
|
|
@@ -976,8 +1002,8 @@ var claudeAdapter = {
|
|
|
976
1002
|
}
|
|
977
1003
|
const turns = [];
|
|
978
1004
|
const timestamps = [];
|
|
979
|
-
for (const [index,
|
|
980
|
-
const event = parseLine(
|
|
1005
|
+
for (const [index, line2] of lines.entries()) {
|
|
1006
|
+
const event = parseLine(line2);
|
|
981
1007
|
if (event === null) {
|
|
982
1008
|
throw new Error(
|
|
983
1009
|
`Malformed JSON on line ${index + 1} of Claude session: ${filePath}`
|
|
@@ -1009,24 +1035,35 @@ function unixToIso(ts) {
|
|
|
1009
1035
|
function extractTurns(mapping) {
|
|
1010
1036
|
const turns = [];
|
|
1011
1037
|
for (const node of Object.values(mapping)) {
|
|
1012
|
-
const
|
|
1013
|
-
if (
|
|
1014
|
-
const role = msg.author?.role;
|
|
1015
|
-
if (role !== "user" && role !== "assistant") continue;
|
|
1016
|
-
const content = (msg.content?.parts ?? []).join("\n").trim();
|
|
1017
|
-
if (content.length === 0) continue;
|
|
1018
|
-
turns.push({
|
|
1019
|
-
role,
|
|
1020
|
-
content,
|
|
1021
|
-
timestamp: msg.create_time != null ? unixToIso(msg.create_time) : void 0
|
|
1022
|
-
});
|
|
1038
|
+
const turn = nodeToTurn(node);
|
|
1039
|
+
if (turn) turns.push(turn);
|
|
1023
1040
|
}
|
|
1024
|
-
turns.sort(
|
|
1025
|
-
if (!a.timestamp || !b.timestamp) return 0;
|
|
1026
|
-
return a.timestamp.localeCompare(b.timestamp);
|
|
1027
|
-
});
|
|
1041
|
+
turns.sort(compareTurnsByTimestamp);
|
|
1028
1042
|
return turns;
|
|
1029
1043
|
}
|
|
1044
|
+
function nodeToTurn(node) {
|
|
1045
|
+
const msg = node.message;
|
|
1046
|
+
if (!msg) return null;
|
|
1047
|
+
const role = normalizeRole(msg.author?.role);
|
|
1048
|
+
if (!role) return null;
|
|
1049
|
+
const content = joinTrimmedParts(msg.content?.parts);
|
|
1050
|
+
if (content.length === 0) return null;
|
|
1051
|
+
return { role, content, timestamp: timestampFromUnix(msg.create_time) };
|
|
1052
|
+
}
|
|
1053
|
+
function normalizeRole(role) {
|
|
1054
|
+
if (role === "user" || role === "assistant") return role;
|
|
1055
|
+
return null;
|
|
1056
|
+
}
|
|
1057
|
+
function joinTrimmedParts(parts) {
|
|
1058
|
+
return (parts ?? []).join("\n").trim();
|
|
1059
|
+
}
|
|
1060
|
+
function timestampFromUnix(ts) {
|
|
1061
|
+
return ts != null ? unixToIso(ts) : void 0;
|
|
1062
|
+
}
|
|
1063
|
+
function compareTurnsByTimestamp(a, b) {
|
|
1064
|
+
if (!a.timestamp || !b.timestamp) return 0;
|
|
1065
|
+
return a.timestamp.localeCompare(b.timestamp);
|
|
1066
|
+
}
|
|
1030
1067
|
function isCodexExport(value) {
|
|
1031
1068
|
return Array.isArray(value) && value.length > 0 && typeof value[0].mapping === "object";
|
|
1032
1069
|
}
|
|
@@ -1319,6 +1356,87 @@ async function buildHealthResponse(snapshot) {
|
|
|
1319
1356
|
};
|
|
1320
1357
|
}
|
|
1321
1358
|
|
|
1359
|
+
// src/viewer/graph.ts
|
|
1360
|
+
var DEFAULT_KIND = "concept";
|
|
1361
|
+
function resolvePageKind(frontmatter) {
|
|
1362
|
+
return typeof frontmatter.kind === "string" && frontmatter.kind.length > 0 ? frontmatter.kind : DEFAULT_KIND;
|
|
1363
|
+
}
|
|
1364
|
+
function buildGraphData(pages) {
|
|
1365
|
+
const pageIds = new Set(pages.map((p) => p.id));
|
|
1366
|
+
const edges = buildEdges(pages);
|
|
1367
|
+
const ghostDisplayMap = buildGhostDisplayMap(pages);
|
|
1368
|
+
const inDegreeMap = buildInDegreeMap(edges);
|
|
1369
|
+
const realNodes = pages.map((p) => buildNode(p, pageIds, inDegreeMap));
|
|
1370
|
+
const ghostNodes = buildGhostNodes(edges, pageIds, inDegreeMap, ghostDisplayMap);
|
|
1371
|
+
return { nodes: [...realNodes, ...ghostNodes], edges };
|
|
1372
|
+
}
|
|
1373
|
+
function buildGhostDisplayMap(pages) {
|
|
1374
|
+
const map = /* @__PURE__ */ new Map();
|
|
1375
|
+
for (const page of pages) {
|
|
1376
|
+
for (const { slug, display } of page.danglingLinks ?? []) {
|
|
1377
|
+
const id = ghostId(slug);
|
|
1378
|
+
if (!map.has(id)) map.set(id, display);
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
return map;
|
|
1382
|
+
}
|
|
1383
|
+
var GHOST_DIRECTORY = "concepts";
|
|
1384
|
+
function ghostId(slug) {
|
|
1385
|
+
return `${GHOST_DIRECTORY}/${slug}`;
|
|
1386
|
+
}
|
|
1387
|
+
function buildEdges(pages) {
|
|
1388
|
+
const edges = [];
|
|
1389
|
+
for (const page of pages) {
|
|
1390
|
+
for (const target of page.outgoingLinks) {
|
|
1391
|
+
edges.push({ source: page.id, target });
|
|
1392
|
+
}
|
|
1393
|
+
for (const { slug } of page.danglingLinks ?? []) {
|
|
1394
|
+
edges.push({ source: page.id, target: ghostId(slug) });
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
return edges;
|
|
1398
|
+
}
|
|
1399
|
+
function buildGhostNodes(edges, pageIds, inDegreeMap, displayMap) {
|
|
1400
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1401
|
+
const ghosts = [];
|
|
1402
|
+
for (const { target } of edges) {
|
|
1403
|
+
if (pageIds.has(target) || seen.has(target)) continue;
|
|
1404
|
+
seen.add(target);
|
|
1405
|
+
const [directory, ...rest] = target.split("/");
|
|
1406
|
+
const slug = rest.join("/");
|
|
1407
|
+
ghosts.push({
|
|
1408
|
+
id: target,
|
|
1409
|
+
title: displayMap.get(target) ?? slug,
|
|
1410
|
+
slug,
|
|
1411
|
+
directory,
|
|
1412
|
+
kind: "dangling",
|
|
1413
|
+
degree: inDegreeMap.get(target) ?? 0,
|
|
1414
|
+
isDangling: true
|
|
1415
|
+
});
|
|
1416
|
+
}
|
|
1417
|
+
return ghosts;
|
|
1418
|
+
}
|
|
1419
|
+
function buildInDegreeMap(edges) {
|
|
1420
|
+
const map = /* @__PURE__ */ new Map();
|
|
1421
|
+
for (const edge of edges) {
|
|
1422
|
+
map.set(edge.target, (map.get(edge.target) ?? 0) + 1);
|
|
1423
|
+
}
|
|
1424
|
+
return map;
|
|
1425
|
+
}
|
|
1426
|
+
function buildNode(page, pageIds, inDegreeMap) {
|
|
1427
|
+
const outDegree = page.outgoingLinks.filter((t) => pageIds.has(t)).length;
|
|
1428
|
+
const inDegree = inDegreeMap.get(page.id) ?? 0;
|
|
1429
|
+
const kind = resolvePageKind(page.frontmatter);
|
|
1430
|
+
return {
|
|
1431
|
+
id: page.id,
|
|
1432
|
+
title: page.title,
|
|
1433
|
+
slug: page.slug,
|
|
1434
|
+
directory: page.pageDirectory,
|
|
1435
|
+
kind,
|
|
1436
|
+
degree: outDegree + inDegree
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1322
1440
|
// src/viewer/shell.ts
|
|
1323
1441
|
import { readFile as readFile12 } from "fs/promises";
|
|
1324
1442
|
import path14 from "path";
|
|
@@ -1342,7 +1460,7 @@ function substitutePageIndex(template, pages) {
|
|
|
1342
1460
|
pageDirectory: page.pageDirectory,
|
|
1343
1461
|
slug: page.slug,
|
|
1344
1462
|
title: page.title,
|
|
1345
|
-
kind:
|
|
1463
|
+
kind: resolvePageKind(page.frontmatter)
|
|
1346
1464
|
}));
|
|
1347
1465
|
const json = JSON.stringify({ pages: embedded }).replace(/</g, "\\u003c");
|
|
1348
1466
|
const block = `<script type="application/json" id="page-index">${json}</script>`;
|
|
@@ -1471,7 +1589,7 @@ import sanitizeHtml from "sanitize-html";
|
|
|
1471
1589
|
// src/wiki/collect.ts
|
|
1472
1590
|
import { readdir as readdir2, readFile as readFile14, realpath as realpath2 } from "fs/promises";
|
|
1473
1591
|
import path17 from "path";
|
|
1474
|
-
var WIKILINK_RE = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
|
|
1592
|
+
var WIKILINK_RE = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
|
|
1475
1593
|
function extractWikilinkSlugs(body) {
|
|
1476
1594
|
const slugs = /* @__PURE__ */ new Set();
|
|
1477
1595
|
WIKILINK_RE.lastIndex = 0;
|
|
@@ -1481,6 +1599,23 @@ function extractWikilinkSlugs(body) {
|
|
|
1481
1599
|
}
|
|
1482
1600
|
return [...slugs];
|
|
1483
1601
|
}
|
|
1602
|
+
function extractWikilinkTargets(body) {
|
|
1603
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1604
|
+
const targets = [];
|
|
1605
|
+
WIKILINK_RE.lastIndex = 0;
|
|
1606
|
+
let match;
|
|
1607
|
+
while ((match = WIKILINK_RE.exec(body)) !== null) {
|
|
1608
|
+
const target = match[1].trim();
|
|
1609
|
+
const alias = match[2]?.trim();
|
|
1610
|
+
const slug = slugify(target);
|
|
1611
|
+
const display = alias ?? target;
|
|
1612
|
+
if (!seen.has(slug)) {
|
|
1613
|
+
seen.add(slug);
|
|
1614
|
+
targets.push({ slug, display });
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
return targets;
|
|
1618
|
+
}
|
|
1484
1619
|
async function safeRealpath(p) {
|
|
1485
1620
|
try {
|
|
1486
1621
|
return await realpath2(p);
|
|
@@ -1576,11 +1711,24 @@ function resolveBareSlugList(targets, pages) {
|
|
|
1576
1711
|
function decoratePages(raw) {
|
|
1577
1712
|
const shells = raw.map(buildPageShell);
|
|
1578
1713
|
for (const page of shells) {
|
|
1579
|
-
const
|
|
1580
|
-
|
|
1714
|
+
const slugTargets = extractWikilinkSlugs(page.body);
|
|
1715
|
+
const richTargets = extractWikilinkTargets(page.body);
|
|
1716
|
+
page.outgoingLinks = resolveBareSlugList(slugTargets, shells);
|
|
1717
|
+
page.danglingLinks = collectDanglingLinks(richTargets, shells);
|
|
1581
1718
|
}
|
|
1582
1719
|
return shells;
|
|
1583
1720
|
}
|
|
1721
|
+
function collectDanglingLinks(targets, pages) {
|
|
1722
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1723
|
+
const dangling = [];
|
|
1724
|
+
for (const t of targets) {
|
|
1725
|
+
if (resolveBareSlug(t.slug, pages) === null && !seen.has(t.slug)) {
|
|
1726
|
+
seen.add(t.slug);
|
|
1727
|
+
dangling.push(t);
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
return dangling;
|
|
1731
|
+
}
|
|
1584
1732
|
function buildPageShell(page) {
|
|
1585
1733
|
const id = `${page.pageDirectory}/${page.slug}`;
|
|
1586
1734
|
return {
|
|
@@ -2041,21 +2189,25 @@ async function routeRegistered(req, res, parsedUrl, snapshot, isLoopback) {
|
|
|
2041
2189
|
if (parsedUrl.pathname === "/api/index") return handleApiIndex(res, snapshot, isLoopback);
|
|
2042
2190
|
if (parsedUrl.pathname === "/api/health") return handleApiHealth(res, snapshot);
|
|
2043
2191
|
if (parsedUrl.pathname === "/api/search") return handleApiSearch(res, parsedUrl, snapshot);
|
|
2192
|
+
if (parsedUrl.pathname === "/api/graph") return handleApiGraph(res, snapshot);
|
|
2044
2193
|
if (parsedUrl.pathname.startsWith("/api/page/")) {
|
|
2045
2194
|
return handleApiPage(res, parsedUrl.pathname, snapshot, isLoopback);
|
|
2046
2195
|
}
|
|
2047
2196
|
throw new Error(`route registration drift: no handler for ${parsedUrl.pathname}`);
|
|
2048
2197
|
}
|
|
2198
|
+
var REGISTERED_EXACT_PATHS = /* @__PURE__ */ new Set([
|
|
2199
|
+
"/",
|
|
2200
|
+
"/api/pages",
|
|
2201
|
+
"/api/index",
|
|
2202
|
+
"/api/health",
|
|
2203
|
+
"/api/search",
|
|
2204
|
+
"/api/graph"
|
|
2205
|
+
]);
|
|
2206
|
+
var REGISTERED_PATH_PREFIXES = ["/assets/", "/api/page/"];
|
|
2049
2207
|
function isRouteRegistered(method, pathname) {
|
|
2050
2208
|
if (method !== "GET") return false;
|
|
2051
|
-
if (pathname
|
|
2052
|
-
|
|
2053
|
-
if (pathname === "/api/pages") return true;
|
|
2054
|
-
if (pathname === "/api/index") return true;
|
|
2055
|
-
if (pathname === "/api/health") return true;
|
|
2056
|
-
if (pathname === "/api/search") return true;
|
|
2057
|
-
if (pathname.startsWith("/api/page/")) return true;
|
|
2058
|
-
return false;
|
|
2209
|
+
if (REGISTERED_EXACT_PATHS.has(pathname)) return true;
|
|
2210
|
+
return REGISTERED_PATH_PREFIXES.some((prefix) => pathname.startsWith(prefix));
|
|
2059
2211
|
}
|
|
2060
2212
|
function applySecurityHeaders(res) {
|
|
2061
2213
|
res.setHeader("Content-Security-Policy", CONTENT_SECURITY_POLICY);
|
|
@@ -2143,7 +2295,7 @@ function pageListRow(page) {
|
|
|
2143
2295
|
pageDirectory: page.pageDirectory,
|
|
2144
2296
|
slug: page.slug,
|
|
2145
2297
|
title: page.title,
|
|
2146
|
-
kind:
|
|
2298
|
+
kind: resolvePageKind(page.frontmatter),
|
|
2147
2299
|
summary: typeof page.frontmatter.summary === "string" ? page.frontmatter.summary : "",
|
|
2148
2300
|
updatedAt: typeof page.frontmatter.updatedAt === "string" ? page.frontmatter.updatedAt : "",
|
|
2149
2301
|
warnings: page.warnings
|
|
@@ -2165,6 +2317,9 @@ function handleApiIndex(res, snapshot, isLoopback) {
|
|
|
2165
2317
|
generatedAt: snapshot.generatedAt
|
|
2166
2318
|
});
|
|
2167
2319
|
}
|
|
2320
|
+
function handleApiGraph(res, snapshot) {
|
|
2321
|
+
writeJson(res, 200, snapshot.graph);
|
|
2322
|
+
}
|
|
2168
2323
|
async function handleApiHealth(res, snapshot) {
|
|
2169
2324
|
const health = await buildHealthResponse(snapshot);
|
|
2170
2325
|
writeJson(res, 200, health);
|
|
@@ -2427,6 +2582,7 @@ async function buildViewerSnapshot(root) {
|
|
|
2427
2582
|
};
|
|
2428
2583
|
const sourceFileSet = new Set(sourceFilenames);
|
|
2429
2584
|
const annotatedPages = pages.map((page) => annotateCitationWarnings(page, sourceFileSet));
|
|
2585
|
+
const graph = buildGraphData(annotatedPages);
|
|
2430
2586
|
return {
|
|
2431
2587
|
root,
|
|
2432
2588
|
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -2435,7 +2591,8 @@ async function buildViewerSnapshot(root) {
|
|
|
2435
2591
|
index: fullIndex,
|
|
2436
2592
|
recentPages: buildRecentPages(annotatedPages),
|
|
2437
2593
|
pages: annotatedPages,
|
|
2438
|
-
sourceFilenames
|
|
2594
|
+
sourceFilenames,
|
|
2595
|
+
graph
|
|
2439
2596
|
};
|
|
2440
2597
|
}
|
|
2441
2598
|
function annotateCitationWarnings(page, sourceFiles) {
|
|
@@ -2560,19 +2717,22 @@ function openInBrowser(url) {
|
|
|
2560
2717
|
function resolveBindConfig(options) {
|
|
2561
2718
|
const hostFlag = typeof options.host === "string" && options.host.length > 0;
|
|
2562
2719
|
const allowLan = options.allowLan === true;
|
|
2563
|
-
|
|
2564
|
-
throw new Error(
|
|
2565
|
-
"Privacy gate: --host and --allow-lan must be supplied together. Use both to bind beyond loopback, or neither to keep the viewer on 127.0.0.1."
|
|
2566
|
-
);
|
|
2567
|
-
}
|
|
2720
|
+
assertHostAllowLanSymmetry(hostFlag, allowLan);
|
|
2568
2721
|
const host = hostFlag ? options.host : LOOPBACK_HOST;
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2722
|
+
assertHostNotWildcard(host);
|
|
2723
|
+
return { host, port: parsePort(options.port) };
|
|
2724
|
+
}
|
|
2725
|
+
function assertHostAllowLanSymmetry(hostFlag, allowLan) {
|
|
2726
|
+
if (hostFlag === allowLan) return;
|
|
2727
|
+
throw new Error(
|
|
2728
|
+
"Privacy gate: --host and --allow-lan must be supplied together. Use both to bind beyond loopback, or neither to keep the viewer on 127.0.0.1."
|
|
2729
|
+
);
|
|
2730
|
+
}
|
|
2731
|
+
function assertHostNotWildcard(host) {
|
|
2732
|
+
if (!WILDCARD_HOSTS.has(host)) return;
|
|
2733
|
+
throw new Error(
|
|
2734
|
+
`--host ${host} is not supported: wildcard binds defeat the viewer's DNS-rebind protection. Use a specific interface IP (e.g. 192.168.1.10) instead.`
|
|
2735
|
+
);
|
|
2576
2736
|
}
|
|
2577
2737
|
function buildReadyUrl(host, port) {
|
|
2578
2738
|
if (host.includes(":")) return `http://[${host}]:${port}`;
|
|
@@ -2581,11 +2741,14 @@ function buildReadyUrl(host, port) {
|
|
|
2581
2741
|
function parsePort(raw) {
|
|
2582
2742
|
if (raw === void 0) return 0;
|
|
2583
2743
|
const value = typeof raw === "number" ? raw : Number(raw);
|
|
2584
|
-
if (!
|
|
2744
|
+
if (!isValidPort(value)) {
|
|
2585
2745
|
throw new Error(`Invalid --port value: ${raw}`);
|
|
2586
2746
|
}
|
|
2587
2747
|
return value;
|
|
2588
2748
|
}
|
|
2749
|
+
function isValidPort(value) {
|
|
2750
|
+
return Number.isInteger(value) && value >= 0 && value <= 65535;
|
|
2751
|
+
}
|
|
2589
2752
|
function registerShutdown(close) {
|
|
2590
2753
|
const shutdown = async () => {
|
|
2591
2754
|
try {
|
|
@@ -2902,6 +3065,11 @@ function getActiveProviderName() {
|
|
|
2902
3065
|
function sleep(ms) {
|
|
2903
3066
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2904
3067
|
}
|
|
3068
|
+
var NON_RETRIABLE_RE = /^4(?!29)\d\d\b/;
|
|
3069
|
+
function isNonRetriable(error2) {
|
|
3070
|
+
const msg = error2 instanceof Error ? error2.message : String(error2);
|
|
3071
|
+
return NON_RETRIABLE_RE.test(msg);
|
|
3072
|
+
}
|
|
2905
3073
|
async function callClaude(options) {
|
|
2906
3074
|
const { system, messages, tools, maxTokens = 4096, stream = false, onToken } = options;
|
|
2907
3075
|
const provider = getProvider();
|
|
@@ -2915,7 +3083,7 @@ async function callClaude(options) {
|
|
|
2915
3083
|
}
|
|
2916
3084
|
return await provider.complete(system, messages, maxTokens);
|
|
2917
3085
|
} catch (error2) {
|
|
2918
|
-
if (attempt === RETRY_COUNT) throw error2;
|
|
3086
|
+
if (attempt === RETRY_COUNT || isNonRetriable(error2)) throw error2;
|
|
2919
3087
|
const delayMs = RETRY_BASE_MS * Math.pow(RETRY_MULTIPLIER, attempt);
|
|
2920
3088
|
const errMsg = error2 instanceof Error ? error2.message : String(error2);
|
|
2921
3089
|
console.warn(`\u26A0 API call failed (attempt ${attempt + 1}/${RETRY_COUNT + 1}): ${errMsg}`);
|
|
@@ -3033,6 +3201,11 @@ function languageDirective() {
|
|
|
3033
3201
|
if (!lang) return "";
|
|
3034
3202
|
return `Write the output in ${lang}.`;
|
|
3035
3203
|
}
|
|
3204
|
+
function applyLanguageOption(lang) {
|
|
3205
|
+
if (lang && lang.trim().length > 0) {
|
|
3206
|
+
process.env.LLMWIKI_OUTPUT_LANG = lang.trim();
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3036
3209
|
|
|
3037
3210
|
// src/compiler/prompts.ts
|
|
3038
3211
|
function withLangLine(...lines) {
|
|
@@ -3151,16 +3324,17 @@ ${relatedPages}` : "";
|
|
|
3151
3324
|
),
|
|
3152
3325
|
"",
|
|
3153
3326
|
"Source attribution: at the end of each prose paragraph, append a citation",
|
|
3154
|
-
"marker
|
|
3155
|
-
"
|
|
3156
|
-
"
|
|
3157
|
-
"
|
|
3158
|
-
"
|
|
3159
|
-
"
|
|
3160
|
-
"Paragraph-level citations remain the default; only switch to claim-level form",
|
|
3161
|
-
"when it materially improves verifiability and the line range is unambiguous.",
|
|
3327
|
+
"marker identifying which source file(s) and line range the paragraph drew from.",
|
|
3328
|
+
"PREFERRED format: ^[filename.md:START-END] where START and END are the line numbers",
|
|
3329
|
+
"shown in the numbered source content below (e.g. ' 42 | some text' \u2192 line 42).",
|
|
3330
|
+
"Use this whenever you can identify the specific numbered lines supporting the claim.",
|
|
3331
|
+
"Fallback format: ^[filename.md] when the claim draws from the source broadly and",
|
|
3332
|
+
"no specific line range applies. For multi-source paragraphs: ^[a.md:1-5, b.md:10-12].",
|
|
3162
3333
|
"Place citations only at the end of prose paragraphs or sentences \u2014 not on",
|
|
3163
3334
|
"headings, list items, or code blocks.",
|
|
3335
|
+
"Do not cite YAML frontmatter lines (the --- ... --- block at the top of a file) as",
|
|
3336
|
+
"source evidence for substantive claims \u2014 those lines are metadata, not content.",
|
|
3337
|
+
"If a claim relates to a metadata field (e.g. document date or author), leave it uncited.",
|
|
3164
3338
|
"Source filenames are visible as `--- SOURCE: filename.md ---` headers in the content below.",
|
|
3165
3339
|
"",
|
|
3166
3340
|
"If a paragraph is your inference rather than a direct extraction, leave it",
|
|
@@ -3349,7 +3523,7 @@ function defaultSchemaInitPath(root) {
|
|
|
3349
3523
|
// src/schema/helpers.ts
|
|
3350
3524
|
import yaml3 from "js-yaml";
|
|
3351
3525
|
var WIKILINK_PATTERN = /\[\[([^\]]+)\]\]/g;
|
|
3352
|
-
function
|
|
3526
|
+
function resolvePageKind2(rawKind, schema) {
|
|
3353
3527
|
if (typeof rawKind === "string" && PAGE_KINDS.includes(rawKind)) {
|
|
3354
3528
|
return rawKind;
|
|
3355
3529
|
}
|
|
@@ -3753,10 +3927,15 @@ function buildBudgetedCombinedContent(concept, slices) {
|
|
|
3753
3927
|
);
|
|
3754
3928
|
return formatSlices(trimmed);
|
|
3755
3929
|
}
|
|
3930
|
+
function numberLines(content) {
|
|
3931
|
+
const lines = content.split("\n");
|
|
3932
|
+
const width = String(lines.length).length;
|
|
3933
|
+
return lines.map((line2, i) => `${String(i + 1).padStart(width)} | ${line2}`).join("\n");
|
|
3934
|
+
}
|
|
3756
3935
|
function formatSlices(slices) {
|
|
3757
3936
|
return slices.map((s) => `--- SOURCE: ${s.file} ---
|
|
3758
3937
|
|
|
3759
|
-
${s.content}`).join("\n\n");
|
|
3938
|
+
${numberLines(s.content)}`).join("\n\n");
|
|
3760
3939
|
}
|
|
3761
3940
|
function warnTruncation(concept, totalRaw, sourceCount, perSource, budget) {
|
|
3762
3941
|
status(
|
|
@@ -4308,15 +4487,16 @@ async function checkBrokenWikilinks(root) {
|
|
|
4308
4487
|
const existingSlugs = buildPageSlugSet(pages);
|
|
4309
4488
|
const results = [];
|
|
4310
4489
|
for (const page of pages) {
|
|
4311
|
-
for (const { captured, line } of findMatchesInContent(page.content, WIKILINK_PATTERN2)) {
|
|
4312
|
-
const
|
|
4490
|
+
for (const { captured, line: line2 } of findMatchesInContent(page.content, WIKILINK_PATTERN2)) {
|
|
4491
|
+
const linkTarget = captured.split("|")[0].trim();
|
|
4492
|
+
const linkSlug = slugify(linkTarget);
|
|
4313
4493
|
if (!existingSlugs.has(linkSlug)) {
|
|
4314
4494
|
results.push({
|
|
4315
4495
|
rule: "broken-wikilink",
|
|
4316
4496
|
severity: "error",
|
|
4317
4497
|
file: page.filePath,
|
|
4318
4498
|
message: `Broken wikilink [[${captured}]] \u2014 no matching page found`,
|
|
4319
|
-
line
|
|
4499
|
+
line: line2
|
|
4320
4500
|
});
|
|
4321
4501
|
}
|
|
4322
4502
|
}
|
|
@@ -4474,7 +4654,7 @@ function countUncitedProseParagraphs(body) {
|
|
|
4474
4654
|
}
|
|
4475
4655
|
return count;
|
|
4476
4656
|
}
|
|
4477
|
-
var COLON_SPAN_PATTERN = /^[^:#]+:(\d+)(
|
|
4657
|
+
var COLON_SPAN_PATTERN = /^[^:#]+:(\d+)(?:[,-]\s*(\d+))?$/;
|
|
4478
4658
|
var HASH_SPAN_PATTERN = /^[^:#]+#L(\d+)(?:-L(\d+))?$/;
|
|
4479
4659
|
async function checkSchemaCrossLinks(root, schema) {
|
|
4480
4660
|
const pages = await collectAllPages(root);
|
|
@@ -4486,7 +4666,7 @@ async function checkSchemaCrossLinks(root, schema) {
|
|
|
4486
4666
|
}
|
|
4487
4667
|
function checkPageCrossLinks(content, filePath, schema) {
|
|
4488
4668
|
const { meta, body } = parseFrontmatter(content);
|
|
4489
|
-
const kind =
|
|
4669
|
+
const kind = resolvePageKind2(meta.kind, schema);
|
|
4490
4670
|
const rule = schema.kinds[kind];
|
|
4491
4671
|
if (rule.minWikilinks <= 0) return [];
|
|
4492
4672
|
const linkCount = countWikilinks(body);
|
|
@@ -4537,13 +4717,13 @@ async function checkBrokenCitations(root) {
|
|
|
4537
4717
|
}
|
|
4538
4718
|
async function checkPageBrokenCitations(content, filePath, sourcesDir, lineCountCache = /* @__PURE__ */ new Map()) {
|
|
4539
4719
|
const results = [];
|
|
4540
|
-
for (const { captured, line } of findMatchesInContent(content, CITATION_PATTERN)) {
|
|
4541
|
-
await collectBrokenForMarker(captured,
|
|
4720
|
+
for (const { captured, line: line2 } of findMatchesInContent(content, CITATION_PATTERN)) {
|
|
4721
|
+
await collectBrokenForMarker(captured, line2, filePath, sourcesDir, lineCountCache, results);
|
|
4542
4722
|
}
|
|
4543
4723
|
return results;
|
|
4544
4724
|
}
|
|
4545
|
-
async function collectBrokenForMarker(captured,
|
|
4546
|
-
for (const part of captured
|
|
4725
|
+
async function collectBrokenForMarker(captured, line2, pageFile, sourcesDir, lineCountCache, out) {
|
|
4726
|
+
for (const part of splitCitationMarker(captured)) {
|
|
4547
4727
|
const trimmed = part.trim();
|
|
4548
4728
|
if (trimmed.length === 0) continue;
|
|
4549
4729
|
const filename = stripSpanSuffix(trimmed);
|
|
@@ -4554,7 +4734,7 @@ async function collectBrokenForMarker(captured, line, pageFile, sourcesDir, line
|
|
|
4554
4734
|
severity: "error",
|
|
4555
4735
|
file: pageFile,
|
|
4556
4736
|
message: `Broken citation ^[${filename}] \u2014 source file not found`,
|
|
4557
|
-
line
|
|
4737
|
+
line: line2
|
|
4558
4738
|
});
|
|
4559
4739
|
continue;
|
|
4560
4740
|
}
|
|
@@ -4567,7 +4747,7 @@ async function collectBrokenForMarker(captured, line, pageFile, sourcesDir, line
|
|
|
4567
4747
|
severity: "error",
|
|
4568
4748
|
file: pageFile,
|
|
4569
4749
|
message: `Claim-level span ^[${trimmed}] is out of bounds (source has only ${lineCount} lines)`,
|
|
4570
|
-
line
|
|
4750
|
+
line: line2
|
|
4571
4751
|
});
|
|
4572
4752
|
}
|
|
4573
4753
|
}
|
|
@@ -4589,15 +4769,15 @@ async function checkMalformedClaimCitations(root) {
|
|
|
4589
4769
|
}
|
|
4590
4770
|
function checkPageMalformedCitations(content, filePath) {
|
|
4591
4771
|
const results = [];
|
|
4592
|
-
for (const { captured, line } of findMatchesInContent(content, CITATION_PATTERN)) {
|
|
4593
|
-
for (const part of captured
|
|
4772
|
+
for (const { captured, line: line2 } of findMatchesInContent(content, CITATION_PATTERN)) {
|
|
4773
|
+
for (const part of splitCitationMarker(captured)) {
|
|
4594
4774
|
if (!isMalformedCitationEntry(part)) continue;
|
|
4595
4775
|
results.push({
|
|
4596
4776
|
rule: "malformed-claim-citation",
|
|
4597
4777
|
severity: "error",
|
|
4598
4778
|
file: filePath,
|
|
4599
4779
|
message: `Malformed claim citation ^[${captured}] \u2014 expected file.md, file.md:N-N, or file.md#LN-LN`,
|
|
4600
|
-
line
|
|
4780
|
+
line: line2
|
|
4601
4781
|
});
|
|
4602
4782
|
}
|
|
4603
4783
|
}
|
|
@@ -5207,8 +5387,8 @@ function collapseToPages(chunks, limit) {
|
|
|
5207
5387
|
return slugs;
|
|
5208
5388
|
}
|
|
5209
5389
|
function buildChunkReasoning(chunks, pages) {
|
|
5210
|
-
const
|
|
5211
|
-
const summary =
|
|
5390
|
+
const top2 = chunks.slice(0, pages.length);
|
|
5391
|
+
const summary = top2.map((c) => `${c.slug}#${c.chunkIndex} (${c.score.toFixed(3)})`).join(", ");
|
|
5212
5392
|
return `Selected ${pages.length} page(s) from ${chunks.length} reranked chunks: ${summary}`;
|
|
5213
5393
|
}
|
|
5214
5394
|
function buildDebug(chunks, pageSlugs, reranked) {
|
|
@@ -5420,18 +5600,21 @@ async function watchCommand() {
|
|
|
5420
5600
|
let compiling = false;
|
|
5421
5601
|
let pendingRecompile = false;
|
|
5422
5602
|
let debounceTimer = null;
|
|
5423
|
-
const
|
|
5424
|
-
if (compiling) {
|
|
5425
|
-
pendingRecompile = true;
|
|
5426
|
-
return;
|
|
5427
|
-
}
|
|
5428
|
-
compiling = true;
|
|
5603
|
+
const runCompileOnce = async () => {
|
|
5429
5604
|
try {
|
|
5430
5605
|
await compile(process.cwd());
|
|
5431
5606
|
} catch (err) {
|
|
5432
5607
|
const msg = err instanceof Error ? err.message : String(err);
|
|
5433
5608
|
status("!", error(`Compile failed: ${msg}`));
|
|
5434
5609
|
}
|
|
5610
|
+
};
|
|
5611
|
+
const triggerCompile = async () => {
|
|
5612
|
+
if (compiling) {
|
|
5613
|
+
pendingRecompile = true;
|
|
5614
|
+
return;
|
|
5615
|
+
}
|
|
5616
|
+
compiling = true;
|
|
5617
|
+
await runCompileOnce();
|
|
5435
5618
|
compiling = false;
|
|
5436
5619
|
if (pendingRecompile) {
|
|
5437
5620
|
pendingRecompile = false;
|
|
@@ -5526,140 +5709,895 @@ async function lintCommand() {
|
|
|
5526
5709
|
}
|
|
5527
5710
|
}
|
|
5528
5711
|
|
|
5529
|
-
// src/
|
|
5712
|
+
// src/eval/health.ts
|
|
5713
|
+
var MAX_SCORE = 100;
|
|
5714
|
+
var ERROR_DEDUCTION = 4;
|
|
5715
|
+
var CONTRADICTED_DEDUCTION = 2;
|
|
5716
|
+
var DEFAULT_DEDUCTION = 1;
|
|
5717
|
+
var ERROR_RULES = /* @__PURE__ */ new Set([
|
|
5718
|
+
"broken-wikilink",
|
|
5719
|
+
"broken-citation",
|
|
5720
|
+
"duplicate-concept"
|
|
5721
|
+
]);
|
|
5722
|
+
function deductionFor(result) {
|
|
5723
|
+
if (ERROR_RULES.has(result.rule)) return ERROR_DEDUCTION;
|
|
5724
|
+
if (result.rule === "contradicted-page") return CONTRADICTED_DEDUCTION;
|
|
5725
|
+
return DEFAULT_DEDUCTION;
|
|
5726
|
+
}
|
|
5727
|
+
function aggregateRules(results) {
|
|
5728
|
+
const map = /* @__PURE__ */ new Map();
|
|
5729
|
+
for (const result of results) {
|
|
5730
|
+
const existing = map.get(result.rule);
|
|
5731
|
+
const deduction = deductionFor(result);
|
|
5732
|
+
if (existing) {
|
|
5733
|
+
existing.count++;
|
|
5734
|
+
existing.deduction += deduction;
|
|
5735
|
+
} else {
|
|
5736
|
+
map.set(result.rule, {
|
|
5737
|
+
rule: result.rule,
|
|
5738
|
+
count: 1,
|
|
5739
|
+
severity: result.severity,
|
|
5740
|
+
deduction
|
|
5741
|
+
});
|
|
5742
|
+
}
|
|
5743
|
+
}
|
|
5744
|
+
return Array.from(map.values());
|
|
5745
|
+
}
|
|
5746
|
+
async function evaluateHealth(root) {
|
|
5747
|
+
const schema = await loadSchema(root);
|
|
5748
|
+
const allResults = (await Promise.all([
|
|
5749
|
+
checkBrokenWikilinks(root),
|
|
5750
|
+
checkBrokenCitations(root),
|
|
5751
|
+
checkMalformedClaimCitations(root),
|
|
5752
|
+
checkOrphanedPages(root),
|
|
5753
|
+
checkMissingSummaries(root),
|
|
5754
|
+
checkDuplicateConcepts(root),
|
|
5755
|
+
checkEmptyPages(root),
|
|
5756
|
+
checkLowConfidencePages(root),
|
|
5757
|
+
checkContradictedPages(root),
|
|
5758
|
+
checkInferredWithoutCitations(root),
|
|
5759
|
+
checkSchemaCrossLinks(root, schema)
|
|
5760
|
+
])).flat();
|
|
5761
|
+
const rules = aggregateRules(allResults);
|
|
5762
|
+
const totalDeduction = rules.reduce((sum, r) => sum + r.deduction, 0);
|
|
5763
|
+
const score = Math.max(0, MAX_SCORE - totalDeduction);
|
|
5764
|
+
return { score, maxScore: MAX_SCORE, rules };
|
|
5765
|
+
}
|
|
5766
|
+
|
|
5767
|
+
// src/eval/citation-coverage.ts
|
|
5768
|
+
import path37 from "path";
|
|
5769
|
+
|
|
5770
|
+
// src/eval/source-path.ts
|
|
5771
|
+
import { realpath as realpath4 } from "fs/promises";
|
|
5530
5772
|
import path36 from "path";
|
|
5531
|
-
|
|
5773
|
+
function containsParentSegment(file) {
|
|
5774
|
+
return file.split(/[/\\]/).some((seg) => seg === "..");
|
|
5775
|
+
}
|
|
5776
|
+
function isInside(parent, candidate) {
|
|
5777
|
+
if (candidate === parent) return true;
|
|
5778
|
+
const parentWithSep = parent.endsWith(path36.sep) ? parent : parent + path36.sep;
|
|
5779
|
+
return candidate.startsWith(parentWithSep);
|
|
5780
|
+
}
|
|
5781
|
+
async function resolveSourceFile(sourcesDir, file) {
|
|
5782
|
+
if (file.length === 0 || path36.isAbsolute(file)) return null;
|
|
5783
|
+
if (containsParentSegment(file)) return null;
|
|
5784
|
+
const joined = path36.join(sourcesDir, file);
|
|
5785
|
+
if (!isInside(sourcesDir, path36.resolve(joined))) return null;
|
|
5786
|
+
try {
|
|
5787
|
+
const realDir = await realpath4(sourcesDir);
|
|
5788
|
+
const realFile = await realpath4(joined);
|
|
5789
|
+
if (!isInside(realDir, realFile)) return null;
|
|
5790
|
+
return realFile;
|
|
5791
|
+
} catch {
|
|
5792
|
+
return null;
|
|
5793
|
+
}
|
|
5794
|
+
}
|
|
5532
5795
|
|
|
5533
|
-
// src/
|
|
5534
|
-
|
|
5535
|
-
|
|
5796
|
+
// src/eval/citation-coverage.ts
|
|
5797
|
+
var PROSE_LEAD_RE = new RegExp("^\\p{L}", "u");
|
|
5798
|
+
async function evaluatePage(slug, body, sourcesDir) {
|
|
5799
|
+
const paragraphs = body.split(/\n\s*\n/).filter((p) => PROSE_LEAD_RE.test(p.trim()));
|
|
5800
|
+
let citedParagraphs = 0;
|
|
5801
|
+
let totalCitations = 0;
|
|
5802
|
+
let validCitations = 0;
|
|
5803
|
+
for (const para of paragraphs) {
|
|
5804
|
+
const citations = extractClaimCitations(para);
|
|
5805
|
+
if (citations.length === 0) continue;
|
|
5806
|
+
citedParagraphs++;
|
|
5807
|
+
for (const { spans } of citations) {
|
|
5808
|
+
for (const span of spans) {
|
|
5809
|
+
totalCitations++;
|
|
5810
|
+
if (await resolveSourceFile(sourcesDir, span.file) !== null) validCitations++;
|
|
5811
|
+
}
|
|
5812
|
+
}
|
|
5813
|
+
}
|
|
5536
5814
|
return {
|
|
5537
|
-
|
|
5538
|
-
|
|
5539
|
-
|
|
5540
|
-
|
|
5541
|
-
|
|
5542
|
-
tags: Array.isArray(meta.tags) ? meta.tags.filter((t) => typeof t === "string") : [],
|
|
5543
|
-
createdAt: typeof meta.createdAt === "string" ? meta.createdAt : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5544
|
-
updatedAt: typeof meta.updatedAt === "string" ? meta.updatedAt : (/* @__PURE__ */ new Date()).toISOString(),
|
|
5545
|
-
links: extractWikilinkSlugs(raw.body),
|
|
5546
|
-
body: raw.body
|
|
5815
|
+
pageResult: { slug, proseParagraphs: paragraphs.length, citedParagraphs },
|
|
5816
|
+
proseParagraphs: paragraphs.length,
|
|
5817
|
+
citedParagraphs,
|
|
5818
|
+
totalCitations,
|
|
5819
|
+
validCitations
|
|
5547
5820
|
};
|
|
5548
5821
|
}
|
|
5549
|
-
async function
|
|
5550
|
-
const
|
|
5551
|
-
const
|
|
5552
|
-
|
|
5553
|
-
|
|
5554
|
-
|
|
5822
|
+
async function evaluateCitationCoverage(root) {
|
|
5823
|
+
const pages = await collectAllPages(root);
|
|
5824
|
+
const sourcesDir = path37.join(root, SOURCES_DIR);
|
|
5825
|
+
let totalProse = 0;
|
|
5826
|
+
let totalCited = 0;
|
|
5827
|
+
let totalCitations = 0;
|
|
5828
|
+
let totalValid = 0;
|
|
5829
|
+
const perPage = [];
|
|
5830
|
+
for (const { filePath, content } of pages) {
|
|
5831
|
+
const { body } = parseFrontmatter(content);
|
|
5832
|
+
const slug = path37.basename(filePath, ".md");
|
|
5833
|
+
const stats = await evaluatePage(slug, body, sourcesDir);
|
|
5834
|
+
totalProse += stats.proseParagraphs;
|
|
5835
|
+
totalCited += stats.citedParagraphs;
|
|
5836
|
+
totalCitations += stats.totalCitations;
|
|
5837
|
+
totalValid += stats.validCitations;
|
|
5838
|
+
perPage.push(stats.pageResult);
|
|
5839
|
+
}
|
|
5840
|
+
const coveragePercent = totalProse === 0 ? 0 : totalCited / totalProse * 100;
|
|
5841
|
+
const precisionPercent = totalCitations === 0 ? 0 : totalValid / totalCitations * 100;
|
|
5842
|
+
return {
|
|
5843
|
+
totalProseParagraphs: totalProse,
|
|
5844
|
+
citedParagraphs: totalCited,
|
|
5845
|
+
coveragePercent,
|
|
5846
|
+
totalCitations,
|
|
5847
|
+
validCitations: totalValid,
|
|
5848
|
+
precisionPercent,
|
|
5849
|
+
perPage
|
|
5850
|
+
};
|
|
5555
5851
|
}
|
|
5556
5852
|
|
|
5557
|
-
// src/
|
|
5558
|
-
|
|
5559
|
-
|
|
5560
|
-
}
|
|
5561
|
-
|
|
5562
|
-
|
|
5563
|
-
|
|
5564
|
-
|
|
5565
|
-
|
|
5566
|
-
|
|
5567
|
-
|
|
5568
|
-
|
|
5853
|
+
// src/eval/citation-support.ts
|
|
5854
|
+
import { createHash as createHash4 } from "crypto";
|
|
5855
|
+
import { readFile as readFile24, appendFile, mkdir as mkdir7 } from "fs/promises";
|
|
5856
|
+
import { existsSync as existsSync10 } from "fs";
|
|
5857
|
+
import path38 from "path";
|
|
5858
|
+
var CACHE_DIR = path38.join(".llmwiki", "eval");
|
|
5859
|
+
var CACHE_FILE = path38.join(CACHE_DIR, "citation-cache.jsonl");
|
|
5860
|
+
var PROSE_LEAD_RE2 = new RegExp("^\\p{L}", "u");
|
|
5861
|
+
var JUDGE_TOOL = {
|
|
5862
|
+
name: "judge_citation",
|
|
5863
|
+
description: "Rate how well the source excerpt supports the claim.",
|
|
5864
|
+
input_schema: {
|
|
5865
|
+
type: "object",
|
|
5866
|
+
properties: {
|
|
5867
|
+
score: {
|
|
5868
|
+
type: "integer",
|
|
5869
|
+
enum: [0, 1, 2],
|
|
5870
|
+
description: "0=not supported or contradicted, 1=partially supported, 2=fully supported"
|
|
5871
|
+
},
|
|
5872
|
+
reason: { type: "string", description: "One sentence explaining the rating." }
|
|
5873
|
+
},
|
|
5874
|
+
required: ["score", "reason"]
|
|
5875
|
+
}
|
|
5876
|
+
};
|
|
5877
|
+
var JUDGE_SYSTEM = "You are an expert fact-checker. Given a claim from a wiki article and a source excerpt, rate whether the source supports the claim. Be strict: partial credit only if the source addresses the claim but is incomplete. Important: if the source excerpt consists entirely of YAML frontmatter (metadata fields such as title, date, author, or tags between --- delimiters), treat it as non-evidence for any substantive claim and score it 0, unless the claim is explicitly about that metadata field.";
|
|
5878
|
+
var JUDGE_CONFIG_HASH = createHash4("sha256").update(JUDGE_SYSTEM + JSON.stringify(JUDGE_TOOL)).digest("hex").slice(0, 8);
|
|
5879
|
+
function hashPair(claimText, spanText) {
|
|
5880
|
+
return createHash4("sha256").update(claimText + spanText).digest("hex").slice(0, 16);
|
|
5881
|
+
}
|
|
5882
|
+
function makeCacheKey(contentHash, model) {
|
|
5883
|
+
return createHash4("sha256").update(contentHash + JUDGE_CONFIG_HASH + model).digest("hex").slice(0, 16);
|
|
5884
|
+
}
|
|
5885
|
+
async function readSourceLines(filePath, start, end) {
|
|
5886
|
+
const content = await readFile24(filePath, "utf-8");
|
|
5887
|
+
return content.split("\n").slice(start - 1, end).join("\n");
|
|
5888
|
+
}
|
|
5889
|
+
function stripCitationMarkers(paragraph) {
|
|
5890
|
+
return paragraph.replace(/\^\[[^\]]+\]/g, "").trim();
|
|
5891
|
+
}
|
|
5892
|
+
async function buildSpanPair(slug, claimText, span, sourcesDir) {
|
|
5893
|
+
if (!span.lines) return null;
|
|
5894
|
+
const sourceFile = await resolveSourceFile(sourcesDir, span.file);
|
|
5895
|
+
if (sourceFile === null) return null;
|
|
5896
|
+
const spanText = await readSourceLines(sourceFile, span.lines.start, span.lines.end);
|
|
5897
|
+
return {
|
|
5898
|
+
claimHash: hashPair(claimText, spanText),
|
|
5899
|
+
pageSlug: slug,
|
|
5900
|
+
claimText,
|
|
5901
|
+
citedFile: span.file,
|
|
5902
|
+
spanText,
|
|
5903
|
+
lineStart: span.lines.start,
|
|
5904
|
+
lineEnd: span.lines.end
|
|
5905
|
+
};
|
|
5569
5906
|
}
|
|
5570
|
-
function
|
|
5571
|
-
const
|
|
5572
|
-
return
|
|
5907
|
+
async function extractParagraphPairs(slug, para, sourcesDir) {
|
|
5908
|
+
const citations = extractClaimCitations(para);
|
|
5909
|
+
if (citations.length === 0) return [];
|
|
5910
|
+
const claimText = stripCitationMarkers(para);
|
|
5911
|
+
const spans = citations.flatMap((c) => c.spans);
|
|
5912
|
+
const pairs = await Promise.all(spans.map((s) => buildSpanPair(slug, claimText, s, sourcesDir)));
|
|
5913
|
+
return pairs.filter((p) => p !== null);
|
|
5573
5914
|
}
|
|
5574
|
-
function
|
|
5575
|
-
|
|
5576
|
-
|
|
5915
|
+
async function extractPagePairs(slug, body, sourcesDir) {
|
|
5916
|
+
const paragraphs = body.split(/\n\s*\n/).filter((p) => PROSE_LEAD_RE2.test(p.trim()));
|
|
5917
|
+
const results = await Promise.all(paragraphs.map((p) => extractParagraphPairs(slug, p, sourcesDir)));
|
|
5918
|
+
return results.flat();
|
|
5577
5919
|
}
|
|
5578
|
-
function
|
|
5579
|
-
const
|
|
5580
|
-
const
|
|
5581
|
-
const
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
...
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
return lines.join("\n");
|
|
5920
|
+
async function extractCitationPairs(root) {
|
|
5921
|
+
const pages = await collectAllPages(root);
|
|
5922
|
+
const sourcesDir = path38.join(root, SOURCES_DIR);
|
|
5923
|
+
const all = [];
|
|
5924
|
+
for (const { filePath, content } of pages) {
|
|
5925
|
+
const { body } = parseFrontmatter(content);
|
|
5926
|
+
const slug = path38.basename(filePath, ".md");
|
|
5927
|
+
const pairs = await extractPagePairs(slug, body, sourcesDir);
|
|
5928
|
+
all.push(...pairs);
|
|
5929
|
+
}
|
|
5930
|
+
return all;
|
|
5590
5931
|
}
|
|
5591
|
-
function
|
|
5592
|
-
const
|
|
5593
|
-
|
|
5594
|
-
const
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
|
|
5607
|
-
|
|
5932
|
+
function selectDeterministicSample(pairs, sampleSize, previousHashes = []) {
|
|
5933
|
+
const pairByHash = new Map(pairs.map((p) => [p.claimHash, p]));
|
|
5934
|
+
const retained = previousHashes.flatMap((h) => {
|
|
5935
|
+
const p = pairByHash.get(h);
|
|
5936
|
+
return p ? [p] : [];
|
|
5937
|
+
});
|
|
5938
|
+
if (retained.length >= sampleSize) return retained.slice(0, sampleSize);
|
|
5939
|
+
const retainedSet = new Set(previousHashes);
|
|
5940
|
+
const newPairs = pairs.filter((p) => !retainedSet.has(p.claimHash)).sort((a, b) => a.claimHash.localeCompare(b.claimHash));
|
|
5941
|
+
return [...retained, ...newPairs].slice(0, sampleSize);
|
|
5942
|
+
}
|
|
5943
|
+
async function loadCachedJudgements(root) {
|
|
5944
|
+
const cachePath = path38.join(root, CACHE_FILE);
|
|
5945
|
+
if (!existsSync10(cachePath)) return /* @__PURE__ */ new Map();
|
|
5946
|
+
const content = await readFile24(cachePath, "utf-8");
|
|
5947
|
+
const map = /* @__PURE__ */ new Map();
|
|
5948
|
+
for (const line2 of content.trim().split("\n").filter(Boolean)) {
|
|
5949
|
+
try {
|
|
5950
|
+
const entry = JSON.parse(line2);
|
|
5951
|
+
map.set(entry.claimHash, entry);
|
|
5952
|
+
} catch {
|
|
5953
|
+
}
|
|
5608
5954
|
}
|
|
5609
|
-
return
|
|
5955
|
+
return map;
|
|
5610
5956
|
}
|
|
5611
|
-
|
|
5612
|
-
|
|
5613
|
-
|
|
5614
|
-
const doc = {
|
|
5615
|
-
exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5616
|
-
pageCount: pages.length,
|
|
5617
|
-
pages
|
|
5618
|
-
};
|
|
5619
|
-
return JSON.stringify(doc, null, 2);
|
|
5957
|
+
async function appendCachedJudgement(root, judgement) {
|
|
5958
|
+
await mkdir7(path38.join(root, CACHE_DIR), { recursive: true });
|
|
5959
|
+
await appendFile(path38.join(root, CACHE_FILE), JSON.stringify(judgement) + "\n");
|
|
5620
5960
|
}
|
|
5621
|
-
|
|
5622
|
-
|
|
5623
|
-
|
|
5624
|
-
function pageIri(slug) {
|
|
5625
|
-
return `${LOCAL_BASE}${slug}`;
|
|
5961
|
+
function resolveModel() {
|
|
5962
|
+
const provider = process.env.LLMWIKI_PROVIDER ?? DEFAULT_PROVIDER;
|
|
5963
|
+
return process.env.LLMWIKI_MODEL ?? PROVIDER_MODELS[provider] ?? provider;
|
|
5626
5964
|
}
|
|
5627
|
-
function
|
|
5628
|
-
const
|
|
5629
|
-
|
|
5630
|
-
|
|
5631
|
-
|
|
5632
|
-
|
|
5633
|
-
|
|
5634
|
-
|
|
5965
|
+
async function callJudge(pair, cacheKey, model) {
|
|
5966
|
+
const userMessage = `Claim: ${pair.claimText}
|
|
5967
|
+
|
|
5968
|
+
Source (${pair.citedFile}, lines ${pair.lineStart}\u2013${pair.lineEnd}):
|
|
5969
|
+
${pair.spanText}`;
|
|
5970
|
+
const raw = await callClaude({
|
|
5971
|
+
system: JUDGE_SYSTEM,
|
|
5972
|
+
messages: [{ role: "user", content: userMessage }],
|
|
5973
|
+
tools: [JUDGE_TOOL],
|
|
5974
|
+
maxTokens: 256
|
|
5975
|
+
});
|
|
5976
|
+
const parsed = JSON.parse(raw);
|
|
5977
|
+
return {
|
|
5978
|
+
claimHash: cacheKey,
|
|
5979
|
+
pageSlug: pair.pageSlug,
|
|
5980
|
+
citedFile: pair.citedFile,
|
|
5981
|
+
lineStart: pair.lineStart,
|
|
5982
|
+
lineEnd: pair.lineEnd,
|
|
5983
|
+
claimText: pair.claimText,
|
|
5984
|
+
spanText: pair.spanText,
|
|
5985
|
+
score: parsed.score,
|
|
5986
|
+
reason: parsed.reason,
|
|
5987
|
+
model,
|
|
5988
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
5635
5989
|
};
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
|
|
5640
|
-
|
|
5990
|
+
}
|
|
5991
|
+
function aggregateJudgements(judgements) {
|
|
5992
|
+
const fullySupported = judgements.filter((j) => j.score === 2).length;
|
|
5993
|
+
const partiallySupported = judgements.filter((j) => j.score === 1).length;
|
|
5994
|
+
const unsupported = judgements.filter((j) => j.score === 0).length;
|
|
5995
|
+
const meanScore = judgements.length === 0 ? 0 : judgements.reduce((sum, j) => sum + j.score, 0) / judgements.length;
|
|
5996
|
+
return { meanScore, fullySupported, partiallySupported, unsupported };
|
|
5997
|
+
}
|
|
5998
|
+
async function judgeNewPairs(sample, cache, root) {
|
|
5999
|
+
const model = resolveModel();
|
|
6000
|
+
const judgements = [];
|
|
6001
|
+
let judgeErrors = 0;
|
|
6002
|
+
let newPairsAttempted = 0;
|
|
6003
|
+
let firstError;
|
|
6004
|
+
for (const pair of sample) {
|
|
6005
|
+
const cacheKey = makeCacheKey(pair.claimHash, model);
|
|
6006
|
+
const cached = cache.get(cacheKey);
|
|
6007
|
+
if (cached) {
|
|
6008
|
+
judgements.push(cached);
|
|
6009
|
+
} else {
|
|
6010
|
+
newPairsAttempted++;
|
|
6011
|
+
try {
|
|
6012
|
+
const judgement = await callJudge(pair, cacheKey, model);
|
|
6013
|
+
await appendCachedJudgement(root, judgement);
|
|
6014
|
+
judgements.push(judgement);
|
|
6015
|
+
} catch (err) {
|
|
6016
|
+
judgeErrors++;
|
|
6017
|
+
if (firstError === void 0) firstError = err;
|
|
6018
|
+
}
|
|
6019
|
+
}
|
|
5641
6020
|
}
|
|
5642
|
-
if (
|
|
5643
|
-
|
|
6021
|
+
if (newPairsAttempted > 0 && judgeErrors === newPairsAttempted) {
|
|
6022
|
+
const msg = firstError instanceof Error ? firstError.message : String(firstError);
|
|
6023
|
+
throw new Error(`Citation judge failed for all ${judgeErrors} sampled pair(s): ${msg}`);
|
|
5644
6024
|
}
|
|
5645
|
-
return
|
|
6025
|
+
return { judgements, judgeErrors };
|
|
5646
6026
|
}
|
|
5647
|
-
function
|
|
5648
|
-
const
|
|
5649
|
-
|
|
5650
|
-
|
|
6027
|
+
async function evaluateCitationSupport(root, sampleSize = 20, previousHashes = []) {
|
|
6028
|
+
const allPairs = await extractCitationPairs(root);
|
|
6029
|
+
if (allPairs.length === 0) return null;
|
|
6030
|
+
const sample = selectDeterministicSample(allPairs, sampleSize, previousHashes);
|
|
6031
|
+
const cache = await loadCachedJudgements(root);
|
|
6032
|
+
const { judgements, judgeErrors } = await judgeNewPairs(sample, cache, root);
|
|
6033
|
+
return {
|
|
6034
|
+
sampledCount: judgements.length,
|
|
6035
|
+
sampledHashes: sample.map((p) => p.claimHash),
|
|
6036
|
+
totalCitations: allPairs.length,
|
|
6037
|
+
judgeErrors,
|
|
6038
|
+
...aggregateJudgements(judgements),
|
|
6039
|
+
judgements
|
|
5651
6040
|
};
|
|
5652
|
-
return JSON.stringify(doc, null, 2);
|
|
5653
6041
|
}
|
|
5654
6042
|
|
|
5655
|
-
// src/
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
5659
|
-
|
|
5660
|
-
|
|
5661
|
-
|
|
5662
|
-
|
|
6043
|
+
// src/eval/stats.ts
|
|
6044
|
+
import { readdir as readdir12, appendFile as appendFile2, mkdir as mkdir8, readFile as readFile25 } from "fs/promises";
|
|
6045
|
+
import { existsSync as existsSync11 } from "fs";
|
|
6046
|
+
import path39 from "path";
|
|
6047
|
+
var HISTORY_DIR = path39.join(".llmwiki", "eval");
|
|
6048
|
+
var HISTORY_FILE = path39.join(HISTORY_DIR, "history.jsonl");
|
|
6049
|
+
async function countFiles(dir) {
|
|
6050
|
+
if (!existsSync11(dir)) return 0;
|
|
6051
|
+
const entries = await readdir12(dir);
|
|
6052
|
+
return entries.filter((e) => e.endsWith(".md")).length;
|
|
6053
|
+
}
|
|
6054
|
+
async function collectStats(root) {
|
|
6055
|
+
const [sourceCount, pages, embeddingStore] = await Promise.all([
|
|
6056
|
+
countFiles(path39.join(root, SOURCES_DIR)),
|
|
6057
|
+
collectAllPages(root),
|
|
6058
|
+
readEmbeddingStore(root)
|
|
6059
|
+
]);
|
|
6060
|
+
let totalWikiChars = 0;
|
|
6061
|
+
for (const { content } of pages) {
|
|
6062
|
+
const { body } = parseFrontmatter(content);
|
|
6063
|
+
totalWikiChars += body.length;
|
|
6064
|
+
}
|
|
6065
|
+
const pageCount = pages.length;
|
|
6066
|
+
const avgPageLengthChars = pageCount === 0 ? 0 : Math.round(totalWikiChars / pageCount);
|
|
6067
|
+
const embeddingCount = embeddingStore?.entries.length ?? 0;
|
|
6068
|
+
const chunkEmbeddingCount = embeddingStore?.chunks?.length ?? 0;
|
|
6069
|
+
return {
|
|
6070
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6071
|
+
sourceCount,
|
|
6072
|
+
pageCount,
|
|
6073
|
+
totalWikiChars,
|
|
6074
|
+
embeddingCount,
|
|
6075
|
+
chunkEmbeddingCount,
|
|
6076
|
+
avgPageLengthChars
|
|
6077
|
+
};
|
|
6078
|
+
}
|
|
6079
|
+
async function appendHistory(root, report) {
|
|
6080
|
+
const historyDir = path39.join(root, HISTORY_DIR);
|
|
6081
|
+
await mkdir8(historyDir, { recursive: true });
|
|
6082
|
+
await appendFile2(path39.join(root, HISTORY_FILE), JSON.stringify(report) + "\n");
|
|
6083
|
+
}
|
|
6084
|
+
async function loadHistory(root, n = 10) {
|
|
6085
|
+
const historyPath = path39.join(root, HISTORY_FILE);
|
|
6086
|
+
if (!existsSync11(historyPath)) return [];
|
|
6087
|
+
const content = await readFile25(historyPath, "utf-8");
|
|
6088
|
+
const lines = content.trim().split("\n").filter(Boolean);
|
|
6089
|
+
const reports = [];
|
|
6090
|
+
for (const line2 of lines.slice(-n)) {
|
|
6091
|
+
try {
|
|
6092
|
+
reports.push(JSON.parse(line2));
|
|
6093
|
+
} catch {
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
6096
|
+
return reports;
|
|
6097
|
+
}
|
|
6098
|
+
async function loadPreviousReport(root) {
|
|
6099
|
+
const historyPath = path39.join(root, HISTORY_FILE);
|
|
6100
|
+
if (!existsSync11(historyPath)) return null;
|
|
6101
|
+
const content = await readFile25(historyPath, "utf-8");
|
|
6102
|
+
const lines = content.trim().split("\n").filter(Boolean);
|
|
6103
|
+
if (lines.length === 0) return null;
|
|
6104
|
+
try {
|
|
6105
|
+
return JSON.parse(lines[lines.length - 1]);
|
|
6106
|
+
} catch {
|
|
6107
|
+
return null;
|
|
6108
|
+
}
|
|
6109
|
+
}
|
|
6110
|
+
|
|
6111
|
+
// src/eval/delta.ts
|
|
6112
|
+
function computeDelta(current, previous) {
|
|
6113
|
+
const delta = {
|
|
6114
|
+
healthScore: current.health.score - previous.health.score,
|
|
6115
|
+
citationCoveragePercent: current.citationCoverage.coveragePercent - previous.citationCoverage.coveragePercent,
|
|
6116
|
+
citationPrecisionPercent: current.citationCoverage.precisionPercent - previous.citationCoverage.precisionPercent
|
|
6117
|
+
};
|
|
6118
|
+
if (current.citationSupport !== void 0 && previous.citationSupport !== void 0) {
|
|
6119
|
+
delta.citationSupportMean = current.citationSupport.meanScore - previous.citationSupport.meanScore;
|
|
6120
|
+
}
|
|
6121
|
+
return delta;
|
|
6122
|
+
}
|
|
6123
|
+
|
|
6124
|
+
// src/eval/thresholds.ts
|
|
6125
|
+
import { readFile as readFile26 } from "fs/promises";
|
|
6126
|
+
import { existsSync as existsSync12 } from "fs";
|
|
6127
|
+
import path40 from "path";
|
|
6128
|
+
import yaml4 from "js-yaml";
|
|
6129
|
+
var THRESHOLDS_FILE = path40.join(".llmwiki", "eval", "thresholds.yaml");
|
|
6130
|
+
async function loadThresholds(root) {
|
|
6131
|
+
const configPath = path40.join(root, THRESHOLDS_FILE);
|
|
6132
|
+
if (!existsSync12(configPath)) return {};
|
|
6133
|
+
const raw = await readFile26(configPath, "utf-8");
|
|
6134
|
+
return yaml4.load(raw) ?? {};
|
|
6135
|
+
}
|
|
6136
|
+
async function checkThresholds(report, root) {
|
|
6137
|
+
const config = await loadThresholds(root);
|
|
6138
|
+
const violations = [];
|
|
6139
|
+
if (config.health_score !== void 0 && report.health.score < config.health_score) {
|
|
6140
|
+
violations.push(
|
|
6141
|
+
`health_score ${report.health.score} is below threshold ${config.health_score}`
|
|
6142
|
+
);
|
|
6143
|
+
}
|
|
6144
|
+
if (config.citation_coverage_percent !== void 0 && report.citationCoverage.coveragePercent < config.citation_coverage_percent) {
|
|
6145
|
+
violations.push(
|
|
6146
|
+
`citation_coverage_percent ${report.citationCoverage.coveragePercent.toFixed(1)}% is below threshold ${config.citation_coverage_percent}%`
|
|
6147
|
+
);
|
|
6148
|
+
}
|
|
6149
|
+
if (config.citation_precision_percent !== void 0 && report.citationCoverage.precisionPercent < config.citation_precision_percent) {
|
|
6150
|
+
violations.push(
|
|
6151
|
+
`citation_precision_percent ${report.citationCoverage.precisionPercent.toFixed(1)}% is below threshold ${config.citation_precision_percent}%`
|
|
6152
|
+
);
|
|
6153
|
+
}
|
|
6154
|
+
if (config.citation_support_mean !== void 0 && report.citationSupport !== void 0 && report.citationSupport.meanScore < config.citation_support_mean) {
|
|
6155
|
+
violations.push(
|
|
6156
|
+
`citation_support_mean ${report.citationSupport.meanScore.toFixed(2)} is below threshold ${config.citation_support_mean}`
|
|
6157
|
+
);
|
|
6158
|
+
}
|
|
6159
|
+
if (config.citation_judge_error_max !== void 0 && report.citationSupport !== void 0 && report.citationSupport.judgeErrors > config.citation_judge_error_max) {
|
|
6160
|
+
violations.push(
|
|
6161
|
+
`citation_judge_errors ${report.citationSupport.judgeErrors} exceeds max ${config.citation_judge_error_max}`
|
|
6162
|
+
);
|
|
6163
|
+
}
|
|
6164
|
+
return violations;
|
|
6165
|
+
}
|
|
6166
|
+
|
|
6167
|
+
// src/eval/report.ts
|
|
6168
|
+
var BOX_WIDTH = 49;
|
|
6169
|
+
var HORIZONTAL = "\u2500".repeat(BOX_WIDTH);
|
|
6170
|
+
function line(content = "") {
|
|
6171
|
+
return `\u2502 ${content.padEnd(BOX_WIDTH - 2)} \u2502`;
|
|
6172
|
+
}
|
|
6173
|
+
function top() {
|
|
6174
|
+
return `\u250C${HORIZONTAL}\u2510`;
|
|
6175
|
+
}
|
|
6176
|
+
function divider() {
|
|
6177
|
+
return `\u251C${HORIZONTAL}\u2524`;
|
|
6178
|
+
}
|
|
6179
|
+
function bottom() {
|
|
6180
|
+
return `\u2514${HORIZONTAL}\u2518`;
|
|
6181
|
+
}
|
|
6182
|
+
function fmtDelta(value) {
|
|
6183
|
+
if (value === void 0 || value === 0) return "";
|
|
6184
|
+
const abs = Math.abs(value).toFixed(1).replace(/\.0$/, "");
|
|
6185
|
+
return value > 0 ? dim(` (\u2191${abs})`) : dim(` (\u2193${abs})`);
|
|
6186
|
+
}
|
|
6187
|
+
function ruleRow(rule) {
|
|
6188
|
+
if (rule.count === 0) return "";
|
|
6189
|
+
const label = ` ${rule.rule}:`;
|
|
6190
|
+
const right = `${rule.count} (\u2212${rule.deduction})`;
|
|
6191
|
+
const gap = BOX_WIDTH - 4 - label.length - right.length;
|
|
6192
|
+
return line(`${label}${" ".repeat(Math.max(1, gap))}${right}`);
|
|
6193
|
+
}
|
|
6194
|
+
function formatHealth(report, delta) {
|
|
6195
|
+
const scoreDelta = fmtDelta(delta?.healthScore);
|
|
6196
|
+
const rows = [
|
|
6197
|
+
line(),
|
|
6198
|
+
line(bold(`Structural Health: ${report.health.score} / 100${scoreDelta}`))
|
|
6199
|
+
];
|
|
6200
|
+
for (const rule of report.health.rules) {
|
|
6201
|
+
const row = ruleRow(rule);
|
|
6202
|
+
if (row) rows.push(row);
|
|
6203
|
+
}
|
|
6204
|
+
return rows;
|
|
6205
|
+
}
|
|
6206
|
+
function formatCoverage(report, delta) {
|
|
6207
|
+
const cov = report.citationCoverage;
|
|
6208
|
+
const covDelta = fmtDelta(delta?.citationCoveragePercent);
|
|
6209
|
+
const precDelta = fmtDelta(delta?.citationPrecisionPercent);
|
|
6210
|
+
return [
|
|
6211
|
+
line(),
|
|
6212
|
+
line(bold(`Citation Coverage: ${cov.coveragePercent.toFixed(0)}%${covDelta}`)),
|
|
6213
|
+
line(` ${cov.citedParagraphs} / ${cov.totalProseParagraphs} prose paragraphs cited`),
|
|
6214
|
+
line(
|
|
6215
|
+
` Precision: ${cov.precisionPercent.toFixed(0)}%${precDelta} (${cov.validCitations}/${cov.totalCitations} valid)`
|
|
6216
|
+
)
|
|
6217
|
+
];
|
|
6218
|
+
}
|
|
6219
|
+
function formatSupport(report, delta) {
|
|
6220
|
+
const s = report.citationSupport;
|
|
6221
|
+
if (!s) return [];
|
|
6222
|
+
const meanDelta = fmtDelta(delta?.citationSupportMean);
|
|
6223
|
+
const pctOf = (n) => s.sampledCount === 0 ? "\u2014" : `${(n / s.sampledCount * 100).toFixed(0)}%`;
|
|
6224
|
+
const rows = [
|
|
6225
|
+
line(),
|
|
6226
|
+
line(bold(`Citation Support (${s.sampledCount} sampled):`)),
|
|
6227
|
+
line(` Mean score: ${s.meanScore.toFixed(2)} / 2.0${meanDelta}`),
|
|
6228
|
+
line(` Fully supported: ${s.fullySupported} (${pctOf(s.fullySupported)})`),
|
|
6229
|
+
line(` Partially supported: ${s.partiallySupported} (${pctOf(s.partiallySupported)})`),
|
|
6230
|
+
line(` Unsupported: ${s.unsupported} (${pctOf(s.unsupported)})`)
|
|
6231
|
+
];
|
|
6232
|
+
if (s.judgeErrors > 0) {
|
|
6233
|
+
rows.push(line(error(` Judge errors: ${s.judgeErrors}`)));
|
|
6234
|
+
}
|
|
6235
|
+
return rows;
|
|
6236
|
+
}
|
|
6237
|
+
function formatStats(report) {
|
|
6238
|
+
const s = report.stats;
|
|
6239
|
+
return [
|
|
6240
|
+
line(),
|
|
6241
|
+
line(bold("Scale:")),
|
|
6242
|
+
line(
|
|
6243
|
+
` Sources: ${s.sourceCount} Pages: ${s.pageCount} Chunks: ${s.chunkEmbeddingCount}`
|
|
6244
|
+
),
|
|
6245
|
+
line(` Wiki size: ${s.totalWikiChars.toLocaleString()} chars`)
|
|
6246
|
+
];
|
|
6247
|
+
}
|
|
6248
|
+
function formatViolations(violations) {
|
|
6249
|
+
if (violations.length === 0) return [];
|
|
6250
|
+
return [line(), ...violations.map((v) => line(error(`[FAIL] ${v}`)))];
|
|
6251
|
+
}
|
|
6252
|
+
function formatTerminalReport(report) {
|
|
6253
|
+
const delta = report.delta;
|
|
6254
|
+
const rows = [
|
|
6255
|
+
top(),
|
|
6256
|
+
line(bold("llmwiki eval \u2014 Wiki Quality Report")),
|
|
6257
|
+
divider(),
|
|
6258
|
+
...formatHealth(report, delta),
|
|
6259
|
+
...formatCoverage(report, delta),
|
|
6260
|
+
...formatSupport(report, delta),
|
|
6261
|
+
...formatStats(report),
|
|
6262
|
+
...formatViolations(report.thresholdViolations),
|
|
6263
|
+
line(),
|
|
6264
|
+
bottom()
|
|
6265
|
+
];
|
|
6266
|
+
return rows.join("\n");
|
|
6267
|
+
}
|
|
6268
|
+
function formatJsonReport(report) {
|
|
6269
|
+
return JSON.stringify(report, null, 2);
|
|
6270
|
+
}
|
|
6271
|
+
function fmtTimestamp(iso) {
|
|
6272
|
+
return iso.slice(0, 16).replace("T", " ");
|
|
6273
|
+
}
|
|
6274
|
+
function formatHistoryTable(reports) {
|
|
6275
|
+
if (reports.length === 0) return "No eval history found. Run `llmwiki eval` to record the first run.";
|
|
6276
|
+
const header2 = `${"Date".padEnd(18)}${"Suite".padEnd(7)}${"Health".padEnd(8)}${"Coverage".padEnd(10)}Support`;
|
|
6277
|
+
const divider2 = "\u2500".repeat(header2.length);
|
|
6278
|
+
const rows = reports.map((r) => {
|
|
6279
|
+
const support = r.citationSupport ? r.citationSupport.meanScore.toFixed(2) : "\u2014";
|
|
6280
|
+
return [
|
|
6281
|
+
fmtTimestamp(r.timestamp).padEnd(18),
|
|
6282
|
+
r.suite.padEnd(7),
|
|
6283
|
+
String(r.health.score).padEnd(8),
|
|
6284
|
+
`${r.citationCoverage.coveragePercent.toFixed(0)}%`.padEnd(10),
|
|
6285
|
+
support
|
|
6286
|
+
].join("");
|
|
6287
|
+
});
|
|
6288
|
+
return [`Eval History (${reports.length} run${reports.length === 1 ? "" : "s"})`, divider2, header2, divider2, ...rows].join("\n");
|
|
6289
|
+
}
|
|
6290
|
+
var SCORE_LABELS = {
|
|
6291
|
+
2: "fully supported",
|
|
6292
|
+
1: "partially supported",
|
|
6293
|
+
0: "unsupported"
|
|
6294
|
+
};
|
|
6295
|
+
function pct(n, total) {
|
|
6296
|
+
return total === 0 ? "0%" : `${(n / total * 100).toFixed(0)}%`;
|
|
6297
|
+
}
|
|
6298
|
+
function formatCacheShow(judgements, summary) {
|
|
6299
|
+
const lines = [bold(`Citation Cache \xB7 ${summary.total} judgements`)];
|
|
6300
|
+
if (summary.total === 0) return lines.join("\n");
|
|
6301
|
+
lines.push("");
|
|
6302
|
+
lines.push(` Score 2 (fully supported): ${summary.fullySupported} (${pct(summary.fullySupported, summary.total)})`);
|
|
6303
|
+
lines.push(` Score 1 (partially supported): ${summary.partiallySupported} (${pct(summary.partiallySupported, summary.total)})`);
|
|
6304
|
+
lines.push(` Score 0 (unsupported): ${summary.unsupported} (${pct(summary.unsupported, summary.total)})`);
|
|
6305
|
+
if (summary.byPage.length > 0) {
|
|
6306
|
+
lines.push("");
|
|
6307
|
+
lines.push(" Top pages:");
|
|
6308
|
+
for (const { slug, count } of summary.byPage.slice(0, 10)) {
|
|
6309
|
+
lines.push(` ${slug}: ${count} judgement${count === 1 ? "" : "s"}`);
|
|
6310
|
+
}
|
|
6311
|
+
}
|
|
6312
|
+
void judgements;
|
|
6313
|
+
return lines.join("\n");
|
|
6314
|
+
}
|
|
6315
|
+
var JUDGEMENT_DIVIDER = "\u2500".repeat(55);
|
|
6316
|
+
function formatJudgementsDisplay(judgements) {
|
|
6317
|
+
if (judgements.length === 0) return "No judgements to display.";
|
|
6318
|
+
const blocks = judgements.map((j, i) => {
|
|
6319
|
+
const scoreLabel = SCORE_LABELS[j.score] ?? "unknown";
|
|
6320
|
+
const header2 = `[${i + 1}/${judgements.length}] Page: ${j.pageSlug} Score: ${j.score} (${scoreLabel})`;
|
|
6321
|
+
return [
|
|
6322
|
+
JUDGEMENT_DIVIDER,
|
|
6323
|
+
header2,
|
|
6324
|
+
` File: ${j.citedFile} Lines: ${j.lineStart}\u2013${j.lineEnd}`,
|
|
6325
|
+
` Claim: "${j.claimText}"`,
|
|
6326
|
+
` Span: "${j.spanText}"`,
|
|
6327
|
+
` Reason: ${j.reason}`
|
|
6328
|
+
].join("\n");
|
|
6329
|
+
});
|
|
6330
|
+
return [...blocks, JUDGEMENT_DIVIDER].join("\n");
|
|
6331
|
+
}
|
|
6332
|
+
|
|
6333
|
+
// src/eval/cache.ts
|
|
6334
|
+
import { unlink as unlink3, readFile as readFile27 } from "fs/promises";
|
|
6335
|
+
import { existsSync as existsSync13 } from "fs";
|
|
6336
|
+
import path41 from "path";
|
|
6337
|
+
var CACHE_FILE2 = path41.join(".llmwiki", "eval", "citation-cache.jsonl");
|
|
6338
|
+
async function clearCitationCache(root) {
|
|
6339
|
+
const cachePath = path41.join(root, CACHE_FILE2);
|
|
6340
|
+
if (!existsSync13(cachePath)) return false;
|
|
6341
|
+
await unlink3(cachePath);
|
|
6342
|
+
return true;
|
|
6343
|
+
}
|
|
6344
|
+
async function loadCitationCache(root) {
|
|
6345
|
+
const cachePath = path41.join(root, CACHE_FILE2);
|
|
6346
|
+
if (!existsSync13(cachePath)) return [];
|
|
6347
|
+
const content = await readFile27(cachePath, "utf-8");
|
|
6348
|
+
const judgements = [];
|
|
6349
|
+
for (const line2 of content.trim().split("\n").filter(Boolean)) {
|
|
6350
|
+
try {
|
|
6351
|
+
judgements.push(JSON.parse(line2));
|
|
6352
|
+
} catch {
|
|
6353
|
+
}
|
|
6354
|
+
}
|
|
6355
|
+
return judgements;
|
|
6356
|
+
}
|
|
6357
|
+
function summarizeCitationCache(judgements) {
|
|
6358
|
+
let fullySupported = 0;
|
|
6359
|
+
let partiallySupported = 0;
|
|
6360
|
+
let unsupported = 0;
|
|
6361
|
+
const pageCounts = /* @__PURE__ */ new Map();
|
|
6362
|
+
for (const j of judgements) {
|
|
6363
|
+
if (j.score === 2) fullySupported++;
|
|
6364
|
+
else if (j.score === 1) partiallySupported++;
|
|
6365
|
+
else unsupported++;
|
|
6366
|
+
pageCounts.set(j.pageSlug, (pageCounts.get(j.pageSlug) ?? 0) + 1);
|
|
6367
|
+
}
|
|
6368
|
+
const byPage = [...pageCounts.entries()].map(([slug, count]) => ({ slug, count })).sort((a, b) => b.count - a.count);
|
|
6369
|
+
return { total: judgements.length, fullySupported, partiallySupported, unsupported, byPage };
|
|
6370
|
+
}
|
|
6371
|
+
|
|
6372
|
+
// src/commands/eval.ts
|
|
6373
|
+
var DEFAULT_SAMPLE_SIZE = 20;
|
|
6374
|
+
function parseSampleSize(raw) {
|
|
6375
|
+
const n = Number(raw);
|
|
6376
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
6377
|
+
throw new Error(`--sample must be a positive integer (got "${raw}")`);
|
|
6378
|
+
}
|
|
6379
|
+
return n;
|
|
6380
|
+
}
|
|
6381
|
+
function resolveEvalOptions(options) {
|
|
6382
|
+
return {
|
|
6383
|
+
suite: options.suite === "full" ? "full" : "fast",
|
|
6384
|
+
sampleSize: parseSampleSize(options.sample ?? String(DEFAULT_SAMPLE_SIZE)),
|
|
6385
|
+
outFormat: options.out === "json" ? "json" : "terminal"
|
|
6386
|
+
};
|
|
6387
|
+
}
|
|
6388
|
+
async function runEvalComponents(root, suite, sampleSize) {
|
|
6389
|
+
const [health, citationCoverage, stats, previousReport] = await Promise.all([
|
|
6390
|
+
evaluateHealth(root),
|
|
6391
|
+
evaluateCitationCoverage(root),
|
|
6392
|
+
collectStats(root),
|
|
6393
|
+
loadPreviousReport(root)
|
|
6394
|
+
]);
|
|
6395
|
+
const citationSupport = suite === "full" ? await evaluateCitationSupport(root, sampleSize, previousReport?.citationSupport?.sampledHashes ?? []) : void 0;
|
|
6396
|
+
return { health, citationCoverage, stats, previousReport, citationSupport };
|
|
6397
|
+
}
|
|
6398
|
+
async function buildReport(root, components, suite) {
|
|
6399
|
+
const { health, citationCoverage, stats, previousReport, citationSupport } = components;
|
|
6400
|
+
const partial = {
|
|
6401
|
+
suite,
|
|
6402
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6403
|
+
health,
|
|
6404
|
+
citationCoverage,
|
|
6405
|
+
stats,
|
|
6406
|
+
...citationSupport ? { citationSupport } : {}
|
|
6407
|
+
};
|
|
6408
|
+
const delta = previousReport ? computeDelta(partial, previousReport) : void 0;
|
|
6409
|
+
const thresholdViolations = await checkThresholds(partial, root);
|
|
6410
|
+
return { ...partial, ...delta ? { delta } : {}, thresholdViolations };
|
|
6411
|
+
}
|
|
6412
|
+
async function evalCommand(options = {}) {
|
|
6413
|
+
const root = process.cwd();
|
|
6414
|
+
const { suite, sampleSize, outFormat } = resolveEvalOptions(options);
|
|
6415
|
+
const components = await runEvalComponents(root, suite, sampleSize);
|
|
6416
|
+
const report = await buildReport(root, components, suite);
|
|
6417
|
+
await appendHistory(root, report);
|
|
6418
|
+
const output = outFormat === "json" ? formatJsonReport(report) : formatTerminalReport(report);
|
|
6419
|
+
console.log(output);
|
|
6420
|
+
if (report.thresholdViolations.length > 0) {
|
|
6421
|
+
process.exit(1);
|
|
6422
|
+
}
|
|
6423
|
+
}
|
|
6424
|
+
async function evalCacheClearCommand() {
|
|
6425
|
+
const deleted = await clearCitationCache(process.cwd());
|
|
6426
|
+
console.log(deleted ? "Citation cache cleared." : "No citation cache found.");
|
|
6427
|
+
}
|
|
6428
|
+
async function evalCacheShowCommand() {
|
|
6429
|
+
const judgements = await loadCitationCache(process.cwd());
|
|
6430
|
+
const summary = summarizeCitationCache(judgements);
|
|
6431
|
+
console.log(formatCacheShow(judgements, summary));
|
|
6432
|
+
}
|
|
6433
|
+
async function evalReportCommand(options = {}) {
|
|
6434
|
+
const report = await loadPreviousReport(process.cwd());
|
|
6435
|
+
if (!report) {
|
|
6436
|
+
console.log("No eval history found. Run `llmwiki eval` to record the first run.");
|
|
6437
|
+
return;
|
|
6438
|
+
}
|
|
6439
|
+
const output = options.out === "json" ? formatJsonReport(report) : formatTerminalReport(report);
|
|
6440
|
+
console.log(output);
|
|
6441
|
+
}
|
|
6442
|
+
async function evalHistoryCommand(options = {}) {
|
|
6443
|
+
const n = parseInt(options.n ?? "10", 10);
|
|
6444
|
+
const reports = await loadHistory(process.cwd(), n);
|
|
6445
|
+
if (options.out === "json") {
|
|
6446
|
+
console.log(JSON.stringify(reports, null, 2));
|
|
6447
|
+
return;
|
|
6448
|
+
}
|
|
6449
|
+
console.log(formatHistoryTable(reports));
|
|
6450
|
+
}
|
|
6451
|
+
function filterJudgements(judgements, options) {
|
|
6452
|
+
let result = judgements;
|
|
6453
|
+
if (options.score !== void 0) result = result.filter((j) => j.score === parseInt(options.score, 10));
|
|
6454
|
+
if (options.page) result = result.filter((j) => j.pageSlug === options.page);
|
|
6455
|
+
if (options.n !== void 0) result = result.slice(0, parseInt(options.n, 10));
|
|
6456
|
+
return result;
|
|
6457
|
+
}
|
|
6458
|
+
async function evalJudgementsCommand(options = {}) {
|
|
6459
|
+
const judgements = filterJudgements(await loadCitationCache(process.cwd()), options);
|
|
6460
|
+
if (options.out === "json") {
|
|
6461
|
+
console.log(JSON.stringify(judgements, null, 2));
|
|
6462
|
+
return;
|
|
6463
|
+
}
|
|
6464
|
+
console.log(formatJudgementsDisplay(judgements));
|
|
6465
|
+
}
|
|
6466
|
+
|
|
6467
|
+
// src/commands/export.ts
|
|
6468
|
+
import path42 from "path";
|
|
6469
|
+
import { createRequire } from "module";
|
|
6470
|
+
|
|
6471
|
+
// src/export/collect.ts
|
|
6472
|
+
function toExportPage(raw) {
|
|
6473
|
+
const meta = raw.frontmatter;
|
|
6474
|
+
return {
|
|
6475
|
+
title: raw.title,
|
|
6476
|
+
slug: raw.slug,
|
|
6477
|
+
pageDirectory: raw.pageDirectory,
|
|
6478
|
+
summary: typeof meta.summary === "string" ? meta.summary : "",
|
|
6479
|
+
sources: Array.isArray(meta.sources) ? meta.sources.filter((s) => typeof s === "string") : [],
|
|
6480
|
+
tags: Array.isArray(meta.tags) ? meta.tags.filter((t) => typeof t === "string") : [],
|
|
6481
|
+
createdAt: typeof meta.createdAt === "string" ? meta.createdAt : (/* @__PURE__ */ new Date()).toISOString(),
|
|
6482
|
+
updatedAt: typeof meta.updatedAt === "string" ? meta.updatedAt : (/* @__PURE__ */ new Date()).toISOString(),
|
|
6483
|
+
links: extractWikilinkSlugs(raw.body),
|
|
6484
|
+
body: raw.body
|
|
6485
|
+
};
|
|
6486
|
+
}
|
|
6487
|
+
async function collectExportPages(root) {
|
|
6488
|
+
const raw = await collectRawWikiPages(root);
|
|
6489
|
+
const kept = raw.filter((page) => page.parseStatus.hasTitle && !page.parseStatus.orphaned);
|
|
6490
|
+
const pages = kept.map(toExportPage);
|
|
6491
|
+
pages.sort((a, b) => a.title.localeCompare(b.title));
|
|
6492
|
+
return pages;
|
|
6493
|
+
}
|
|
6494
|
+
|
|
6495
|
+
// src/export/llms-txt.ts
|
|
6496
|
+
function pageRelativePath(page) {
|
|
6497
|
+
return `wiki/${page.pageDirectory}/${page.slug}.md`;
|
|
6498
|
+
}
|
|
6499
|
+
function buildEntryNote(page) {
|
|
6500
|
+
const parts = [];
|
|
6501
|
+
if (page.summary) parts.push(page.summary);
|
|
6502
|
+
if (page.tags.length > 0) parts.push(`tags: ${page.tags.join(", ")}`);
|
|
6503
|
+
if (page.sources.length > 0) parts.push(`sources: ${page.sources.join(", ")}`);
|
|
6504
|
+
parts.push(`created: ${page.createdAt}`);
|
|
6505
|
+
parts.push(`updated: ${page.updatedAt}`);
|
|
6506
|
+
return parts.join(" | ");
|
|
6507
|
+
}
|
|
6508
|
+
function formatPageEntry(page) {
|
|
6509
|
+
const note = buildEntryNote(page);
|
|
6510
|
+
return `- [${page.title}](${pageRelativePath(page)}): ${note}`;
|
|
6511
|
+
}
|
|
6512
|
+
function buildSection(heading, pages) {
|
|
6513
|
+
if (pages.length === 0) return [];
|
|
6514
|
+
return [`## ${heading}`, "", ...pages.map(formatPageEntry), ""];
|
|
6515
|
+
}
|
|
6516
|
+
function buildLlmsTxt(pages, projectTitle) {
|
|
6517
|
+
const concepts = pages.filter((p) => p.pageDirectory === "concepts");
|
|
6518
|
+
const queries = pages.filter((p) => p.pageDirectory === "queries");
|
|
6519
|
+
const lines = [
|
|
6520
|
+
`# ${projectTitle}`,
|
|
6521
|
+
"",
|
|
6522
|
+
`> ${pages.length} pages \u2014 exported ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
6523
|
+
"",
|
|
6524
|
+
...buildSection("Concepts", concepts),
|
|
6525
|
+
...buildSection("Saved Queries", queries)
|
|
6526
|
+
];
|
|
6527
|
+
return lines.join("\n");
|
|
6528
|
+
}
|
|
6529
|
+
function buildLlmsFullTxt(pages, projectTitle) {
|
|
6530
|
+
const sections = [buildLlmsTxt(pages, projectTitle)];
|
|
6531
|
+
for (const page of pages) {
|
|
6532
|
+
const tags = page.tags.length > 0 ? `
|
|
6533
|
+
Tags: ${page.tags.join(", ")}` : "";
|
|
6534
|
+
const sources = page.sources.length > 0 ? `
|
|
6535
|
+
Sources: ${page.sources.join(", ")}` : "";
|
|
6536
|
+
const header2 = [
|
|
6537
|
+
"---",
|
|
6538
|
+
`## ${page.title}`,
|
|
6539
|
+
`> ${page.summary}${tags}${sources}`,
|
|
6540
|
+
`Created: ${page.createdAt} | Updated: ${page.updatedAt}`,
|
|
6541
|
+
""
|
|
6542
|
+
].join("\n");
|
|
6543
|
+
sections.push(`${header2}
|
|
6544
|
+
${page.body.trim()}
|
|
6545
|
+
`);
|
|
6546
|
+
}
|
|
6547
|
+
return sections.join("\n");
|
|
6548
|
+
}
|
|
6549
|
+
|
|
6550
|
+
// src/export/json-export.ts
|
|
6551
|
+
function buildJsonExport(pages) {
|
|
6552
|
+
const doc = {
|
|
6553
|
+
exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6554
|
+
pageCount: pages.length,
|
|
6555
|
+
pages
|
|
6556
|
+
};
|
|
6557
|
+
return JSON.stringify(doc, null, 2);
|
|
6558
|
+
}
|
|
6559
|
+
|
|
6560
|
+
// src/export/json-ld.ts
|
|
6561
|
+
var LOCAL_BASE = "urn:llmwiki:";
|
|
6562
|
+
function pageIri(slug) {
|
|
6563
|
+
return `${LOCAL_BASE}${slug}`;
|
|
6564
|
+
}
|
|
6565
|
+
function pageToJsonLd(page) {
|
|
6566
|
+
const node = {
|
|
6567
|
+
"@id": pageIri(page.slug),
|
|
6568
|
+
"@type": "Article",
|
|
6569
|
+
name: page.title,
|
|
6570
|
+
description: page.summary,
|
|
6571
|
+
dateCreated: page.createdAt,
|
|
6572
|
+
dateModified: page.updatedAt
|
|
6573
|
+
};
|
|
6574
|
+
if (page.tags.length > 0) {
|
|
6575
|
+
node["keywords"] = page.tags;
|
|
6576
|
+
}
|
|
6577
|
+
if (page.sources.length > 0) {
|
|
6578
|
+
node["isBasedOn"] = page.sources;
|
|
6579
|
+
}
|
|
6580
|
+
if (page.links.length > 0) {
|
|
6581
|
+
node["mentions"] = page.links.map((slug) => ({ "@id": pageIri(slug) }));
|
|
6582
|
+
}
|
|
6583
|
+
return node;
|
|
6584
|
+
}
|
|
6585
|
+
function buildJsonLd(pages) {
|
|
6586
|
+
const doc = {
|
|
6587
|
+
"@context": "https://schema.org",
|
|
6588
|
+
"@graph": pages.map(pageToJsonLd)
|
|
6589
|
+
};
|
|
6590
|
+
return JSON.stringify(doc, null, 2);
|
|
6591
|
+
}
|
|
6592
|
+
|
|
6593
|
+
// src/export/graphml.ts
|
|
6594
|
+
var XML_ESCAPES = {
|
|
6595
|
+
"&": "&",
|
|
6596
|
+
"<": "<",
|
|
6597
|
+
">": ">",
|
|
6598
|
+
'"': """,
|
|
6599
|
+
"'": "'"
|
|
6600
|
+
};
|
|
5663
6601
|
function escapeXml(value) {
|
|
5664
6602
|
return value.replace(/[&<>"']/g, (ch) => XML_ESCAPES[ch] ?? ch);
|
|
5665
6603
|
}
|
|
@@ -5789,7 +6727,7 @@ var TARGET_FILENAMES = {
|
|
|
5789
6727
|
};
|
|
5790
6728
|
function resolveProjectTitle(root) {
|
|
5791
6729
|
try {
|
|
5792
|
-
const pkg = require2(
|
|
6730
|
+
const pkg = require2(path42.join(root, "package.json"));
|
|
5793
6731
|
return typeof pkg.name === "string" ? pkg.name : "Knowledge Wiki";
|
|
5794
6732
|
} catch {
|
|
5795
6733
|
return "Knowledge Wiki";
|
|
@@ -5841,7 +6779,7 @@ async function runExport(root, options = {}) {
|
|
|
5841
6779
|
const written = [];
|
|
5842
6780
|
for (const target of targets) {
|
|
5843
6781
|
const content = buildContent(target, pages, projectTitle, marpSource);
|
|
5844
|
-
const outPath =
|
|
6782
|
+
const outPath = path42.join(root, EXPORT_DIR, TARGET_FILENAMES[target]);
|
|
5845
6783
|
await atomicWrite(outPath, content);
|
|
5846
6784
|
written.push(outPath);
|
|
5847
6785
|
status("+", success(`Exported ${target} \u2192 ${source(outPath)}`));
|
|
@@ -5867,18 +6805,18 @@ async function exportCommand(root, options) {
|
|
|
5867
6805
|
}
|
|
5868
6806
|
|
|
5869
6807
|
// src/commands/schema.ts
|
|
5870
|
-
import { existsSync as
|
|
5871
|
-
import { mkdir as
|
|
5872
|
-
import
|
|
6808
|
+
import { existsSync as existsSync14 } from "fs";
|
|
6809
|
+
import { mkdir as mkdir9, writeFile as writeFile5 } from "fs/promises";
|
|
6810
|
+
import path43 from "path";
|
|
5873
6811
|
async function schemaInitCommand() {
|
|
5874
6812
|
const root = process.cwd();
|
|
5875
6813
|
const defaults = buildDefaultSchema();
|
|
5876
6814
|
const targetPath = defaultSchemaInitPath(root);
|
|
5877
|
-
if (
|
|
6815
|
+
if (existsSync14(targetPath)) {
|
|
5878
6816
|
status("!", warn(`Schema file already exists at ${targetPath}`));
|
|
5879
6817
|
return;
|
|
5880
6818
|
}
|
|
5881
|
-
await
|
|
6819
|
+
await mkdir9(path43.dirname(targetPath), { recursive: true });
|
|
5882
6820
|
const serializable = {
|
|
5883
6821
|
version: defaults.version,
|
|
5884
6822
|
defaultKind: defaults.defaultKind,
|
|
@@ -5914,121 +6852,1814 @@ async function reviewListCommand() {
|
|
|
5914
6852
|
dim(`Use \`llmwiki review show <id>\` to inspect a candidate.`)
|
|
5915
6853
|
);
|
|
5916
6854
|
}
|
|
5917
|
-
|
|
5918
|
-
// src/commands/review-show.ts
|
|
5919
|
-
async function reviewShowCommand(id) {
|
|
5920
|
-
const candidate = await loadCandidateOrFail(process.cwd(), id);
|
|
5921
|
-
if (!candidate) return;
|
|
5922
|
-
header(`Candidate ${candidate.id}`);
|
|
5923
|
-
status("i", dim(`title: ${candidate.title}`));
|
|
5924
|
-
status("i", dim(`slug: ${candidate.slug}`));
|
|
5925
|
-
status("i", dim(`summary: ${candidate.summary}`));
|
|
5926
|
-
status("i", dim(`sources: ${candidate.sources.join(", ")}`));
|
|
5927
|
-
status("i", dim(`generated: ${candidate.generatedAt}`));
|
|
5928
|
-
console.log();
|
|
5929
|
-
console.log(candidate.body);
|
|
5930
|
-
if (candidate.schemaViolations && candidate.schemaViolations.length > 0) {
|
|
5931
|
-
console.log();
|
|
5932
|
-
header("Schema violations");
|
|
5933
|
-
for (const v of candidate.schemaViolations) {
|
|
5934
|
-
status("!", warn(`[${v.severity}] ${v.message}`));
|
|
5935
|
-
}
|
|
6855
|
+
|
|
6856
|
+
// src/commands/review-show.ts
|
|
6857
|
+
async function reviewShowCommand(id) {
|
|
6858
|
+
const candidate = await loadCandidateOrFail(process.cwd(), id);
|
|
6859
|
+
if (!candidate) return;
|
|
6860
|
+
header(`Candidate ${candidate.id}`);
|
|
6861
|
+
status("i", dim(`title: ${candidate.title}`));
|
|
6862
|
+
status("i", dim(`slug: ${candidate.slug}`));
|
|
6863
|
+
status("i", dim(`summary: ${candidate.summary}`));
|
|
6864
|
+
status("i", dim(`sources: ${candidate.sources.join(", ")}`));
|
|
6865
|
+
status("i", dim(`generated: ${candidate.generatedAt}`));
|
|
6866
|
+
console.log();
|
|
6867
|
+
console.log(candidate.body);
|
|
6868
|
+
if (candidate.schemaViolations && candidate.schemaViolations.length > 0) {
|
|
6869
|
+
console.log();
|
|
6870
|
+
header("Schema violations");
|
|
6871
|
+
for (const v of candidate.schemaViolations) {
|
|
6872
|
+
status("!", warn(`[${v.severity}] ${v.message}`));
|
|
6873
|
+
}
|
|
6874
|
+
}
|
|
6875
|
+
if (candidate.provenanceViolations && candidate.provenanceViolations.length > 0) {
|
|
6876
|
+
console.log();
|
|
6877
|
+
header("Provenance violations");
|
|
6878
|
+
for (const v of candidate.provenanceViolations) {
|
|
6879
|
+
status("!", warn(`[${v.severity}] ${v.message}`));
|
|
6880
|
+
}
|
|
6881
|
+
}
|
|
6882
|
+
}
|
|
6883
|
+
|
|
6884
|
+
// src/commands/review-approve.ts
|
|
6885
|
+
import path44 from "path";
|
|
6886
|
+
|
|
6887
|
+
// src/commands/review-helpers.ts
|
|
6888
|
+
async function runReviewUnderLock(id, underLock) {
|
|
6889
|
+
const root = process.cwd();
|
|
6890
|
+
const preCheck = await loadCandidateOrFail(root, id);
|
|
6891
|
+
if (!preCheck) return;
|
|
6892
|
+
const locked = await acquireLock(root);
|
|
6893
|
+
if (!locked) {
|
|
6894
|
+
status("!", error("Could not acquire lock. Try again later."));
|
|
6895
|
+
process.exitCode = 1;
|
|
6896
|
+
return;
|
|
6897
|
+
}
|
|
6898
|
+
try {
|
|
6899
|
+
await underLock(root, id);
|
|
6900
|
+
} finally {
|
|
6901
|
+
await releaseLock(root);
|
|
6902
|
+
}
|
|
6903
|
+
}
|
|
6904
|
+
|
|
6905
|
+
// src/commands/review-approve.ts
|
|
6906
|
+
async function reviewApproveCommand(id) {
|
|
6907
|
+
await runReviewUnderLock(id, approveUnderLock);
|
|
6908
|
+
}
|
|
6909
|
+
async function approveUnderLock(root, id) {
|
|
6910
|
+
const candidate = await loadCandidateUnderLockOrFail(root, id);
|
|
6911
|
+
if (!candidate) return;
|
|
6912
|
+
if (!validateWikiPage(candidate.body)) {
|
|
6913
|
+
status("!", error(`Candidate ${id} failed page validation; not approved.`));
|
|
6914
|
+
process.exitCode = 1;
|
|
6915
|
+
return;
|
|
6916
|
+
}
|
|
6917
|
+
const pagePath = path44.join(root, CONCEPTS_DIR, `${candidate.slug}.md`);
|
|
6918
|
+
await atomicWrite(pagePath, candidate.body);
|
|
6919
|
+
status("+", success(`Approved \u2192 ${source(pagePath)}`));
|
|
6920
|
+
await persistCandidateSourceStates(root, candidate);
|
|
6921
|
+
await refreshWikiAfterApproval(root, candidate.slug);
|
|
6922
|
+
await deleteCandidate(root, id);
|
|
6923
|
+
status("\u2713", dim(`Candidate ${id} cleared.`));
|
|
6924
|
+
}
|
|
6925
|
+
async function persistCandidateSourceStates(root, candidate) {
|
|
6926
|
+
const states = candidate.sourceStates;
|
|
6927
|
+
if (!states) return;
|
|
6928
|
+
const otherSources = await collectOtherCandidateSources(root, candidate.id);
|
|
6929
|
+
for (const [sourceFile, entry] of Object.entries(states)) {
|
|
6930
|
+
if (otherSources.has(sourceFile)) continue;
|
|
6931
|
+
await updateSourceState(root, sourceFile, entry);
|
|
6932
|
+
}
|
|
6933
|
+
}
|
|
6934
|
+
async function collectOtherCandidateSources(root, approvingId) {
|
|
6935
|
+
const pending = await listCandidates(root);
|
|
6936
|
+
const sources = /* @__PURE__ */ new Set();
|
|
6937
|
+
for (const candidate of pending) {
|
|
6938
|
+
if (candidate.id === approvingId) continue;
|
|
6939
|
+
for (const source2 of candidate.sources) sources.add(source2);
|
|
6940
|
+
}
|
|
6941
|
+
return sources;
|
|
6942
|
+
}
|
|
6943
|
+
async function refreshWikiAfterApproval(root, slug) {
|
|
6944
|
+
await resolveLinks(root, [slug], [slug]);
|
|
6945
|
+
await generateIndex(root);
|
|
6946
|
+
await generateMOC(root);
|
|
6947
|
+
await safelyUpdateEmbeddings2(root, [slug]);
|
|
6948
|
+
}
|
|
6949
|
+
async function safelyUpdateEmbeddings2(root, slugs) {
|
|
6950
|
+
try {
|
|
6951
|
+
await updateEmbeddings(root, slugs);
|
|
6952
|
+
} catch (err) {
|
|
6953
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
6954
|
+
status("!", warn(`Skipped embeddings update: ${message}`));
|
|
6955
|
+
}
|
|
6956
|
+
}
|
|
6957
|
+
|
|
6958
|
+
// src/commands/review-reject.ts
|
|
6959
|
+
async function reviewRejectCommand(id) {
|
|
6960
|
+
await runReviewUnderLock(id, rejectUnderLock);
|
|
6961
|
+
}
|
|
6962
|
+
async function rejectUnderLock(root, id) {
|
|
6963
|
+
const candidate = await loadCandidateUnderLockOrFail(root, id);
|
|
6964
|
+
if (!candidate) return;
|
|
6965
|
+
await archiveCandidate(root, id);
|
|
6966
|
+
status(
|
|
6967
|
+
"-",
|
|
6968
|
+
warn(`Rejected candidate ${id} (${candidate.slug}) \u2014 archived, wiki unchanged.`)
|
|
6969
|
+
);
|
|
6970
|
+
}
|
|
6971
|
+
|
|
6972
|
+
// src/project/state.ts
|
|
6973
|
+
import { stat as stat2, readdir as readdir13, readFile as readFile28 } from "fs/promises";
|
|
6974
|
+
import path45 from "path";
|
|
6975
|
+
var MARKDOWN_EXT = ".md";
|
|
6976
|
+
async function collectProjectState(root) {
|
|
6977
|
+
const rootReadable = await isDirectory(root);
|
|
6978
|
+
if (!rootReadable) return brokenProjectState(root);
|
|
6979
|
+
const dirs = await collectDirPresence(root);
|
|
6980
|
+
const counts = await collectPageCounts(root, dirs);
|
|
6981
|
+
const lint2 = await collectLintCacheStatus(root);
|
|
6982
|
+
const mtimes = await collectMtimes(root, dirs);
|
|
6983
|
+
return assembleState({ root, dirs, counts, lint: lint2, mtimes });
|
|
6984
|
+
}
|
|
6985
|
+
function brokenProjectState(root) {
|
|
6986
|
+
return {
|
|
6987
|
+
root,
|
|
6988
|
+
hasSourcesDir: false,
|
|
6989
|
+
hasWikiDir: false,
|
|
6990
|
+
hasInternalDir: false,
|
|
6991
|
+
sourceCount: 0,
|
|
6992
|
+
conceptCount: 0,
|
|
6993
|
+
queryCount: 0,
|
|
6994
|
+
pendingCandidates: 0,
|
|
6995
|
+
hasIndex: false,
|
|
6996
|
+
lint: { present: false, entry: null },
|
|
6997
|
+
latestWikiMtimeMs: null,
|
|
6998
|
+
latestSourceMtimeMs: null,
|
|
6999
|
+
warnings: [
|
|
7000
|
+
{
|
|
7001
|
+
code: "project-unreadable",
|
|
7002
|
+
message: `Project root is unreadable or could not be inspected: ${root}`
|
|
7003
|
+
}
|
|
7004
|
+
]
|
|
7005
|
+
};
|
|
7006
|
+
}
|
|
7007
|
+
async function collectDirPresence(root) {
|
|
7008
|
+
const [hasSourcesDir, hasWikiDir, hasInternalDir] = await Promise.all([
|
|
7009
|
+
isDirectory(path45.join(root, SOURCES_DIR)),
|
|
7010
|
+
isDirectory(path45.join(root, "wiki")),
|
|
7011
|
+
isDirectory(path45.join(root, LLMWIKI_DIR))
|
|
7012
|
+
]);
|
|
7013
|
+
return { hasSourcesDir, hasWikiDir, hasInternalDir };
|
|
7014
|
+
}
|
|
7015
|
+
async function collectPageCounts(root, dirs) {
|
|
7016
|
+
const [sourceCount, conceptCount, queryCount, pendingCandidates, hasIndex] = await Promise.all([
|
|
7017
|
+
dirs.hasSourcesDir ? countMarkdownFiles(path45.join(root, SOURCES_DIR)) : 0,
|
|
7018
|
+
dirs.hasWikiDir ? countMarkdownFiles(path45.join(root, CONCEPTS_DIR)) : 0,
|
|
7019
|
+
dirs.hasWikiDir ? countMarkdownFiles(path45.join(root, QUERIES_DIR)) : 0,
|
|
7020
|
+
dirs.hasInternalDir ? safeCountCandidates(root) : 0,
|
|
7021
|
+
dirs.hasWikiDir ? isFile(path45.join(root, INDEX_FILE)) : false
|
|
7022
|
+
]);
|
|
7023
|
+
return { sourceCount, conceptCount, queryCount, pendingCandidates, hasIndex };
|
|
7024
|
+
}
|
|
7025
|
+
async function collectMtimes(root, dirs) {
|
|
7026
|
+
const [latestWikiMtimeMs, latestSourceMtimeMs] = await Promise.all([
|
|
7027
|
+
dirs.hasWikiDir ? safeMtime(path45.join(root, "wiki")) : Promise.resolve(null),
|
|
7028
|
+
dirs.hasSourcesDir ? safeMtime(path45.join(root, SOURCES_DIR)) : Promise.resolve(null)
|
|
7029
|
+
]);
|
|
7030
|
+
return { latestWikiMtimeMs, latestSourceMtimeMs };
|
|
7031
|
+
}
|
|
7032
|
+
async function collectLintCacheStatus(root) {
|
|
7033
|
+
const cachePath = path45.join(root, LAST_LINT_FILE);
|
|
7034
|
+
const exists = await isFile(cachePath);
|
|
7035
|
+
if (!exists) return { present: false, entry: null };
|
|
7036
|
+
const entry = await readLintCacheEntry(cachePath);
|
|
7037
|
+
return { present: true, entry };
|
|
7038
|
+
}
|
|
7039
|
+
async function readLintCacheEntry(cachePath) {
|
|
7040
|
+
let raw;
|
|
7041
|
+
try {
|
|
7042
|
+
raw = await readFile28(cachePath, "utf-8");
|
|
7043
|
+
} catch {
|
|
7044
|
+
return null;
|
|
7045
|
+
}
|
|
7046
|
+
try {
|
|
7047
|
+
const parsed = JSON.parse(raw);
|
|
7048
|
+
return validateCacheShape(parsed);
|
|
7049
|
+
} catch {
|
|
7050
|
+
return null;
|
|
7051
|
+
}
|
|
7052
|
+
}
|
|
7053
|
+
function validateCacheShape(value) {
|
|
7054
|
+
if (typeof value !== "object" || value === null) return null;
|
|
7055
|
+
const candidate = value;
|
|
7056
|
+
const { warnings, errors, at } = candidate;
|
|
7057
|
+
if (!isNonNegativeInteger2(warnings)) return null;
|
|
7058
|
+
if (!isNonNegativeInteger2(errors)) return null;
|
|
7059
|
+
if (typeof at !== "string" || !LINT_CACHE_TIMESTAMP_PATTERN.test(at)) return null;
|
|
7060
|
+
return { warnings, errors, at };
|
|
7061
|
+
}
|
|
7062
|
+
function isNonNegativeInteger2(value) {
|
|
7063
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
7064
|
+
}
|
|
7065
|
+
function assembleState(input) {
|
|
7066
|
+
const { root, dirs, counts, lint: lint2, mtimes } = input;
|
|
7067
|
+
const warnings = buildWarnings({ dirs, counts, lint: lint2, mtimes });
|
|
7068
|
+
return { root, ...dirs, ...counts, lint: lint2, ...mtimes, warnings };
|
|
7069
|
+
}
|
|
7070
|
+
function buildWarnings(input) {
|
|
7071
|
+
const warnings = [];
|
|
7072
|
+
appendLintWarnings(warnings, input.lint, input.mtimes.latestWikiMtimeMs);
|
|
7073
|
+
appendStructuralWarnings(warnings, input.dirs, input.counts);
|
|
7074
|
+
return warnings;
|
|
7075
|
+
}
|
|
7076
|
+
function appendLintWarnings(warnings, lint2, latestWikiMtimeMs) {
|
|
7077
|
+
if (lint2.present && lint2.entry === null) {
|
|
7078
|
+
warnings.push({
|
|
7079
|
+
code: "lint-cache-unparseable",
|
|
7080
|
+
message: "Lint cache file exists but could not be parsed. Re-run `llmwiki lint`."
|
|
7081
|
+
});
|
|
7082
|
+
return;
|
|
7083
|
+
}
|
|
7084
|
+
if (lint2.entry && isLintCacheStale(lint2.entry, latestWikiMtimeMs)) {
|
|
7085
|
+
warnings.push({
|
|
7086
|
+
code: "lint-cache-stale",
|
|
7087
|
+
message: "Lint cache is older than the wiki directory; pages were added or removed since the last lint."
|
|
7088
|
+
});
|
|
7089
|
+
}
|
|
7090
|
+
}
|
|
7091
|
+
function appendStructuralWarnings(warnings, dirs, counts) {
|
|
7092
|
+
const hasPages = counts.conceptCount > 0 || counts.queryCount > 0;
|
|
7093
|
+
if (hasPages && !counts.hasIndex) {
|
|
7094
|
+
warnings.push({
|
|
7095
|
+
code: "index-missing",
|
|
7096
|
+
message: "wiki/index.md is missing even though pages exist."
|
|
7097
|
+
});
|
|
7098
|
+
}
|
|
7099
|
+
if (counts.pendingCandidates > 0) {
|
|
7100
|
+
warnings.push({
|
|
7101
|
+
code: "pending-candidates",
|
|
7102
|
+
message: `${counts.pendingCandidates} generated candidate${counts.pendingCandidates === 1 ? "" : "s"} waiting for review.`
|
|
7103
|
+
});
|
|
7104
|
+
}
|
|
7105
|
+
if (dirs.hasSourcesDir && counts.sourceCount > 0 && !hasPages) {
|
|
7106
|
+
warnings.push({
|
|
7107
|
+
code: "sources-not-compiled",
|
|
7108
|
+
message: "Sources exist but no wiki pages were found. Run `llmwiki compile`."
|
|
7109
|
+
});
|
|
7110
|
+
}
|
|
7111
|
+
}
|
|
7112
|
+
function isLintCacheStale(entry, latestWikiMtimeMs) {
|
|
7113
|
+
if (latestWikiMtimeMs === null) return false;
|
|
7114
|
+
const lintMs = Date.parse(entry.at);
|
|
7115
|
+
if (Number.isNaN(lintMs)) return false;
|
|
7116
|
+
return latestWikiMtimeMs > lintMs;
|
|
7117
|
+
}
|
|
7118
|
+
async function isDirectory(target) {
|
|
7119
|
+
try {
|
|
7120
|
+
const stats = await stat2(target);
|
|
7121
|
+
return stats.isDirectory();
|
|
7122
|
+
} catch {
|
|
7123
|
+
return false;
|
|
7124
|
+
}
|
|
7125
|
+
}
|
|
7126
|
+
async function isFile(target) {
|
|
7127
|
+
try {
|
|
7128
|
+
const stats = await stat2(target);
|
|
7129
|
+
return stats.isFile();
|
|
7130
|
+
} catch {
|
|
7131
|
+
return false;
|
|
7132
|
+
}
|
|
7133
|
+
}
|
|
7134
|
+
async function safeMtime(target) {
|
|
7135
|
+
try {
|
|
7136
|
+
const stats = await stat2(target);
|
|
7137
|
+
return stats.mtimeMs;
|
|
7138
|
+
} catch {
|
|
7139
|
+
return null;
|
|
7140
|
+
}
|
|
7141
|
+
}
|
|
7142
|
+
async function countMarkdownFiles(dir) {
|
|
7143
|
+
try {
|
|
7144
|
+
const entries = await readdir13(dir, { withFileTypes: true });
|
|
7145
|
+
let count = 0;
|
|
7146
|
+
for (const entry of entries) {
|
|
7147
|
+
if (entry.isFile() && entry.name.endsWith(MARKDOWN_EXT)) count += 1;
|
|
7148
|
+
}
|
|
7149
|
+
return count;
|
|
7150
|
+
} catch {
|
|
7151
|
+
return 0;
|
|
7152
|
+
}
|
|
7153
|
+
}
|
|
7154
|
+
async function safeCountCandidates(root) {
|
|
7155
|
+
try {
|
|
7156
|
+
return await countCandidates(root);
|
|
7157
|
+
} catch {
|
|
7158
|
+
return 0;
|
|
7159
|
+
}
|
|
7160
|
+
}
|
|
7161
|
+
|
|
7162
|
+
// src/project/recommendations.ts
|
|
7163
|
+
function recommendNextAction(state) {
|
|
7164
|
+
const kind = classifyState(state);
|
|
7165
|
+
return { state: kind, recommended: primaryAction(kind), otherActions: otherActionsFor(kind) };
|
|
7166
|
+
}
|
|
7167
|
+
function classifyState(state) {
|
|
7168
|
+
if (isBrokenProject(state)) return "broken-project";
|
|
7169
|
+
if (isSourcesOnly(state)) return "sources-only";
|
|
7170
|
+
if (state.pendingCandidates > 0) return "review-pending";
|
|
7171
|
+
if (hasLintErrors(state)) return "lint-attention";
|
|
7172
|
+
if (hasWikiPages(state)) return "wiki-ready";
|
|
7173
|
+
if (isEmptyWiki(state)) return "empty-wiki";
|
|
7174
|
+
return "fresh";
|
|
7175
|
+
}
|
|
7176
|
+
function isBrokenProject(state) {
|
|
7177
|
+
return state.warnings.some((w) => w.code === "project-unreadable");
|
|
7178
|
+
}
|
|
7179
|
+
function isEmptyWiki(state) {
|
|
7180
|
+
return state.hasWikiDir && !hasWikiPages(state);
|
|
7181
|
+
}
|
|
7182
|
+
function isSourcesOnly(state) {
|
|
7183
|
+
return state.sourceCount > 0 && !hasWikiPages(state);
|
|
7184
|
+
}
|
|
7185
|
+
function hasLintErrors(state) {
|
|
7186
|
+
return state.lint.entry !== null && state.lint.entry.errors > 0;
|
|
7187
|
+
}
|
|
7188
|
+
function hasWikiPages(state) {
|
|
7189
|
+
return state.conceptCount > 0 || state.queryCount > 0;
|
|
7190
|
+
}
|
|
7191
|
+
function primaryAction(kind) {
|
|
7192
|
+
return PRIMARY_ACTIONS[kind];
|
|
7193
|
+
}
|
|
7194
|
+
function otherActionsFor(kind) {
|
|
7195
|
+
return OTHER_ACTIONS[kind].map((a) => ({ ...a }));
|
|
7196
|
+
}
|
|
7197
|
+
var QUICKSTART_ACTION = {
|
|
7198
|
+
command: "llmwiki quickstart <source>",
|
|
7199
|
+
reason: "Ingest a source and compile a wiki in one step.",
|
|
7200
|
+
executable: { binary: "llmwiki", args: ["quickstart"], placeholders: ["source"] }
|
|
7201
|
+
};
|
|
7202
|
+
var INGEST_ACTION = {
|
|
7203
|
+
command: "llmwiki ingest <source>",
|
|
7204
|
+
reason: "Add sources manually before compiling.",
|
|
7205
|
+
executable: { binary: "llmwiki", args: ["ingest"], placeholders: ["source"] }
|
|
7206
|
+
};
|
|
7207
|
+
var COMPILE_ACTION = {
|
|
7208
|
+
command: "llmwiki compile",
|
|
7209
|
+
reason: "Compile sources/ into wiki pages.",
|
|
7210
|
+
executable: { binary: "llmwiki", args: ["compile"] }
|
|
7211
|
+
};
|
|
7212
|
+
var REVIEW_LIST_ACTION = {
|
|
7213
|
+
command: "llmwiki review list",
|
|
7214
|
+
reason: "List pending candidate pages.",
|
|
7215
|
+
executable: { binary: "llmwiki", args: ["review", "list"] }
|
|
7216
|
+
};
|
|
7217
|
+
var REVIEW_APPROVE_ACTION = {
|
|
7218
|
+
command: "llmwiki review approve <id>",
|
|
7219
|
+
reason: "Approve a candidate after inspecting it with `llmwiki review show <id>`.",
|
|
7220
|
+
executable: { binary: "llmwiki", args: ["review", "approve"], placeholders: ["id"] }
|
|
7221
|
+
};
|
|
7222
|
+
var LINT_ACTION = {
|
|
7223
|
+
command: "llmwiki lint",
|
|
7224
|
+
reason: "Re-run lint to inspect outstanding errors.",
|
|
7225
|
+
executable: { binary: "llmwiki", args: ["lint"] }
|
|
7226
|
+
};
|
|
7227
|
+
var VIEW_OPEN_ACTION = {
|
|
7228
|
+
command: "llmwiki view --open",
|
|
7229
|
+
reason: "Browse the compiled wiki in the local viewer.",
|
|
7230
|
+
executable: { binary: "llmwiki", args: ["view", "--open"] }
|
|
7231
|
+
};
|
|
7232
|
+
var QUERY_ACTION = {
|
|
7233
|
+
command: 'llmwiki query "<question>"',
|
|
7234
|
+
reason: "Ask a natural-language question against the compiled wiki.",
|
|
7235
|
+
executable: { binary: "llmwiki", args: ["query"], placeholders: ["question"] }
|
|
7236
|
+
};
|
|
7237
|
+
var BROKEN_PROJECT_ACTION = {
|
|
7238
|
+
command: null,
|
|
7239
|
+
reason: "Project root is unreadable or could not be inspected.",
|
|
7240
|
+
executable: null
|
|
7241
|
+
};
|
|
7242
|
+
var PRIMARY_ACTIONS = {
|
|
7243
|
+
"broken-project": BROKEN_PROJECT_ACTION,
|
|
7244
|
+
fresh: { ...QUICKSTART_ACTION, reason: "No sources or wiki pages were found." },
|
|
7245
|
+
"sources-only": { ...COMPILE_ACTION, reason: "Sources exist but no wiki pages have been compiled." },
|
|
7246
|
+
"review-pending": { ...REVIEW_LIST_ACTION, reason: "Generated candidates are waiting for review." },
|
|
7247
|
+
"lint-attention": { ...LINT_ACTION, reason: "Lint has reported errors; rerun lint to inspect them." },
|
|
7248
|
+
"wiki-ready": { ...VIEW_OPEN_ACTION, reason: "Wiki pages are ready to browse." },
|
|
7249
|
+
"empty-wiki": {
|
|
7250
|
+
...COMPILE_ACTION,
|
|
7251
|
+
reason: "wiki/ exists but is empty; run compile or add sources first."
|
|
7252
|
+
}
|
|
7253
|
+
};
|
|
7254
|
+
var OTHER_ACTIONS = {
|
|
7255
|
+
fresh: [INGEST_ACTION, QUICKSTART_ACTION],
|
|
7256
|
+
"sources-only": [COMPILE_ACTION, QUICKSTART_ACTION],
|
|
7257
|
+
"empty-wiki": [COMPILE_ACTION, INGEST_ACTION],
|
|
7258
|
+
"review-pending": [REVIEW_LIST_ACTION, REVIEW_APPROVE_ACTION],
|
|
7259
|
+
"lint-attention": [LINT_ACTION, VIEW_OPEN_ACTION],
|
|
7260
|
+
"wiki-ready": [VIEW_OPEN_ACTION, QUERY_ACTION],
|
|
7261
|
+
"broken-project": []
|
|
7262
|
+
};
|
|
7263
|
+
|
|
7264
|
+
// src/commands/next.ts
|
|
7265
|
+
var NEXT_JSON_VERSION = 1;
|
|
7266
|
+
var STATE_LINE_RENDERERS = {
|
|
7267
|
+
"broken-project": () => "project root unreadable",
|
|
7268
|
+
fresh: () => "no llmwiki project detected",
|
|
7269
|
+
"sources-only": (state) => `${state.sourceCount} source${plural(state.sourceCount)}, no wiki pages yet`,
|
|
7270
|
+
"review-pending": (state) => `review pending, ${state.pendingCandidates} candidate${plural(state.pendingCandidates)}`,
|
|
7271
|
+
"lint-attention": formatLintAttentionLine,
|
|
7272
|
+
"empty-wiki": () => "wiki/ exists but is empty",
|
|
7273
|
+
"wiki-ready": formatWikiReadyLine
|
|
7274
|
+
};
|
|
7275
|
+
async function nextCommand(options = {}) {
|
|
7276
|
+
const state = await collectProjectState(process.cwd());
|
|
7277
|
+
const recommendation = recommendNextAction(state);
|
|
7278
|
+
if (options.json) {
|
|
7279
|
+
process.stdout.write(`${JSON.stringify(buildJsonPayload(state, recommendation), null, 2)}
|
|
7280
|
+
`);
|
|
7281
|
+
} else {
|
|
7282
|
+
process.stdout.write(`${renderHuman(state, recommendation)}
|
|
7283
|
+
`);
|
|
7284
|
+
}
|
|
7285
|
+
return 0;
|
|
7286
|
+
}
|
|
7287
|
+
function buildJsonPayload(state, recommendation) {
|
|
7288
|
+
return {
|
|
7289
|
+
version: NEXT_JSON_VERSION,
|
|
7290
|
+
projectRoot: state.root,
|
|
7291
|
+
state: recommendation.state,
|
|
7292
|
+
summary: buildSummary(state),
|
|
7293
|
+
recommended: recommendation.recommended,
|
|
7294
|
+
otherActions: recommendation.otherActions,
|
|
7295
|
+
warnings: state.warnings
|
|
7296
|
+
};
|
|
7297
|
+
}
|
|
7298
|
+
function buildSummary(state) {
|
|
7299
|
+
return {
|
|
7300
|
+
sourceCount: state.sourceCount,
|
|
7301
|
+
conceptCount: state.conceptCount,
|
|
7302
|
+
queryCount: state.queryCount,
|
|
7303
|
+
pendingCandidates: state.pendingCandidates,
|
|
7304
|
+
hasIndex: state.hasIndex,
|
|
7305
|
+
hasLintCache: state.lint.present,
|
|
7306
|
+
lint: state.lint.entry
|
|
7307
|
+
};
|
|
7308
|
+
}
|
|
7309
|
+
function renderHuman(state, recommendation) {
|
|
7310
|
+
const lines = [];
|
|
7311
|
+
lines.push("llmwiki next");
|
|
7312
|
+
lines.push("------------");
|
|
7313
|
+
lines.push("");
|
|
7314
|
+
lines.push(`Project: ${state.root}`);
|
|
7315
|
+
lines.push(`State: ${describeStateLine(state, recommendation.state)}`);
|
|
7316
|
+
appendHumanRecommendation(lines, recommendation.recommended);
|
|
7317
|
+
appendHumanOtherActions(lines, recommendation.otherActions);
|
|
7318
|
+
appendHumanWarnings(lines, state.warnings);
|
|
7319
|
+
return lines.join("\n");
|
|
7320
|
+
}
|
|
7321
|
+
function appendHumanRecommendation(lines, action) {
|
|
7322
|
+
lines.push("");
|
|
7323
|
+
lines.push("Recommended next action:");
|
|
7324
|
+
lines.push(` ${action.command ?? action.reason}`);
|
|
7325
|
+
}
|
|
7326
|
+
function appendHumanOtherActions(lines, actions) {
|
|
7327
|
+
if (actions.length === 0) return;
|
|
7328
|
+
lines.push("");
|
|
7329
|
+
lines.push("Other useful actions:");
|
|
7330
|
+
for (const action of actions) {
|
|
7331
|
+
if (action.command) lines.push(` ${action.command}`);
|
|
7332
|
+
}
|
|
7333
|
+
}
|
|
7334
|
+
function appendHumanWarnings(lines, warnings) {
|
|
7335
|
+
if (warnings.length === 0) return;
|
|
7336
|
+
lines.push("");
|
|
7337
|
+
lines.push("Notes:");
|
|
7338
|
+
for (const warning of warnings) lines.push(` - ${warning.message}`);
|
|
7339
|
+
}
|
|
7340
|
+
function describeStateLine(state, kind) {
|
|
7341
|
+
return STATE_LINE_RENDERERS[kind](state);
|
|
7342
|
+
}
|
|
7343
|
+
function formatWikiReadyLine(state) {
|
|
7344
|
+
return `wiki ready, ${pageTotal(state)} page${plural(pageTotal(state))}, ${state.pendingCandidates} pending candidate${plural(state.pendingCandidates)}`;
|
|
7345
|
+
}
|
|
7346
|
+
function formatLintAttentionLine(state) {
|
|
7347
|
+
const entry = state.lint.entry;
|
|
7348
|
+
if (!entry) return "lint cache reports errors";
|
|
7349
|
+
return `lint reports ${entry.errors} error${plural(entry.errors)}, ${entry.warnings} warning${plural(entry.warnings)}`;
|
|
7350
|
+
}
|
|
7351
|
+
function pageTotal(state) {
|
|
7352
|
+
return state.conceptCount + state.queryCount;
|
|
7353
|
+
}
|
|
7354
|
+
function plural(count) {
|
|
7355
|
+
return count === 1 ? "" : "s";
|
|
7356
|
+
}
|
|
7357
|
+
|
|
7358
|
+
// src/commands/quickstart.ts
|
|
7359
|
+
import path46 from "path";
|
|
7360
|
+
|
|
7361
|
+
// src/utils/provider-guard.ts
|
|
7362
|
+
var PROVIDER_KEY_VARS = {
|
|
7363
|
+
anthropic: "ANTHROPIC_API_KEY",
|
|
7364
|
+
openai: "OPENAI_API_KEY",
|
|
7365
|
+
ollama: null,
|
|
7366
|
+
minimax: "MINIMAX_API_KEY",
|
|
7367
|
+
copilot: "GITHUB_TOKEN"
|
|
7368
|
+
};
|
|
7369
|
+
function ensureProviderAvailable() {
|
|
7370
|
+
const provider = process.env.LLMWIKI_PROVIDER ?? DEFAULT_PROVIDER;
|
|
7371
|
+
if (provider === "anthropic") {
|
|
7372
|
+
const auth = resolveAnthropicAuthFromEnv();
|
|
7373
|
+
if (!auth.apiKey && !auth.authToken) {
|
|
7374
|
+
throw new Error(
|
|
7375
|
+
`Anthropic credentials are required for the "anthropic" provider.
|
|
7376
|
+
Set one of: export ANTHROPIC_API_KEY=<your-key> OR export ANTHROPIC_AUTH_TOKEN=<your-token>`
|
|
7377
|
+
);
|
|
7378
|
+
}
|
|
7379
|
+
return;
|
|
7380
|
+
}
|
|
7381
|
+
const keyVar = PROVIDER_KEY_VARS[provider];
|
|
7382
|
+
if (keyVar === void 0) {
|
|
7383
|
+
throw new Error(
|
|
7384
|
+
`Unknown provider "${provider}".
|
|
7385
|
+
Supported: ${Object.keys(PROVIDER_KEY_VARS).join(", ")}`
|
|
7386
|
+
);
|
|
7387
|
+
}
|
|
7388
|
+
if (keyVar && !process.env[keyVar]) {
|
|
7389
|
+
throw new Error(
|
|
7390
|
+
`${keyVar} environment variable is required for the "${provider}" provider.
|
|
7391
|
+
Set it with: export ${keyVar}=<your-key>`
|
|
7392
|
+
);
|
|
7393
|
+
}
|
|
7394
|
+
}
|
|
7395
|
+
|
|
7396
|
+
// src/commands/quickstart.ts
|
|
7397
|
+
var QUICKSTART_JSON_VERSION = 1;
|
|
7398
|
+
var VIEW_OPEN_ARGS_KEY = "view\0--open";
|
|
7399
|
+
var NOOP_RESTORE = () => {
|
|
7400
|
+
};
|
|
7401
|
+
async function quickstartCommand(source2, options = {}) {
|
|
7402
|
+
const jsonMode = options.json === true;
|
|
7403
|
+
setQuiet(jsonMode);
|
|
7404
|
+
const restoreEnv = applyEnvOverrides(options);
|
|
7405
|
+
try {
|
|
7406
|
+
return await runQuickstart(source2, options, jsonMode);
|
|
7407
|
+
} finally {
|
|
7408
|
+
restoreEnv();
|
|
7409
|
+
setQuiet(false);
|
|
7410
|
+
}
|
|
7411
|
+
}
|
|
7412
|
+
async function runQuickstart(source2, options, jsonMode) {
|
|
7413
|
+
const root = process.cwd();
|
|
7414
|
+
const ingest2 = await runIngestStep(source2);
|
|
7415
|
+
if (!ingest2.ok) {
|
|
7416
|
+
return finalizeFailure({ source: source2, ingest: ingest2, jsonMode });
|
|
7417
|
+
}
|
|
7418
|
+
const compile2 = await runCompileStep(root, options.review === true);
|
|
7419
|
+
const viewer = buildViewerEnvelope();
|
|
7420
|
+
const run = { source: source2, ingest: ingest2, compile: compile2, viewer };
|
|
7421
|
+
return await finalizeSuccess(run, options, jsonMode, root);
|
|
7422
|
+
}
|
|
7423
|
+
function applyEnvOverrides(options) {
|
|
7424
|
+
return combineRestorers([
|
|
7425
|
+
applyProviderOverride(options.provider),
|
|
7426
|
+
applyLanguageOverride(options.lang)
|
|
7427
|
+
]);
|
|
7428
|
+
}
|
|
7429
|
+
function combineRestorers(restorers) {
|
|
7430
|
+
return () => {
|
|
7431
|
+
for (const restore of restorers) restore();
|
|
7432
|
+
};
|
|
7433
|
+
}
|
|
7434
|
+
function applyProviderOverride(provider) {
|
|
7435
|
+
const trimmed = provider?.trim();
|
|
7436
|
+
if (!trimmed) return NOOP_RESTORE;
|
|
7437
|
+
const restore = snapshotEnv("LLMWIKI_PROVIDER");
|
|
7438
|
+
process.env.LLMWIKI_PROVIDER = trimmed;
|
|
7439
|
+
return restore;
|
|
7440
|
+
}
|
|
7441
|
+
function applyLanguageOverride(lang) {
|
|
7442
|
+
const trimmed = lang?.trim();
|
|
7443
|
+
if (!trimmed) return NOOP_RESTORE;
|
|
7444
|
+
const restore = snapshotEnv("LLMWIKI_OUTPUT_LANG");
|
|
7445
|
+
applyLanguageOption(trimmed);
|
|
7446
|
+
return restore;
|
|
7447
|
+
}
|
|
7448
|
+
function snapshotEnv(name) {
|
|
7449
|
+
const previous = process.env[name];
|
|
7450
|
+
return () => {
|
|
7451
|
+
if (previous === void 0) {
|
|
7452
|
+
delete process.env[name];
|
|
7453
|
+
} else {
|
|
7454
|
+
process.env[name] = previous;
|
|
7455
|
+
}
|
|
7456
|
+
};
|
|
7457
|
+
}
|
|
7458
|
+
async function runIngestStep(source2) {
|
|
7459
|
+
header("llmwiki quickstart");
|
|
7460
|
+
status("*", info(`Ingesting ${source2}`));
|
|
7461
|
+
try {
|
|
7462
|
+
const result = await ingestSource(source2);
|
|
7463
|
+
const relPath = path46.join(SOURCES_DIR, result.filename);
|
|
7464
|
+
status("+", success(`Ingested \u2192 ${relPath}`));
|
|
7465
|
+
return buildIngestSuccess(result, relPath);
|
|
7466
|
+
} catch (err) {
|
|
7467
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
7468
|
+
status("!", error(`Ingest failed: ${message}`));
|
|
7469
|
+
return buildIngestFailure(message);
|
|
7470
|
+
}
|
|
7471
|
+
}
|
|
7472
|
+
function buildIngestSuccess(result, relPath) {
|
|
7473
|
+
return {
|
|
7474
|
+
ok: true,
|
|
7475
|
+
path: relPath,
|
|
7476
|
+
sourceType: result.sourceType ?? null,
|
|
7477
|
+
error: null
|
|
7478
|
+
};
|
|
7479
|
+
}
|
|
7480
|
+
function buildIngestFailure(message) {
|
|
7481
|
+
return {
|
|
7482
|
+
ok: false,
|
|
7483
|
+
path: null,
|
|
7484
|
+
sourceType: null,
|
|
7485
|
+
error: { code: "ingest_failed", message, recoverable: false }
|
|
7486
|
+
};
|
|
7487
|
+
}
|
|
7488
|
+
function emptyCompileEnvelope() {
|
|
7489
|
+
return {
|
|
7490
|
+
ok: false,
|
|
7491
|
+
compiled: 0,
|
|
7492
|
+
skipped: 0,
|
|
7493
|
+
deleted: 0,
|
|
7494
|
+
pendingCandidates: 0,
|
|
7495
|
+
errors: null,
|
|
7496
|
+
error: null
|
|
7497
|
+
};
|
|
7498
|
+
}
|
|
7499
|
+
var COMPILE_ERROR_LABEL = {
|
|
7500
|
+
provider_unavailable: "Compile prerequisite",
|
|
7501
|
+
compile_failed: "Compile"
|
|
7502
|
+
};
|
|
7503
|
+
async function buildCompileFailureEnvelope(root, code, err) {
|
|
7504
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
7505
|
+
status("!", error(`${COMPILE_ERROR_LABEL[code]} failed: ${message}`));
|
|
7506
|
+
return {
|
|
7507
|
+
...emptyCompileEnvelope(),
|
|
7508
|
+
pendingCandidates: await safeCountCandidates2(root),
|
|
7509
|
+
error: { code, message, recoverable: true }
|
|
7510
|
+
};
|
|
7511
|
+
}
|
|
7512
|
+
async function runCompileStep(root, review) {
|
|
7513
|
+
try {
|
|
7514
|
+
ensureProviderAvailable();
|
|
7515
|
+
} catch (err) {
|
|
7516
|
+
return await buildCompileFailureEnvelope(root, "provider_unavailable", err);
|
|
7517
|
+
}
|
|
7518
|
+
try {
|
|
7519
|
+
const result = await compileAndReport(root, { review });
|
|
7520
|
+
return await buildCompileEnvelopeFromResult(root, result);
|
|
7521
|
+
} catch (err) {
|
|
7522
|
+
return await buildCompileFailureEnvelope(root, "compile_failed", err);
|
|
7523
|
+
}
|
|
7524
|
+
}
|
|
7525
|
+
async function buildCompileEnvelopeFromResult(root, result) {
|
|
7526
|
+
const pendingCandidates = await safeCountCandidates2(root);
|
|
7527
|
+
const hasErrors = result.errors.length > 0;
|
|
7528
|
+
return {
|
|
7529
|
+
ok: !hasErrors,
|
|
7530
|
+
compiled: result.compiled,
|
|
7531
|
+
skipped: result.skipped,
|
|
7532
|
+
deleted: result.deleted,
|
|
7533
|
+
pendingCandidates,
|
|
7534
|
+
errors: hasErrors ? [...result.errors] : [],
|
|
7535
|
+
error: null
|
|
7536
|
+
};
|
|
7537
|
+
}
|
|
7538
|
+
async function safeCountCandidates2(root) {
|
|
7539
|
+
try {
|
|
7540
|
+
return await countCandidates(root);
|
|
7541
|
+
} catch {
|
|
7542
|
+
return 0;
|
|
7543
|
+
}
|
|
7544
|
+
}
|
|
7545
|
+
function buildViewerEnvelope() {
|
|
7546
|
+
return { opened: false, url: null };
|
|
7547
|
+
}
|
|
7548
|
+
function finalizeFailure(ctx) {
|
|
7549
|
+
const compile2 = emptyCompileEnvelope();
|
|
7550
|
+
const next = {
|
|
7551
|
+
command: `llmwiki ingest ${ctx.source}`,
|
|
7552
|
+
reason: "Source could not be ingested. Inspect the input and rerun ingest.",
|
|
7553
|
+
executable: { binary: "llmwiki", args: ["ingest"], placeholders: ["source"] }
|
|
7554
|
+
};
|
|
7555
|
+
const envelope = {
|
|
7556
|
+
version: QUICKSTART_JSON_VERSION,
|
|
7557
|
+
source: ctx.source,
|
|
7558
|
+
ingest: ctx.ingest,
|
|
7559
|
+
compile: compile2,
|
|
7560
|
+
viewer: buildViewerEnvelope(),
|
|
7561
|
+
next
|
|
7562
|
+
};
|
|
7563
|
+
emitEnvelope(envelope, ctx.jsonMode);
|
|
7564
|
+
return 1;
|
|
7565
|
+
}
|
|
7566
|
+
async function finalizeSuccess(run, options, jsonMode, root) {
|
|
7567
|
+
const projectState = await safeCollectState(root);
|
|
7568
|
+
const next = deriveNextAction(run, options, projectState);
|
|
7569
|
+
const handoff = shouldStartViewer({ options, jsonMode, compile: run.compile, projectState });
|
|
7570
|
+
const envelope = buildSuccessEnvelope(run, suppressRedundantViewerNext(next, handoff));
|
|
7571
|
+
emitEnvelope(envelope, jsonMode);
|
|
7572
|
+
if (handoff) await handoffToViewer();
|
|
7573
|
+
return 0;
|
|
7574
|
+
}
|
|
7575
|
+
function buildSuccessEnvelope(run, next) {
|
|
7576
|
+
return {
|
|
7577
|
+
version: QUICKSTART_JSON_VERSION,
|
|
7578
|
+
source: run.source,
|
|
7579
|
+
ingest: run.ingest,
|
|
7580
|
+
compile: run.compile,
|
|
7581
|
+
viewer: run.viewer,
|
|
7582
|
+
next
|
|
7583
|
+
};
|
|
7584
|
+
}
|
|
7585
|
+
async function safeCollectState(root) {
|
|
7586
|
+
try {
|
|
7587
|
+
return await collectProjectState(root);
|
|
7588
|
+
} catch {
|
|
7589
|
+
return null;
|
|
7590
|
+
}
|
|
7591
|
+
}
|
|
7592
|
+
function deriveNextAction(run, options, projectState) {
|
|
7593
|
+
if (options.review === true && run.compile.ok) return reviewListAction();
|
|
7594
|
+
if (!run.compile.ok) return resumeCompileAction(run.compile);
|
|
7595
|
+
return postCompileRecommendation(projectState);
|
|
7596
|
+
}
|
|
7597
|
+
function reviewListAction() {
|
|
7598
|
+
return {
|
|
7599
|
+
command: "llmwiki review list",
|
|
7600
|
+
reason: "Generated candidates are waiting for review.",
|
|
7601
|
+
executable: { binary: "llmwiki", args: ["review", "list"] }
|
|
7602
|
+
};
|
|
7603
|
+
}
|
|
7604
|
+
function resumeCompileAction(compile2) {
|
|
7605
|
+
const reason = compile2.error ? "Source was ingested, but compile did not complete." : "Compile reported errors. Address them and rerun compile.";
|
|
7606
|
+
return {
|
|
7607
|
+
command: "llmwiki compile",
|
|
7608
|
+
reason,
|
|
7609
|
+
executable: { binary: "llmwiki", args: ["compile"] }
|
|
7610
|
+
};
|
|
7611
|
+
}
|
|
7612
|
+
function postCompileRecommendation(projectState) {
|
|
7613
|
+
if (!projectState) {
|
|
7614
|
+
return {
|
|
7615
|
+
command: "llmwiki view --open",
|
|
7616
|
+
reason: "Wiki pages are ready to browse.",
|
|
7617
|
+
executable: { binary: "llmwiki", args: ["view", "--open"] }
|
|
7618
|
+
};
|
|
7619
|
+
}
|
|
7620
|
+
const { recommended } = recommendNextAction(projectState);
|
|
7621
|
+
return {
|
|
7622
|
+
command: recommended.command,
|
|
7623
|
+
reason: recommended.reason,
|
|
7624
|
+
executable: recommended.executable
|
|
7625
|
+
};
|
|
7626
|
+
}
|
|
7627
|
+
var VIEWER_HANDOFF_BLOCKERS = [
|
|
7628
|
+
(gate) => gate.jsonMode,
|
|
7629
|
+
(gate) => gate.options.open === false,
|
|
7630
|
+
(gate) => gate.options.review === true,
|
|
7631
|
+
(gate) => !gate.compile.ok,
|
|
7632
|
+
(gate) => !hasRenderablePages(gate.projectState)
|
|
7633
|
+
];
|
|
7634
|
+
function shouldStartViewer(gate) {
|
|
7635
|
+
return VIEWER_HANDOFF_BLOCKERS.every((isBlocked) => !isBlocked(gate));
|
|
7636
|
+
}
|
|
7637
|
+
function hasRenderablePages(state) {
|
|
7638
|
+
return conceptCountOf(state) + queryCountOf(state) > 0;
|
|
7639
|
+
}
|
|
7640
|
+
function conceptCountOf(state) {
|
|
7641
|
+
return state === null ? 0 : state.conceptCount;
|
|
7642
|
+
}
|
|
7643
|
+
function queryCountOf(state) {
|
|
7644
|
+
return state === null ? 0 : state.queryCount;
|
|
7645
|
+
}
|
|
7646
|
+
function suppressRedundantViewerNext(next, handoff) {
|
|
7647
|
+
if (!handoff) return next;
|
|
7648
|
+
if (!isViewOpenAction(next)) return next;
|
|
7649
|
+
return { command: null, reason: next.reason, executable: null };
|
|
7650
|
+
}
|
|
7651
|
+
function isViewOpenAction(next) {
|
|
7652
|
+
return next.executable?.args.join("\0") === VIEW_OPEN_ARGS_KEY;
|
|
7653
|
+
}
|
|
7654
|
+
async function handoffToViewer() {
|
|
7655
|
+
process.stdout.write("\nStarting viewer. Press Ctrl+C to stop.\n");
|
|
7656
|
+
await viewCommand({ open: true });
|
|
7657
|
+
}
|
|
7658
|
+
function emitEnvelope(envelope, jsonMode) {
|
|
7659
|
+
if (jsonMode) {
|
|
7660
|
+
process.stdout.write(`${JSON.stringify(envelope, null, 2)}
|
|
7661
|
+
`);
|
|
7662
|
+
return;
|
|
7663
|
+
}
|
|
7664
|
+
renderHuman2(envelope);
|
|
7665
|
+
}
|
|
7666
|
+
function renderHuman2(envelope) {
|
|
7667
|
+
const lines = [];
|
|
7668
|
+
appendIngestLine(lines, envelope.ingest);
|
|
7669
|
+
appendCompileLines(lines, envelope.compile);
|
|
7670
|
+
appendNextLines(lines, envelope.next);
|
|
7671
|
+
process.stdout.write(`
|
|
7672
|
+
${lines.join("\n")}
|
|
7673
|
+
`);
|
|
7674
|
+
}
|
|
7675
|
+
function appendIngestLine(lines, ingest2) {
|
|
7676
|
+
if (ingest2.ok && ingest2.path) {
|
|
7677
|
+
lines.push(`1. Ingested source \u2192 ${ingest2.path}`);
|
|
7678
|
+
return;
|
|
7679
|
+
}
|
|
7680
|
+
lines.push("1. Ingest failed \u2014 see error above.");
|
|
7681
|
+
}
|
|
7682
|
+
function appendCompileLines(lines, compile2) {
|
|
7683
|
+
const rule = COMPILE_LINE_RULES.find((candidate) => candidate.matches(compile2));
|
|
7684
|
+
if (rule) lines.push(rule.render(compile2));
|
|
7685
|
+
}
|
|
7686
|
+
var COMPILE_LINE_RULES = [
|
|
7687
|
+
{
|
|
7688
|
+
matches: (compile2) => compile2.error !== null,
|
|
7689
|
+
render: () => "2. Compile did not complete."
|
|
7690
|
+
},
|
|
7691
|
+
{
|
|
7692
|
+
matches: (compile2) => (compile2.errors?.length ?? 0) > 0,
|
|
7693
|
+
render: (compile2) => `2. Compile reported ${compile2.errors?.length ?? 0} error(s).`
|
|
7694
|
+
},
|
|
7695
|
+
{
|
|
7696
|
+
matches: (compile2) => compile2.ok && compile2.pendingCandidates > 0,
|
|
7697
|
+
render: (compile2) => `2. Compiled review candidates \u2192 ${compile2.pendingCandidates} pending`
|
|
7698
|
+
},
|
|
7699
|
+
{
|
|
7700
|
+
matches: (compile2) => compile2.ok,
|
|
7701
|
+
render: (compile2) => `2. Compiled wiki \u2192 ${compile2.compiled} new, ${compile2.skipped} skipped`
|
|
7702
|
+
}
|
|
7703
|
+
];
|
|
7704
|
+
function appendNextLines(lines, next) {
|
|
7705
|
+
if (!next.command) return;
|
|
7706
|
+
lines.push("");
|
|
7707
|
+
lines.push("Next:");
|
|
7708
|
+
lines.push(` ${next.command}`);
|
|
7709
|
+
}
|
|
7710
|
+
|
|
7711
|
+
// src/commands/context.ts
|
|
7712
|
+
import path48 from "path";
|
|
7713
|
+
|
|
7714
|
+
// src/context/provenance.ts
|
|
7715
|
+
import { promises as fs } from "fs";
|
|
7716
|
+
import path47 from "path";
|
|
7717
|
+
var MAX_SOURCE_WINDOWS = 20;
|
|
7718
|
+
var MAX_LINES_PER_WINDOW = 30;
|
|
7719
|
+
var CITATION_KEY_SEPARATOR = " <#> ";
|
|
7720
|
+
function flattenCitations(citations) {
|
|
7721
|
+
const out = [];
|
|
7722
|
+
const seen = /* @__PURE__ */ new Set();
|
|
7723
|
+
for (const citation of citations) {
|
|
7724
|
+
for (const span of citation.spans) {
|
|
7725
|
+
const flat = toFlatCitation(span);
|
|
7726
|
+
const key = citationKey(flat);
|
|
7727
|
+
if (seen.has(key)) continue;
|
|
7728
|
+
seen.add(key);
|
|
7729
|
+
out.push(flat);
|
|
7730
|
+
}
|
|
7731
|
+
}
|
|
7732
|
+
return out;
|
|
7733
|
+
}
|
|
7734
|
+
function toFlatCitation(span) {
|
|
7735
|
+
if (!span.lines) return { file: span.file };
|
|
7736
|
+
return { file: span.file, start: span.lines.start, end: span.lines.end };
|
|
7737
|
+
}
|
|
7738
|
+
function citationKey(citation) {
|
|
7739
|
+
const start = citation.start ?? "-";
|
|
7740
|
+
const end = citation.end ?? "-";
|
|
7741
|
+
return [citation.file, String(start), String(end)].join(CITATION_KEY_SEPARATOR);
|
|
7742
|
+
}
|
|
7743
|
+
function createSourceWindowBudget() {
|
|
7744
|
+
return { remaining: MAX_SOURCE_WINDOWS };
|
|
7745
|
+
}
|
|
7746
|
+
async function materializeSourceWindows(root, citations, budget) {
|
|
7747
|
+
if (budget.remaining <= 0) return [];
|
|
7748
|
+
const windows = [];
|
|
7749
|
+
for (const citation of citations) {
|
|
7750
|
+
if (budget.remaining <= 0) break;
|
|
7751
|
+
if (citation.start === void 0 || citation.end === void 0) continue;
|
|
7752
|
+
const window = await readSourceWindow(root, citation);
|
|
7753
|
+
if (!window) continue;
|
|
7754
|
+
windows.push(window);
|
|
7755
|
+
budget.remaining -= 1;
|
|
7756
|
+
}
|
|
7757
|
+
return windows;
|
|
7758
|
+
}
|
|
7759
|
+
async function readSourceWindow(root, citation) {
|
|
7760
|
+
if (citation.start === void 0 || citation.end === void 0) return null;
|
|
7761
|
+
const sourcesRoot = await resolveSourcesRoot(root);
|
|
7762
|
+
if (!sourcesRoot) return null;
|
|
7763
|
+
const realPath = await resolveSafePath(sourcesRoot, citation.file);
|
|
7764
|
+
if (!realPath) return null;
|
|
7765
|
+
return readClampedWindow(realPath, citation);
|
|
7766
|
+
}
|
|
7767
|
+
async function resolveSourcesRoot(root) {
|
|
7768
|
+
const candidate = path47.join(root, SOURCES_DIR);
|
|
7769
|
+
try {
|
|
7770
|
+
return await fs.realpath(candidate);
|
|
7771
|
+
} catch {
|
|
7772
|
+
return null;
|
|
7773
|
+
}
|
|
7774
|
+
}
|
|
7775
|
+
async function resolveSafePath(sourcesRoot, file) {
|
|
7776
|
+
if (file.length === 0) return null;
|
|
7777
|
+
if (path47.isAbsolute(file)) return null;
|
|
7778
|
+
if (containsParentSegment2(file)) return null;
|
|
7779
|
+
const joined = path47.join(sourcesRoot, file);
|
|
7780
|
+
const resolved = path47.resolve(joined);
|
|
7781
|
+
if (!isInside2(sourcesRoot, resolved)) return null;
|
|
7782
|
+
try {
|
|
7783
|
+
const realPath = await fs.realpath(resolved);
|
|
7784
|
+
if (!isInside2(sourcesRoot, realPath)) return null;
|
|
7785
|
+
return realPath;
|
|
7786
|
+
} catch {
|
|
7787
|
+
return null;
|
|
7788
|
+
}
|
|
7789
|
+
}
|
|
7790
|
+
function containsParentSegment2(file) {
|
|
7791
|
+
const segments = file.split(/[/\\]/);
|
|
7792
|
+
return segments.some((segment) => segment === "..");
|
|
7793
|
+
}
|
|
7794
|
+
function isInside2(parent, candidate) {
|
|
7795
|
+
if (candidate === parent) return true;
|
|
7796
|
+
const normalizedParent = parent.endsWith(path47.sep) ? parent : `${parent}${path47.sep}`;
|
|
7797
|
+
return candidate.startsWith(normalizedParent);
|
|
7798
|
+
}
|
|
7799
|
+
async function readClampedWindow(realPath, citation) {
|
|
7800
|
+
if (citation.start === void 0 || citation.end === void 0) return null;
|
|
7801
|
+
let raw;
|
|
7802
|
+
try {
|
|
7803
|
+
raw = await fs.readFile(realPath, "utf-8");
|
|
7804
|
+
} catch {
|
|
7805
|
+
return null;
|
|
7806
|
+
}
|
|
7807
|
+
const lines = raw.split(/\r?\n/);
|
|
7808
|
+
const startIndex = Math.max(0, citation.start - 1);
|
|
7809
|
+
const inclusiveEnd = Math.min(lines.length, citation.end);
|
|
7810
|
+
if (startIndex >= inclusiveEnd) return null;
|
|
7811
|
+
const clampedEnd = Math.min(inclusiveEnd, startIndex + MAX_LINES_PER_WINDOW);
|
|
7812
|
+
const text = lines.slice(startIndex, clampedEnd).join("\n");
|
|
7813
|
+
return { file: citation.file, start: citation.start, end: startIndex + (clampedEnd - startIndex), text };
|
|
7814
|
+
}
|
|
7815
|
+
|
|
7816
|
+
// src/context/ranking.ts
|
|
7817
|
+
var WEIGHT_TITLE_MATCH = 0.5;
|
|
7818
|
+
var WEIGHT_BODY_MATCH = 0.3;
|
|
7819
|
+
var WEIGHT_EXACT_SLUG = 0.4;
|
|
7820
|
+
var WEIGHT_EXACT_TITLE = 0.5;
|
|
7821
|
+
var WEIGHT_SEMANTIC_CHUNK = 0.5;
|
|
7822
|
+
var WEIGHT_SEMANTIC_CHUNK_BONUS = 0.05;
|
|
7823
|
+
var MAX_SEMANTIC_BONUS_CHUNKS = 3;
|
|
7824
|
+
var MAX_NORMALIZED_SCORE = 1;
|
|
7825
|
+
function rankPages(snapshot, prompt, topN, semanticHits = []) {
|
|
7826
|
+
const rows = /* @__PURE__ */ new Map();
|
|
7827
|
+
applyLexicalSignals(rows, snapshot, prompt);
|
|
7828
|
+
applyExactSignals(rows, snapshot, prompt);
|
|
7829
|
+
applySemanticSignals(rows, snapshot, semanticHits);
|
|
7830
|
+
const sorted = Array.from(rows.values()).sort(compareRows);
|
|
7831
|
+
return sorted.slice(0, Math.max(0, topN)).map(rowToPrimary);
|
|
7832
|
+
}
|
|
7833
|
+
function applyLexicalSignals(rows, snapshot, prompt) {
|
|
7834
|
+
const { results } = searchPages(snapshot, prompt);
|
|
7835
|
+
for (const result of results) {
|
|
7836
|
+
const page = snapshot.pages.find((p) => p.id === result.id);
|
|
7837
|
+
if (!page) continue;
|
|
7838
|
+
const row = ensureRow(rows, page);
|
|
7839
|
+
row.snippet = row.snippet || result.snippet;
|
|
7840
|
+
if (result.matchedIn === "title") {
|
|
7841
|
+
addReason(row, "title-match", WEIGHT_TITLE_MATCH);
|
|
7842
|
+
} else {
|
|
7843
|
+
addReason(row, "body-match", WEIGHT_BODY_MATCH);
|
|
7844
|
+
}
|
|
7845
|
+
}
|
|
7846
|
+
}
|
|
7847
|
+
function applyExactSignals(rows, snapshot, prompt) {
|
|
7848
|
+
const normalized = prompt.trim().toLowerCase();
|
|
7849
|
+
if (normalized.length === 0) return;
|
|
7850
|
+
for (const page of snapshot.pages) {
|
|
7851
|
+
if (page.slug.toLowerCase() === normalized) {
|
|
7852
|
+
addReason(ensureRow(rows, page), "exact-slug", WEIGHT_EXACT_SLUG);
|
|
7853
|
+
}
|
|
7854
|
+
if (page.title.trim().toLowerCase() === normalized) {
|
|
7855
|
+
addReason(ensureRow(rows, page), "exact-title", WEIGHT_EXACT_TITLE);
|
|
7856
|
+
}
|
|
7857
|
+
}
|
|
7858
|
+
}
|
|
7859
|
+
function applySemanticSignals(rows, snapshot, hits) {
|
|
7860
|
+
if (hits.length === 0) return;
|
|
7861
|
+
const bySlug = groupHitsBySlug(hits);
|
|
7862
|
+
for (const [slug, slugHits] of bySlug) {
|
|
7863
|
+
const page = findPageBySlug(snapshot, slug);
|
|
7864
|
+
if (!page) continue;
|
|
7865
|
+
const row = ensureRow(rows, page);
|
|
7866
|
+
addReason(row, "semantic-chunk", WEIGHT_SEMANTIC_CHUNK);
|
|
7867
|
+
row.weight += semanticMultiChunkBonus(slugHits.length);
|
|
7868
|
+
for (const hit of slugHits) {
|
|
7869
|
+
row.chunks.push({
|
|
7870
|
+
text: hit.text,
|
|
7871
|
+
score: hit.score,
|
|
7872
|
+
contentHash: hit.contentHash
|
|
7873
|
+
});
|
|
7874
|
+
}
|
|
7875
|
+
}
|
|
7876
|
+
}
|
|
7877
|
+
function groupHitsBySlug(hits) {
|
|
7878
|
+
const bySlug = /* @__PURE__ */ new Map();
|
|
7879
|
+
for (const hit of hits) {
|
|
7880
|
+
const existing = bySlug.get(hit.slug);
|
|
7881
|
+
if (existing) existing.push(hit);
|
|
7882
|
+
else bySlug.set(hit.slug, [hit]);
|
|
7883
|
+
}
|
|
7884
|
+
return bySlug;
|
|
7885
|
+
}
|
|
7886
|
+
function semanticMultiChunkBonus(chunkCount) {
|
|
7887
|
+
const extra = Math.max(0, Math.min(chunkCount - 1, MAX_SEMANTIC_BONUS_CHUNKS));
|
|
7888
|
+
return extra * WEIGHT_SEMANTIC_CHUNK_BONUS;
|
|
7889
|
+
}
|
|
7890
|
+
function findPageBySlug(snapshot, slug) {
|
|
7891
|
+
const concept = snapshot.pages.find(
|
|
7892
|
+
(p) => p.pageDirectory === "concepts" && p.slug === slug
|
|
7893
|
+
);
|
|
7894
|
+
if (concept) return concept;
|
|
7895
|
+
const query = snapshot.pages.find(
|
|
7896
|
+
(p) => p.pageDirectory === "queries" && p.slug === slug
|
|
7897
|
+
);
|
|
7898
|
+
return query ?? null;
|
|
7899
|
+
}
|
|
7900
|
+
function ensureRow(rows, page) {
|
|
7901
|
+
const existing = rows.get(page.id);
|
|
7902
|
+
if (existing) return existing;
|
|
7903
|
+
const created = {
|
|
7904
|
+
page,
|
|
7905
|
+
reasons: /* @__PURE__ */ new Set(),
|
|
7906
|
+
weight: 0,
|
|
7907
|
+
snippet: "",
|
|
7908
|
+
chunks: []
|
|
7909
|
+
};
|
|
7910
|
+
rows.set(page.id, created);
|
|
7911
|
+
return created;
|
|
7912
|
+
}
|
|
7913
|
+
function addReason(row, reason, weight) {
|
|
7914
|
+
row.reasons.add(reason);
|
|
7915
|
+
row.weight += weight;
|
|
7916
|
+
}
|
|
7917
|
+
function compareRows(a, b) {
|
|
7918
|
+
if (a.weight !== b.weight) return b.weight - a.weight;
|
|
7919
|
+
const byTitle = a.page.title.localeCompare(b.page.title);
|
|
7920
|
+
if (byTitle !== 0) return byTitle;
|
|
7921
|
+
return a.page.id.localeCompare(b.page.id);
|
|
7922
|
+
}
|
|
7923
|
+
function rowToPrimary(row) {
|
|
7924
|
+
return {
|
|
7925
|
+
id: row.page.id,
|
|
7926
|
+
title: row.page.title,
|
|
7927
|
+
pageDirectory: row.page.pageDirectory,
|
|
7928
|
+
score: normalizeWeight(row.weight),
|
|
7929
|
+
reasons: Array.from(row.reasons).sort(),
|
|
7930
|
+
summary: row.snippet,
|
|
7931
|
+
chunks: row.chunks,
|
|
7932
|
+
citations: flattenCitations(row.page.citations),
|
|
7933
|
+
sourceWindows: [],
|
|
7934
|
+
warnings: row.page.warnings.map((w) => ({ code: w.code, message: w.message }))
|
|
7935
|
+
};
|
|
7936
|
+
}
|
|
7937
|
+
function normalizeWeight(weight) {
|
|
7938
|
+
if (weight <= 0) return 0;
|
|
7939
|
+
if (weight >= MAX_NORMALIZED_SCORE) return MAX_NORMALIZED_SCORE;
|
|
7940
|
+
return Math.round(weight * 100) / 100;
|
|
7941
|
+
}
|
|
7942
|
+
|
|
7943
|
+
// src/context/retrieval.ts
|
|
7944
|
+
async function retrieveSemanticChunks(root, prompt, topChunks) {
|
|
7945
|
+
if (topChunks <= 0) return emptyOutcome(null);
|
|
7946
|
+
if (await isStoreUnusable(root)) return emptyOutcome("embedding-store-missing");
|
|
7947
|
+
let raw;
|
|
7948
|
+
try {
|
|
7949
|
+
raw = await findRelevantChunks(root, prompt, topChunks);
|
|
7950
|
+
} catch (err) {
|
|
7951
|
+
return emptyOutcome(classifyRetrievalError(err));
|
|
7952
|
+
}
|
|
7953
|
+
if (raw.length === 0) {
|
|
7954
|
+
return emptyOutcome("embedding-store-missing");
|
|
7955
|
+
}
|
|
7956
|
+
return { hits: raw.map(toSemanticChunkHit), warning: null };
|
|
7957
|
+
}
|
|
7958
|
+
function emptyOutcome(warning) {
|
|
7959
|
+
return { hits: [], warning };
|
|
7960
|
+
}
|
|
7961
|
+
async function isStoreUnusable(root) {
|
|
7962
|
+
const store = await tryReadEmbeddingStore(root);
|
|
7963
|
+
if (!store) return true;
|
|
7964
|
+
if (!store.chunks || store.chunks.length === 0) return true;
|
|
7965
|
+
if (isStaleModel(store)) return true;
|
|
7966
|
+
return false;
|
|
7967
|
+
}
|
|
7968
|
+
async function tryReadEmbeddingStore(root) {
|
|
7969
|
+
try {
|
|
7970
|
+
return await readEmbeddingStore(root);
|
|
7971
|
+
} catch {
|
|
7972
|
+
return null;
|
|
7973
|
+
}
|
|
7974
|
+
}
|
|
7975
|
+
function isStaleModel(store) {
|
|
7976
|
+
try {
|
|
7977
|
+
return store.model !== resolveEmbeddingModel();
|
|
7978
|
+
} catch {
|
|
7979
|
+
return true;
|
|
7980
|
+
}
|
|
7981
|
+
}
|
|
7982
|
+
function classifyRetrievalError(err) {
|
|
7983
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
7984
|
+
return looksLikeProviderFailure(message) ? "query-embedding-unavailable" : "semantic-retrieval-error";
|
|
7985
|
+
}
|
|
7986
|
+
function looksLikeProviderFailure(message) {
|
|
7987
|
+
return /api[_ -]?key|auth|credential|token|provider|voyage|openai|ollama|timeout|fetch|econn|enotfound/i.test(message);
|
|
7988
|
+
}
|
|
7989
|
+
function toSemanticChunkHit(raw) {
|
|
7990
|
+
return {
|
|
7991
|
+
slug: raw.chunk.slug,
|
|
7992
|
+
text: raw.chunk.text,
|
|
7993
|
+
score: raw.score,
|
|
7994
|
+
contentHash: raw.chunk.contentHash
|
|
7995
|
+
};
|
|
7996
|
+
}
|
|
7997
|
+
|
|
7998
|
+
// src/context/graph.ts
|
|
7999
|
+
var NEIGHBOR_REASON_WIKILINK = "wikilink";
|
|
8000
|
+
var CANONICAL_PAIR_SEPARATOR = " <-> ";
|
|
8001
|
+
var MAX_GRAPH_NEIGHBORS = 20;
|
|
8002
|
+
var WEIGHT_NEIGHBOR_DIRECT = 0.5;
|
|
8003
|
+
var WEIGHT_NEIGHBOR_SECOND_HOP = 0.25;
|
|
8004
|
+
var WEIGHT_PRIMARY_CONNECTION_BONUS = 0.05;
|
|
8005
|
+
var MAX_PRIMARY_CONNECTION_BONUS_HITS = 3;
|
|
8006
|
+
var WEIGHT_SAME_KIND_BONUS = 0.03;
|
|
8007
|
+
var DEFAULT_PAGE_KIND = "concept";
|
|
8008
|
+
var MAX_NORMALIZED_SCORE2 = 1;
|
|
8009
|
+
function expandGraphNeighborhood(input) {
|
|
8010
|
+
if (input.depth <= 0 || input.primaryIds.size === 0) {
|
|
8011
|
+
return { neighbors: [], gaps: [] };
|
|
8012
|
+
}
|
|
8013
|
+
const adjacency = buildAdjacency(input.graph);
|
|
8014
|
+
const ghostIds = collectGhostIds(input.graph.nodes);
|
|
8015
|
+
const pageKinds = buildPageKindMap(input.pages);
|
|
8016
|
+
const depth1Raw = expandDepthOne({
|
|
8017
|
+
primaryIds: input.primaryIds,
|
|
8018
|
+
adjacency,
|
|
8019
|
+
ghostIds,
|
|
8020
|
+
pageKinds
|
|
8021
|
+
});
|
|
8022
|
+
const depth1 = depth1Raw.sort(compareNeighbors).slice(0, MAX_GRAPH_NEIGHBORS);
|
|
8023
|
+
const depth2 = input.depth >= 2 ? expandDepthTwo({
|
|
8024
|
+
primaryIds: input.primaryIds,
|
|
8025
|
+
adjacency,
|
|
8026
|
+
ghostIds,
|
|
8027
|
+
pageKinds,
|
|
8028
|
+
depthOneTargets: collectDepthOneTargets(depth1)
|
|
8029
|
+
}) : [];
|
|
8030
|
+
const neighbors = [...depth1, ...depth2].sort(compareNeighbors).slice(0, MAX_GRAPH_NEIGHBORS);
|
|
8031
|
+
return { neighbors, gaps: emitGapsFromPrimary(input) };
|
|
8032
|
+
}
|
|
8033
|
+
function emitGapsFromPrimary(input) {
|
|
8034
|
+
const gaps = [];
|
|
8035
|
+
for (const page of input.pages) {
|
|
8036
|
+
if (!input.primaryIds.has(page.id)) continue;
|
|
8037
|
+
for (const dangling of page.danglingLinks ?? []) {
|
|
8038
|
+
gaps.push({
|
|
8039
|
+
code: "dangling-link",
|
|
8040
|
+
message: `Page links to [[${dangling.display}]], but no page exists.`,
|
|
8041
|
+
pageId: page.id
|
|
8042
|
+
});
|
|
8043
|
+
}
|
|
8044
|
+
}
|
|
8045
|
+
return gaps;
|
|
8046
|
+
}
|
|
8047
|
+
function buildAdjacency(graph) {
|
|
8048
|
+
const outgoing = /* @__PURE__ */ new Map();
|
|
8049
|
+
const incoming = /* @__PURE__ */ new Map();
|
|
8050
|
+
for (const edge of graph.edges) {
|
|
8051
|
+
addToSetMap(outgoing, edge.source, edge.target);
|
|
8052
|
+
addToSetMap(incoming, edge.target, edge.source);
|
|
8053
|
+
}
|
|
8054
|
+
return { outgoing, incoming };
|
|
8055
|
+
}
|
|
8056
|
+
function addToSetMap(map, key, value) {
|
|
8057
|
+
const existing = map.get(key);
|
|
8058
|
+
if (existing) existing.add(value);
|
|
8059
|
+
else map.set(key, /* @__PURE__ */ new Set([value]));
|
|
8060
|
+
}
|
|
8061
|
+
function collectGhostIds(nodes) {
|
|
8062
|
+
const ghosts = /* @__PURE__ */ new Set();
|
|
8063
|
+
for (const node of nodes) if (node.isDangling) ghosts.add(node.id);
|
|
8064
|
+
return ghosts;
|
|
8065
|
+
}
|
|
8066
|
+
function expandDepthOne(input) {
|
|
8067
|
+
const emitted = /* @__PURE__ */ new Map();
|
|
8068
|
+
const connectionCount = /* @__PURE__ */ new Map();
|
|
8069
|
+
for (const primary of input.primaryIds) {
|
|
8070
|
+
addNeighborsForPrimary({ ...input, primary, emitted, connectionCount });
|
|
8071
|
+
}
|
|
8072
|
+
applyPrimaryConnectionBonus(emitted, connectionCount);
|
|
8073
|
+
return Array.from(emitted.values());
|
|
8074
|
+
}
|
|
8075
|
+
function addNeighborsForPrimary(ctx) {
|
|
8076
|
+
walkIncidentEdges(ctx.adjacency, ctx.primary, (other, direction) => {
|
|
8077
|
+
tryEmitDirect({ ctx, other, direction });
|
|
8078
|
+
});
|
|
8079
|
+
}
|
|
8080
|
+
function tryEmitDirect(input) {
|
|
8081
|
+
const { ctx, other, direction } = input;
|
|
8082
|
+
if (ctx.ghostIds.has(other)) return;
|
|
8083
|
+
if (ctx.primaryIds.has(other)) return;
|
|
8084
|
+
bumpConnection(ctx.connectionCount, other);
|
|
8085
|
+
mergeOrInsertNeighbor(ctx.emitted, {
|
|
8086
|
+
from: ctx.primary,
|
|
8087
|
+
to: other,
|
|
8088
|
+
direction,
|
|
8089
|
+
distance: 1,
|
|
8090
|
+
score: scoreWithSameKindBonus(
|
|
8091
|
+
WEIGHT_NEIGHBOR_DIRECT,
|
|
8092
|
+
ctx.primary,
|
|
8093
|
+
other,
|
|
8094
|
+
ctx.pageKinds
|
|
8095
|
+
)
|
|
8096
|
+
});
|
|
8097
|
+
}
|
|
8098
|
+
function bumpConnection(counter, target) {
|
|
8099
|
+
counter.set(target, (counter.get(target) ?? 0) + 1);
|
|
8100
|
+
}
|
|
8101
|
+
var EMPTY_NEIGHBOR_SET = /* @__PURE__ */ new Set();
|
|
8102
|
+
function walkIncidentEdges(adjacency, node, onEdge) {
|
|
8103
|
+
const outgoing = adjacency.outgoing.get(node) ?? EMPTY_NEIGHBOR_SET;
|
|
8104
|
+
const incoming = adjacency.incoming.get(node) ?? EMPTY_NEIGHBOR_SET;
|
|
8105
|
+
for (const target of outgoing) onEdge(target, "outgoing");
|
|
8106
|
+
for (const source2 of incoming) onEdge(source2, "incoming");
|
|
8107
|
+
}
|
|
8108
|
+
function mergeOrInsertNeighbor(emitted, candidate) {
|
|
8109
|
+
const key = canonicalPairKey(candidate.from, candidate.to);
|
|
8110
|
+
const existing = emitted.get(key);
|
|
8111
|
+
if (existing) {
|
|
8112
|
+
if (existing.direction === "incoming" && candidate.direction === "outgoing") {
|
|
8113
|
+
existing.direction = "outgoing";
|
|
8114
|
+
}
|
|
8115
|
+
return;
|
|
8116
|
+
}
|
|
8117
|
+
emitted.set(key, { ...candidate, reason: NEIGHBOR_REASON_WIKILINK });
|
|
8118
|
+
}
|
|
8119
|
+
function applyPrimaryConnectionBonus(emitted, connectionCount) {
|
|
8120
|
+
for (const neighbor of emitted.values()) {
|
|
8121
|
+
const hits = connectionCount.get(neighbor.to) ?? 0;
|
|
8122
|
+
const extraHits = Math.min(
|
|
8123
|
+
Math.max(0, hits - 1),
|
|
8124
|
+
MAX_PRIMARY_CONNECTION_BONUS_HITS
|
|
8125
|
+
);
|
|
8126
|
+
neighbor.score = clampScore(
|
|
8127
|
+
neighbor.score + extraHits * WEIGHT_PRIMARY_CONNECTION_BONUS
|
|
8128
|
+
);
|
|
8129
|
+
}
|
|
8130
|
+
}
|
|
8131
|
+
function expandDepthTwo(input) {
|
|
8132
|
+
const emitted = /* @__PURE__ */ new Map();
|
|
8133
|
+
for (const bridge of input.depthOneTargets) {
|
|
8134
|
+
walkDepthTwoFromBridge({ ...input, bridge, emitted });
|
|
8135
|
+
}
|
|
8136
|
+
return Array.from(emitted.values());
|
|
8137
|
+
}
|
|
8138
|
+
function walkDepthTwoFromBridge(ctx) {
|
|
8139
|
+
walkIncidentEdges(ctx.adjacency, ctx.bridge, (other, direction) => {
|
|
8140
|
+
tryEmitSecondHop({ ctx, other, direction });
|
|
8141
|
+
});
|
|
8142
|
+
}
|
|
8143
|
+
function tryEmitSecondHop(input) {
|
|
8144
|
+
const { ctx, other, direction } = input;
|
|
8145
|
+
if (ctx.ghostIds.has(other)) return;
|
|
8146
|
+
if (ctx.primaryIds.has(other)) return;
|
|
8147
|
+
if (ctx.depthOneTargets.has(other)) return;
|
|
8148
|
+
mergeOrInsertNeighbor(ctx.emitted, {
|
|
8149
|
+
from: ctx.bridge,
|
|
8150
|
+
to: other,
|
|
8151
|
+
direction,
|
|
8152
|
+
distance: 2,
|
|
8153
|
+
score: scoreWithSameKindBonus(
|
|
8154
|
+
WEIGHT_NEIGHBOR_SECOND_HOP,
|
|
8155
|
+
ctx.bridge,
|
|
8156
|
+
other,
|
|
8157
|
+
ctx.pageKinds
|
|
8158
|
+
)
|
|
8159
|
+
});
|
|
8160
|
+
}
|
|
8161
|
+
function buildPageKindMap(pages) {
|
|
8162
|
+
const kinds = /* @__PURE__ */ new Map();
|
|
8163
|
+
for (const page of pages) {
|
|
8164
|
+
const kind = page.frontmatter.kind;
|
|
8165
|
+
kinds.set(page.id, typeof kind === "string" && kind.length > 0 ? kind : DEFAULT_PAGE_KIND);
|
|
8166
|
+
}
|
|
8167
|
+
return kinds;
|
|
8168
|
+
}
|
|
8169
|
+
function scoreWithSameKindBonus(base, from, to, pageKinds) {
|
|
8170
|
+
return samePageKind(from, to, pageKinds) ? clampScore(base + WEIGHT_SAME_KIND_BONUS) : base;
|
|
8171
|
+
}
|
|
8172
|
+
function samePageKind(from, to, pageKinds) {
|
|
8173
|
+
const fromKind = pageKinds.get(from);
|
|
8174
|
+
const toKind = pageKinds.get(to);
|
|
8175
|
+
return fromKind !== void 0 && toKind !== void 0 && fromKind === toKind;
|
|
8176
|
+
}
|
|
8177
|
+
function collectDepthOneTargets(neighbors) {
|
|
8178
|
+
const ids = /* @__PURE__ */ new Set();
|
|
8179
|
+
for (const n of neighbors) ids.add(n.to);
|
|
8180
|
+
return ids;
|
|
8181
|
+
}
|
|
8182
|
+
function canonicalPairKey(a, b) {
|
|
8183
|
+
return a < b ? `${a}${CANONICAL_PAIR_SEPARATOR}${b}` : `${b}${CANONICAL_PAIR_SEPARATOR}${a}`;
|
|
8184
|
+
}
|
|
8185
|
+
function clampScore(weight) {
|
|
8186
|
+
if (weight <= 0) return 0;
|
|
8187
|
+
if (weight >= MAX_NORMALIZED_SCORE2) return MAX_NORMALIZED_SCORE2;
|
|
8188
|
+
return Math.round(weight * 100) / 100;
|
|
8189
|
+
}
|
|
8190
|
+
function compareNeighbors(a, b) {
|
|
8191
|
+
if (a.score !== b.score) return b.score - a.score;
|
|
8192
|
+
return a.to.localeCompare(b.to);
|
|
8193
|
+
}
|
|
8194
|
+
|
|
8195
|
+
// src/context/budget.ts
|
|
8196
|
+
var APPROX_CHARS_PER_TOKEN = 4;
|
|
8197
|
+
function estimateTokens(text) {
|
|
8198
|
+
if (text === null || text === void 0) return 0;
|
|
8199
|
+
const stringified = typeof text === "string" ? text : String(text);
|
|
8200
|
+
if (stringified.length === 0) return 0;
|
|
8201
|
+
return Math.ceil(stringified.length / APPROX_CHARS_PER_TOKEN);
|
|
8202
|
+
}
|
|
8203
|
+
function estimatePackTokens(pack) {
|
|
8204
|
+
return estimateTokens(JSON.stringify(pack));
|
|
8205
|
+
}
|
|
8206
|
+
function buildBudget(requestedTokens, estimatedTokens) {
|
|
8207
|
+
return {
|
|
8208
|
+
requestedTokens,
|
|
8209
|
+
estimatedTokens,
|
|
8210
|
+
truncated: false,
|
|
8211
|
+
trimmedSections: []
|
|
8212
|
+
};
|
|
8213
|
+
}
|
|
8214
|
+
function trimToBudget(pack, requestedTokens) {
|
|
8215
|
+
const clone = clonePack(pack);
|
|
8216
|
+
const trimmed = /* @__PURE__ */ new Set();
|
|
8217
|
+
if (estimatePackTokens(clone) <= requestedTokens) {
|
|
8218
|
+
return { pack: clone, trimmedSections: [] };
|
|
8219
|
+
}
|
|
8220
|
+
trimNeighbors(clone, requestedTokens, trimmed);
|
|
8221
|
+
trimSourceWindows(clone, requestedTokens, trimmed);
|
|
8222
|
+
trimChunks(clone, requestedTokens, trimmed);
|
|
8223
|
+
trimPrimary(clone, requestedTokens, trimmed);
|
|
8224
|
+
return { pack: clone, trimmedSections: orderedSections(trimmed) };
|
|
8225
|
+
}
|
|
8226
|
+
function clonePack(pack) {
|
|
8227
|
+
return structuredClone(pack);
|
|
8228
|
+
}
|
|
8229
|
+
function orderedSections(trimmed) {
|
|
8230
|
+
const order = ["neighbors", "sourceWindows", "chunks", "primary"];
|
|
8231
|
+
return order.filter((section) => trimmed.has(section));
|
|
8232
|
+
}
|
|
8233
|
+
function trimNeighbors(pack, budget, trimmed) {
|
|
8234
|
+
while (pack.neighbors.length > 0 && estimatePackTokens(pack) > budget) {
|
|
8235
|
+
pack.neighbors.pop();
|
|
8236
|
+
trimmed.add("neighbors");
|
|
8237
|
+
}
|
|
8238
|
+
}
|
|
8239
|
+
function trimSourceWindows(pack, budget, trimmed) {
|
|
8240
|
+
for (let i = pack.primary.length - 1; i >= 0; i--) {
|
|
8241
|
+
while (pack.primary[i].sourceWindows.length > 0 && estimatePackTokens(pack) > budget) {
|
|
8242
|
+
pack.primary[i].sourceWindows.pop();
|
|
8243
|
+
trimmed.add("sourceWindows");
|
|
8244
|
+
}
|
|
8245
|
+
if (estimatePackTokens(pack) <= budget) return;
|
|
8246
|
+
}
|
|
8247
|
+
}
|
|
8248
|
+
function trimChunks(pack, budget, trimmed) {
|
|
8249
|
+
for (let i = pack.primary.length - 1; i >= 0; i--) {
|
|
8250
|
+
while (pack.primary[i].chunks.length > 0 && estimatePackTokens(pack) > budget) {
|
|
8251
|
+
pack.primary[i].chunks.pop();
|
|
8252
|
+
trimmed.add("chunks");
|
|
8253
|
+
}
|
|
8254
|
+
if (estimatePackTokens(pack) <= budget) return;
|
|
8255
|
+
}
|
|
8256
|
+
}
|
|
8257
|
+
function trimPrimary(pack, budget, trimmed) {
|
|
8258
|
+
while (pack.primary.length > 0 && estimatePackTokens(pack) > budget) {
|
|
8259
|
+
pack.primary.pop();
|
|
8260
|
+
trimmed.add("primary");
|
|
8261
|
+
}
|
|
8262
|
+
}
|
|
8263
|
+
|
|
8264
|
+
// src/context/types.ts
|
|
8265
|
+
var PROMPT_ECHO_MAX_LENGTH = 1024;
|
|
8266
|
+
var DEFAULT_BUDGET_TOKENS = 8e3;
|
|
8267
|
+
var DEFAULT_DEPTH = 1;
|
|
8268
|
+
var MAX_DEPTH = 2;
|
|
8269
|
+
var DEFAULT_TOP_PAGES = 5;
|
|
8270
|
+
var MAX_TOP_PAGES = 20;
|
|
8271
|
+
var DEFAULT_TOP_CHUNKS = 8;
|
|
8272
|
+
var MAX_TOP_CHUNKS = 50;
|
|
8273
|
+
|
|
8274
|
+
// src/context/build.ts
|
|
8275
|
+
async function buildContextPack(options) {
|
|
8276
|
+
const normalized = normalizeOptions(options);
|
|
8277
|
+
const snapshot = await buildViewerSnapshot(options.root);
|
|
8278
|
+
const state = await collectProjectState(options.root);
|
|
8279
|
+
const recommendation = recommendNextAction(state);
|
|
8280
|
+
const semantic = await retrieveSemanticChunks(
|
|
8281
|
+
options.root,
|
|
8282
|
+
normalized.rankingPrompt,
|
|
8283
|
+
normalized.topChunks
|
|
8284
|
+
);
|
|
8285
|
+
const draft = assembleDraft({
|
|
8286
|
+
snapshot,
|
|
8287
|
+
state,
|
|
8288
|
+
recommendation,
|
|
8289
|
+
options: normalized,
|
|
8290
|
+
semantic
|
|
8291
|
+
});
|
|
8292
|
+
const withSources = normalized.includeSources ? await attachSourceWindows(draft, options.root) : draft;
|
|
8293
|
+
const withProjectWarnings = appendProjectWarnings(withSources, state, normalized);
|
|
8294
|
+
const graph = normalized.neighborsEnabled && normalized.depth >= 1 ? snapshot.graph : null;
|
|
8295
|
+
return finalizeBudget(withProjectWarnings, normalized.budget, graph);
|
|
8296
|
+
}
|
|
8297
|
+
async function attachSourceWindows(pack, root) {
|
|
8298
|
+
const budget = createSourceWindowBudget();
|
|
8299
|
+
const primary = [];
|
|
8300
|
+
for (const entry of pack.primary) {
|
|
8301
|
+
const windows = await materializeSourceWindows(root, entry.citations, budget);
|
|
8302
|
+
primary.push({ ...entry, sourceWindows: windows });
|
|
8303
|
+
}
|
|
8304
|
+
return { ...pack, primary };
|
|
8305
|
+
}
|
|
8306
|
+
function normalizeOptions(options) {
|
|
8307
|
+
const rankingPrompt = options.prompt ?? "";
|
|
8308
|
+
const { display, truncated } = truncatePrompt(rankingPrompt);
|
|
8309
|
+
return {
|
|
8310
|
+
displayPrompt: display,
|
|
8311
|
+
rankingPrompt,
|
|
8312
|
+
budget: clampPositive(options.budget, DEFAULT_BUDGET_TOKENS),
|
|
8313
|
+
depth: clampDepth(options.depth),
|
|
8314
|
+
topPages: clampBounded(options.topPages, DEFAULT_TOP_PAGES, MAX_TOP_PAGES),
|
|
8315
|
+
topChunks: clampBounded(options.topChunks, DEFAULT_TOP_CHUNKS, MAX_TOP_CHUNKS),
|
|
8316
|
+
omitRoot: options.omitRoot === true,
|
|
8317
|
+
// `--no-neighbors` is a Commander negated flag: absence means
|
|
8318
|
+
// expansion is ON; only `options.neighbors === false` disables it.
|
|
8319
|
+
neighborsEnabled: options.neighbors !== false,
|
|
8320
|
+
includeSources: options.includeSources === true,
|
|
8321
|
+
promptTruncated: truncated
|
|
8322
|
+
};
|
|
8323
|
+
}
|
|
8324
|
+
function truncatePrompt(raw) {
|
|
8325
|
+
if (raw.length <= PROMPT_ECHO_MAX_LENGTH) return { display: raw, truncated: false };
|
|
8326
|
+
return { display: raw.slice(0, PROMPT_ECHO_MAX_LENGTH), truncated: true };
|
|
8327
|
+
}
|
|
8328
|
+
function clampPositive(value, fallback) {
|
|
8329
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
8330
|
+
return Math.max(0, Math.floor(value));
|
|
8331
|
+
}
|
|
8332
|
+
function clampBounded(value, fallback, max) {
|
|
8333
|
+
return Math.min(max, clampPositive(value, fallback));
|
|
8334
|
+
}
|
|
8335
|
+
function clampDepth(value) {
|
|
8336
|
+
if (value === void 0 || !Number.isFinite(value)) return DEFAULT_DEPTH;
|
|
8337
|
+
return Math.max(0, Math.min(MAX_DEPTH, Math.floor(value)));
|
|
8338
|
+
}
|
|
8339
|
+
function assembleDraft(input) {
|
|
8340
|
+
const { snapshot, state, recommendation, options, semantic } = input;
|
|
8341
|
+
const project = buildProject2(snapshot, state, options.omitRoot);
|
|
8342
|
+
const primary = rankPages(snapshot, options.rankingPrompt, options.topPages, semantic.hits);
|
|
8343
|
+
const graphEnabled = options.neighborsEnabled && options.depth >= 1;
|
|
8344
|
+
const expansion = graphEnabled ? expandGraphNeighborhood({
|
|
8345
|
+
graph: snapshot.graph,
|
|
8346
|
+
pages: snapshot.pages,
|
|
8347
|
+
primaryIds: collectPrimaryIds(primary),
|
|
8348
|
+
depth: options.depth
|
|
8349
|
+
}) : emptyExpansion();
|
|
8350
|
+
const annotatedPrimary = graphEnabled ? annotateGraphNeighbors(primary, snapshot.graph) : primary;
|
|
8351
|
+
return {
|
|
8352
|
+
version: 1,
|
|
8353
|
+
prompt: options.displayPrompt,
|
|
8354
|
+
budget: buildBudget(options.budget, 0),
|
|
8355
|
+
project,
|
|
8356
|
+
primary: annotatedPrimary,
|
|
8357
|
+
neighbors: expansion.neighbors,
|
|
8358
|
+
warnings: buildTopLevelWarnings(options.promptTruncated, semantic.warning),
|
|
8359
|
+
gaps: expansion.gaps,
|
|
8360
|
+
suggestedActions: collectSuggestedActions(recommendation, {
|
|
8361
|
+
hasPages: snapshot.pages.length > 0,
|
|
8362
|
+
semanticWarning: semantic.warning
|
|
8363
|
+
})
|
|
8364
|
+
};
|
|
8365
|
+
}
|
|
8366
|
+
function annotateGraphNeighbors(primary, graph) {
|
|
8367
|
+
if (primary.length < 2) return primary;
|
|
8368
|
+
const primaryIds = collectPrimaryIds(primary);
|
|
8369
|
+
const connected = /* @__PURE__ */ new Set();
|
|
8370
|
+
for (const edge of graph.edges) {
|
|
8371
|
+
if (primaryIds.has(edge.source) && primaryIds.has(edge.target)) {
|
|
8372
|
+
connected.add(edge.source);
|
|
8373
|
+
connected.add(edge.target);
|
|
8374
|
+
}
|
|
8375
|
+
}
|
|
8376
|
+
if (connected.size === 0) return primary;
|
|
8377
|
+
return primary.map((entry) => {
|
|
8378
|
+
if (!connected.has(entry.id)) return entry;
|
|
8379
|
+
if (entry.reasons.includes("graph-neighbor")) return entry;
|
|
8380
|
+
const widened = Array.from(/* @__PURE__ */ new Set([...entry.reasons, "graph-neighbor"])).sort();
|
|
8381
|
+
return { ...entry, reasons: widened };
|
|
8382
|
+
});
|
|
8383
|
+
}
|
|
8384
|
+
function collectPrimaryIds(primary) {
|
|
8385
|
+
const ids = /* @__PURE__ */ new Set();
|
|
8386
|
+
for (const entry of primary) ids.add(entry.id);
|
|
8387
|
+
return ids;
|
|
8388
|
+
}
|
|
8389
|
+
function stripGraphNeighborReason(primary) {
|
|
8390
|
+
return primary.map((entry) => ({
|
|
8391
|
+
...entry,
|
|
8392
|
+
reasons: entry.reasons.filter((reason) => reason !== "graph-neighbor")
|
|
8393
|
+
}));
|
|
8394
|
+
}
|
|
8395
|
+
function reconcileGraphNeighborReasons(pack, graph) {
|
|
8396
|
+
const stripped = stripGraphNeighborReason(pack.primary);
|
|
8397
|
+
const primary = graph ? annotateGraphNeighbors(stripped, graph) : stripped;
|
|
8398
|
+
return primary === pack.primary ? pack : { ...pack, primary };
|
|
8399
|
+
}
|
|
8400
|
+
function emptyExpansion() {
|
|
8401
|
+
return { neighbors: [], gaps: [] };
|
|
8402
|
+
}
|
|
8403
|
+
function buildProject2(snapshot, state, omitRoot) {
|
|
8404
|
+
return {
|
|
8405
|
+
root: omitRoot ? null : snapshot.root,
|
|
8406
|
+
pages: snapshot.pages.length,
|
|
8407
|
+
pendingCandidates: state.pendingCandidates,
|
|
8408
|
+
lint: state.lint.entry
|
|
8409
|
+
};
|
|
8410
|
+
}
|
|
8411
|
+
function buildTopLevelWarnings(promptTruncated, retrievalWarning) {
|
|
8412
|
+
const warnings = [];
|
|
8413
|
+
if (promptTruncated) {
|
|
8414
|
+
warnings.push({
|
|
8415
|
+
code: "truncated-prompt",
|
|
8416
|
+
message: `Prompt exceeded ${PROMPT_ECHO_MAX_LENGTH} characters; the echoed copy was truncated.`
|
|
8417
|
+
});
|
|
8418
|
+
}
|
|
8419
|
+
if (retrievalWarning === "embedding-store-missing") {
|
|
8420
|
+
warnings.push({
|
|
8421
|
+
code: "embedding-store-missing",
|
|
8422
|
+
message: "No usable embedding store found; semantic retrieval skipped. Run `llmwiki compile` to populate embeddings."
|
|
8423
|
+
});
|
|
8424
|
+
} else if (retrievalWarning === "query-embedding-unavailable") {
|
|
8425
|
+
warnings.push({
|
|
8426
|
+
code: "query-embedding-unavailable",
|
|
8427
|
+
message: "Could not embed the prompt with the active provider; semantic retrieval skipped, lexical signals still applied."
|
|
8428
|
+
});
|
|
8429
|
+
} else if (retrievalWarning === "semantic-retrieval-error") {
|
|
8430
|
+
warnings.push({
|
|
8431
|
+
code: "semantic-retrieval-error",
|
|
8432
|
+
message: "Semantic retrieval failed unexpectedly; lexical signals still applied and raw provider errors were not exposed."
|
|
8433
|
+
});
|
|
8434
|
+
}
|
|
8435
|
+
return warnings;
|
|
8436
|
+
}
|
|
8437
|
+
var CONTEXT_COMPILE_ACTION = {
|
|
8438
|
+
command: "llmwiki compile",
|
|
8439
|
+
reason: "Refresh compiled pages and rebuild the embedding store for semantic context.",
|
|
8440
|
+
executable: { binary: "llmwiki", args: ["compile"] }
|
|
8441
|
+
};
|
|
8442
|
+
var CONTEXT_VIEW_ACTION = {
|
|
8443
|
+
command: "llmwiki view --open",
|
|
8444
|
+
reason: "Browse the compiled wiki in the local viewer.",
|
|
8445
|
+
executable: { binary: "llmwiki", args: ["view", "--open"] }
|
|
8446
|
+
};
|
|
8447
|
+
var CONTEXT_QUERY_ACTION = {
|
|
8448
|
+
command: 'llmwiki query "<prompt>"',
|
|
8449
|
+
reason: "Generate an answer with the same prompt after reviewing this context pack.",
|
|
8450
|
+
executable: { binary: "llmwiki", args: ["query"], placeholders: ["prompt"] }
|
|
8451
|
+
};
|
|
8452
|
+
function collectSuggestedActions(recommendation, input) {
|
|
8453
|
+
const actions = [];
|
|
8454
|
+
for (const action of [recommendation.recommended, ...recommendation.otherActions]) {
|
|
8455
|
+
appendUniqueAction(actions, action);
|
|
8456
|
+
}
|
|
8457
|
+
if (input.hasPages && input.semanticWarning === "embedding-store-missing") {
|
|
8458
|
+
appendUniqueAction(actions, CONTEXT_COMPILE_ACTION);
|
|
8459
|
+
}
|
|
8460
|
+
if (input.hasPages) {
|
|
8461
|
+
appendUniqueAction(actions, CONTEXT_VIEW_ACTION);
|
|
8462
|
+
appendUniqueAction(actions, CONTEXT_QUERY_ACTION);
|
|
8463
|
+
}
|
|
8464
|
+
return actions;
|
|
8465
|
+
}
|
|
8466
|
+
function appendUniqueAction(actions, candidate) {
|
|
8467
|
+
if (actions.some((action) => actionKey(action) === actionKey(candidate))) return;
|
|
8468
|
+
actions.push(candidate);
|
|
8469
|
+
}
|
|
8470
|
+
function actionKey(action) {
|
|
8471
|
+
if (!action.executable) return action.command ?? action.reason;
|
|
8472
|
+
return `${action.executable.binary} ${action.executable.args.join(" ")}`;
|
|
8473
|
+
}
|
|
8474
|
+
function finalizeBudget(draft, requestedTokens, graph = null) {
|
|
8475
|
+
const initialEstimate = estimatePackTokens(draft);
|
|
8476
|
+
const trim = initialEstimate <= requestedTokens ? { pack: draft, trimmedSections: [] } : trimToBudget(draft, requestedTokens);
|
|
8477
|
+
const trimmedAny = trim.trimmedSections.length > 0;
|
|
8478
|
+
const reconciledPack = reconcileGraphNeighborReasons(trim.pack, graph);
|
|
8479
|
+
const e1 = estimatePackTokens(
|
|
8480
|
+
applyBudget(reconciledPack, {
|
|
8481
|
+
requestedTokens,
|
|
8482
|
+
estimatedTokens: 0,
|
|
8483
|
+
truncated: trimmedAny,
|
|
8484
|
+
trimmedSections: trim.trimmedSections
|
|
8485
|
+
})
|
|
8486
|
+
);
|
|
8487
|
+
const e2 = estimatePackTokens(
|
|
8488
|
+
applyBudget(reconciledPack, {
|
|
8489
|
+
requestedTokens,
|
|
8490
|
+
estimatedTokens: e1,
|
|
8491
|
+
truncated: trimmedAny,
|
|
8492
|
+
trimmedSections: trim.trimmedSections
|
|
8493
|
+
})
|
|
8494
|
+
);
|
|
8495
|
+
const truncated = trimmedAny || e2 > requestedTokens;
|
|
8496
|
+
return applyBudget(reconciledPack, {
|
|
8497
|
+
requestedTokens,
|
|
8498
|
+
estimatedTokens: e2,
|
|
8499
|
+
truncated,
|
|
8500
|
+
trimmedSections: trim.trimmedSections
|
|
8501
|
+
});
|
|
8502
|
+
}
|
|
8503
|
+
function applyBudget(pack, budget) {
|
|
8504
|
+
return { ...pack, budget };
|
|
8505
|
+
}
|
|
8506
|
+
function appendProjectWarnings(pack, state, options) {
|
|
8507
|
+
const warnings = [...pack.warnings];
|
|
8508
|
+
if (state.pendingCandidates > 0) {
|
|
8509
|
+
warnings.push({
|
|
8510
|
+
code: "pending-candidates",
|
|
8511
|
+
message: `${state.pendingCandidates} review candidate${state.pendingCandidates === 1 ? "" : "s"} pending approval. Run \`llmwiki review list\` to inspect.`
|
|
8512
|
+
});
|
|
5936
8513
|
}
|
|
5937
|
-
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
}
|
|
8514
|
+
const lintErrors = state.lint.entry?.errors ?? 0;
|
|
8515
|
+
if (lintErrors > 0) {
|
|
8516
|
+
warnings.push({
|
|
8517
|
+
code: "lint-errors",
|
|
8518
|
+
message: `Last lint run reported ${lintErrors} error${lintErrors === 1 ? "" : "s"}.`
|
|
8519
|
+
});
|
|
8520
|
+
}
|
|
8521
|
+
if (options.includeSources && hasUnmaterializedSpans(pack)) {
|
|
8522
|
+
warnings.push({
|
|
8523
|
+
code: "source-window-unavailable",
|
|
8524
|
+
message: "One or more line-range citations did not produce a source window (path-confined, missing source file, or per-pack window cap reached)."
|
|
8525
|
+
});
|
|
5943
8526
|
}
|
|
8527
|
+
return { ...pack, warnings };
|
|
8528
|
+
}
|
|
8529
|
+
function hasUnmaterializedSpans(pack) {
|
|
8530
|
+
for (const entry of pack.primary) {
|
|
8531
|
+
const lineRangeCount = entry.citations.filter(
|
|
8532
|
+
(c) => c.start !== void 0 && c.end !== void 0
|
|
8533
|
+
).length;
|
|
8534
|
+
if (lineRangeCount > entry.sourceWindows.length) return true;
|
|
8535
|
+
}
|
|
8536
|
+
return false;
|
|
5944
8537
|
}
|
|
5945
8538
|
|
|
5946
|
-
// src/commands/
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
8539
|
+
// src/commands/context.ts
|
|
8540
|
+
async function contextCommand(prompt, options = {}) {
|
|
8541
|
+
const pack = await buildContextPack({
|
|
8542
|
+
root: process.cwd(),
|
|
8543
|
+
prompt,
|
|
8544
|
+
budget: coerceNumber(options.budget, DEFAULT_BUDGET_TOKENS),
|
|
8545
|
+
depth: coerceNumber(options.depth, DEFAULT_DEPTH),
|
|
8546
|
+
topPages: coerceNumber(options.topPages, DEFAULT_TOP_PAGES),
|
|
8547
|
+
topChunks: coerceNumber(options.topChunks, DEFAULT_TOP_CHUNKS),
|
|
8548
|
+
omitRoot: options.omitRoot === true,
|
|
8549
|
+
neighbors: options.neighbors,
|
|
8550
|
+
includeSources: options.includeSources === true
|
|
8551
|
+
});
|
|
8552
|
+
emit(pack, resolveFormat(options));
|
|
8553
|
+
return 0;
|
|
8554
|
+
}
|
|
8555
|
+
function coerceNumber(raw, fallback) {
|
|
8556
|
+
if (raw === void 0) return fallback;
|
|
8557
|
+
const value = typeof raw === "number" ? raw : Number(raw);
|
|
8558
|
+
return Number.isFinite(value) ? value : fallback;
|
|
8559
|
+
}
|
|
8560
|
+
function resolveFormat(options) {
|
|
8561
|
+
if (options.json === true) return "json";
|
|
8562
|
+
if (options.format === "json") return "json";
|
|
8563
|
+
return "markdown";
|
|
8564
|
+
}
|
|
8565
|
+
function emit(pack, format) {
|
|
8566
|
+
if (format === "json") {
|
|
8567
|
+
process.stdout.write(`${JSON.stringify(pack, null, 2)}
|
|
8568
|
+
`);
|
|
5958
8569
|
return;
|
|
5959
8570
|
}
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
8571
|
+
process.stdout.write(`${renderMarkdown(pack)}
|
|
8572
|
+
`);
|
|
8573
|
+
}
|
|
8574
|
+
function renderMarkdown(pack) {
|
|
8575
|
+
const lines = [];
|
|
8576
|
+
appendHeader(lines, pack);
|
|
8577
|
+
appendPrimaryPages(lines, pack.primary);
|
|
8578
|
+
appendGraphNeighborhood(lines, pack.neighbors);
|
|
8579
|
+
appendWarnings(lines, pack.warnings);
|
|
8580
|
+
appendSuggestedActions(lines, pack.suggestedActions);
|
|
8581
|
+
return lines.join("\n");
|
|
8582
|
+
}
|
|
8583
|
+
function appendGraphNeighborhood(lines, neighbors) {
|
|
8584
|
+
if (neighbors.length === 0) return;
|
|
8585
|
+
lines.push("## Graph Neighborhood");
|
|
8586
|
+
lines.push("");
|
|
8587
|
+
for (const neighbor of neighbors) {
|
|
8588
|
+
const arrow = neighbor.direction === "outgoing" ? "->" : "<-";
|
|
8589
|
+
lines.push(
|
|
8590
|
+
`- \`${neighbor.from}\` ${arrow} \`${neighbor.to}\` (${neighbor.reason}, distance ${neighbor.distance})`
|
|
8591
|
+
);
|
|
5964
8592
|
}
|
|
8593
|
+
lines.push("");
|
|
5965
8594
|
}
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
5969
|
-
|
|
8595
|
+
function appendHeader(lines, pack) {
|
|
8596
|
+
lines.push("# Context Pack");
|
|
8597
|
+
lines.push("");
|
|
8598
|
+
lines.push(`Prompt: ${pack.prompt}`);
|
|
8599
|
+
lines.push(`Budget: ${pack.budget.estimatedTokens} / ${pack.budget.requestedTokens} estimated tokens`);
|
|
5970
8600
|
}
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
8601
|
+
function appendPrimaryPages(lines, primary) {
|
|
8602
|
+
lines.push("");
|
|
8603
|
+
lines.push("## Primary Pages");
|
|
8604
|
+
lines.push("");
|
|
8605
|
+
if (primary.length === 0) {
|
|
8606
|
+
lines.push("_No primary pages matched the prompt._");
|
|
5977
8607
|
return;
|
|
5978
8608
|
}
|
|
5979
|
-
const
|
|
5980
|
-
await atomicWrite(pagePath, candidate.body);
|
|
5981
|
-
status("+", success(`Approved \u2192 ${source(pagePath)}`));
|
|
5982
|
-
await persistCandidateSourceStates(root, candidate);
|
|
5983
|
-
await refreshWikiAfterApproval(root, candidate.slug);
|
|
5984
|
-
await deleteCandidate(root, id);
|
|
5985
|
-
status("\u2713", dim(`Candidate ${id} cleared.`));
|
|
8609
|
+
for (const page of primary) appendPrimaryPage(lines, page);
|
|
5986
8610
|
}
|
|
5987
|
-
|
|
5988
|
-
const
|
|
5989
|
-
|
|
5990
|
-
|
|
5991
|
-
|
|
5992
|
-
|
|
5993
|
-
|
|
8611
|
+
function appendPrimaryPage(lines, page) {
|
|
8612
|
+
const pageFile = path48.join("wiki", page.pageDirectory, `${slugFromId(page.id)}.md`);
|
|
8613
|
+
lines.push(`### ${page.title} (\`${pageFile}\`)`);
|
|
8614
|
+
lines.push("");
|
|
8615
|
+
lines.push(`Why included: ${page.reasons.join(", ") || "(no signals)"}`);
|
|
8616
|
+
if (page.summary) {
|
|
8617
|
+
lines.push("");
|
|
8618
|
+
lines.push(page.summary);
|
|
5994
8619
|
}
|
|
8620
|
+
appendCitations(lines, page);
|
|
8621
|
+
appendSourceWindows(lines, page);
|
|
8622
|
+
lines.push("");
|
|
5995
8623
|
}
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
const
|
|
5999
|
-
|
|
6000
|
-
|
|
6001
|
-
for (const source2 of candidate.sources) sources.add(source2);
|
|
6002
|
-
}
|
|
6003
|
-
return sources;
|
|
8624
|
+
function appendCitations(lines, page) {
|
|
8625
|
+
if (page.citations.length === 0) return;
|
|
8626
|
+
const refs = page.citations.map(renderCitation).join(", ");
|
|
8627
|
+
lines.push("");
|
|
8628
|
+
lines.push(`Sources: ${refs}`);
|
|
6004
8629
|
}
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
6008
|
-
|
|
6009
|
-
|
|
8630
|
+
function renderCitation(citation) {
|
|
8631
|
+
if (citation.start !== void 0 && citation.end !== void 0) {
|
|
8632
|
+
return `\`${citation.file}:${citation.start}-${citation.end}\``;
|
|
8633
|
+
}
|
|
8634
|
+
return `\`${citation.file}\``;
|
|
6010
8635
|
}
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
8636
|
+
function appendSourceWindows(lines, page) {
|
|
8637
|
+
if (page.sourceWindows.length === 0) return;
|
|
8638
|
+
for (const window of page.sourceWindows) {
|
|
8639
|
+
lines.push("");
|
|
8640
|
+
lines.push(`From \`${window.file}:${window.start}-${window.end}\`:`);
|
|
8641
|
+
lines.push("");
|
|
8642
|
+
for (const line2 of window.text.split(/\r?\n/)) lines.push(`> ${line2}`);
|
|
6017
8643
|
}
|
|
6018
8644
|
}
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6022
|
-
await runReviewUnderLock(id, rejectUnderLock);
|
|
8645
|
+
function slugFromId(id) {
|
|
8646
|
+
const idx = id.indexOf("/");
|
|
8647
|
+
return idx === -1 ? id : id.slice(idx + 1);
|
|
6023
8648
|
}
|
|
6024
|
-
|
|
6025
|
-
|
|
6026
|
-
|
|
6027
|
-
|
|
6028
|
-
|
|
6029
|
-
|
|
6030
|
-
|
|
6031
|
-
|
|
8649
|
+
function appendWarnings(lines, warnings) {
|
|
8650
|
+
if (warnings.length === 0) return;
|
|
8651
|
+
lines.push("## Warnings");
|
|
8652
|
+
lines.push("");
|
|
8653
|
+
for (const warning of warnings) lines.push(`- ${warning.message}`);
|
|
8654
|
+
lines.push("");
|
|
8655
|
+
}
|
|
8656
|
+
function appendSuggestedActions(lines, actions) {
|
|
8657
|
+
if (actions.length === 0) return;
|
|
8658
|
+
lines.push("## Suggested Next Actions");
|
|
8659
|
+
lines.push("");
|
|
8660
|
+
for (const action of actions) {
|
|
8661
|
+
if (action.command) lines.push(`- \`${action.command}\``);
|
|
8662
|
+
}
|
|
6032
8663
|
}
|
|
6033
8664
|
|
|
6034
8665
|
// src/mcp/server.ts
|
|
@@ -6036,42 +8667,8 @@ import { McpServer as McpServer2 } from "@modelcontextprotocol/sdk/server/mcp.js
|
|
|
6036
8667
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
6037
8668
|
|
|
6038
8669
|
// src/mcp/tools.ts
|
|
6039
|
-
import
|
|
8670
|
+
import path49 from "path";
|
|
6040
8671
|
import { z } from "zod";
|
|
6041
|
-
|
|
6042
|
-
// src/mcp/provider-check.ts
|
|
6043
|
-
var PROVIDER_KEY_VARS = {
|
|
6044
|
-
anthropic: "ANTHROPIC_API_KEY",
|
|
6045
|
-
openai: "OPENAI_API_KEY",
|
|
6046
|
-
ollama: null,
|
|
6047
|
-
minimax: "MINIMAX_API_KEY",
|
|
6048
|
-
copilot: "GITHUB_TOKEN"
|
|
6049
|
-
};
|
|
6050
|
-
function ensureProviderAvailable() {
|
|
6051
|
-
const provider = process.env.LLMWIKI_PROVIDER ?? DEFAULT_PROVIDER;
|
|
6052
|
-
if (provider === "anthropic") {
|
|
6053
|
-
const auth = resolveAnthropicAuthFromEnv();
|
|
6054
|
-
if (!auth.apiKey && !auth.authToken) {
|
|
6055
|
-
throw new Error(
|
|
6056
|
-
'Anthropic credentials are required for the "anthropic" provider. Set ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN.'
|
|
6057
|
-
);
|
|
6058
|
-
}
|
|
6059
|
-
return;
|
|
6060
|
-
}
|
|
6061
|
-
const keyVar = PROVIDER_KEY_VARS[provider];
|
|
6062
|
-
if (keyVar === void 0) {
|
|
6063
|
-
throw new Error(
|
|
6064
|
-
`Unknown provider "${provider}". Supported: ${Object.keys(PROVIDER_KEY_VARS).join(", ")}`
|
|
6065
|
-
);
|
|
6066
|
-
}
|
|
6067
|
-
if (keyVar && !process.env[keyVar]) {
|
|
6068
|
-
throw new Error(
|
|
6069
|
-
`${keyVar} environment variable is required for the "${provider}" provider.`
|
|
6070
|
-
);
|
|
6071
|
-
}
|
|
6072
|
-
}
|
|
6073
|
-
|
|
6074
|
-
// src/mcp/tools.ts
|
|
6075
8672
|
var PAGE_DIRS2 = [CONCEPTS_DIR, QUERIES_DIR];
|
|
6076
8673
|
function jsonResult(payload) {
|
|
6077
8674
|
return {
|
|
@@ -6087,6 +8684,7 @@ function registerWikiTools(server, root) {
|
|
|
6087
8684
|
registerReadTool(server, root);
|
|
6088
8685
|
registerLintTool(server, root);
|
|
6089
8686
|
registerStatusTool(server, root);
|
|
8687
|
+
registerContextPackTool(server, root);
|
|
6090
8688
|
}
|
|
6091
8689
|
function registerIngestTool(server, root) {
|
|
6092
8690
|
server.registerTool(
|
|
@@ -6173,7 +8771,7 @@ async function pickSearchSlugs(root, question) {
|
|
|
6173
8771
|
if (candidates.length > 0) return candidates.map((c) => c.slug);
|
|
6174
8772
|
} catch {
|
|
6175
8773
|
}
|
|
6176
|
-
const indexContent = await safeReadFile(
|
|
8774
|
+
const indexContent = await safeReadFile(path49.join(root, INDEX_FILE));
|
|
6177
8775
|
const { pages } = await selectPages(question, indexContent);
|
|
6178
8776
|
return pages;
|
|
6179
8777
|
}
|
|
@@ -6231,9 +8829,48 @@ function registerStatusTool(server, root) {
|
|
|
6231
8829
|
async () => jsonResult(await collectStatus(root))
|
|
6232
8830
|
);
|
|
6233
8831
|
}
|
|
8832
|
+
function registerContextPackTool(server, root) {
|
|
8833
|
+
server.registerTool(
|
|
8834
|
+
"get_context_pack",
|
|
8835
|
+
contextPackToolConfig(),
|
|
8836
|
+
async (args) => jsonResult(await buildContextPackFromArgs(root, args))
|
|
8837
|
+
);
|
|
8838
|
+
}
|
|
8839
|
+
function contextPackToolConfig() {
|
|
8840
|
+
return {
|
|
8841
|
+
title: "Get Context Pack",
|
|
8842
|
+
description: "Build an agent-ready evidence pack for `prompt` over the compiled wiki: primary pages, semantic chunks, graph neighbors, citations, warnings, and suggested next actions. Returns the same v1 JSON envelope as `llmwiki context --json`. Read-only; no provider credentials required. Use this to PREPARE evidence; use `query_wiki` to GENERATE a grounded natural-language answer.",
|
|
8843
|
+
inputSchema: contextPackInputSchema()
|
|
8844
|
+
};
|
|
8845
|
+
}
|
|
8846
|
+
function contextPackInputSchema() {
|
|
8847
|
+
return {
|
|
8848
|
+
prompt: z.string().describe("Free-text task or topic to assemble context for."),
|
|
8849
|
+
budget: z.number().optional().describe("Approximate output token budget (default 8000)."),
|
|
8850
|
+
depth: z.number().optional().describe("Graph neighborhood depth, 0..2 (default 1, 0 disables expansion)."),
|
|
8851
|
+
topPages: z.number().optional().describe("Max primary pages (default 5, max 20)."),
|
|
8852
|
+
topChunks: z.number().optional().describe("Max semantic chunks to surface (default 8, max 50)."),
|
|
8853
|
+
omitRoot: z.boolean().optional().describe("Emit `project.root` as null instead of the absolute path."),
|
|
8854
|
+
includeSources: z.boolean().optional().describe(
|
|
8855
|
+
"Materialize `primary[].sourceWindows` from claim-level citations (reads files under `sources/` only; path-confined)."
|
|
8856
|
+
)
|
|
8857
|
+
};
|
|
8858
|
+
}
|
|
8859
|
+
async function buildContextPackFromArgs(root, args) {
|
|
8860
|
+
return buildContextPack({
|
|
8861
|
+
root,
|
|
8862
|
+
prompt: args.prompt,
|
|
8863
|
+
budget: args.budget,
|
|
8864
|
+
depth: args.depth,
|
|
8865
|
+
topPages: args.topPages,
|
|
8866
|
+
topChunks: args.topChunks,
|
|
8867
|
+
omitRoot: args.omitRoot,
|
|
8868
|
+
includeSources: args.includeSources
|
|
8869
|
+
});
|
|
8870
|
+
}
|
|
6234
8871
|
async function collectStatus(root) {
|
|
6235
|
-
const concepts = await collectPageSummaries(
|
|
6236
|
-
const queries = await collectPageSummaries(
|
|
8872
|
+
const concepts = await collectPageSummaries(path49.join(root, CONCEPTS_DIR));
|
|
8873
|
+
const queries = await collectPageSummaries(path49.join(root, QUERIES_DIR));
|
|
6237
8874
|
const state = await readState(root);
|
|
6238
8875
|
const changes = await detectChanges(root, state);
|
|
6239
8876
|
const orphans = await findOrphanedSlugs(root);
|
|
@@ -6250,7 +8887,7 @@ async function collectStatus(root) {
|
|
|
6250
8887
|
};
|
|
6251
8888
|
}
|
|
6252
8889
|
async function findOrphanedSlugs(root) {
|
|
6253
|
-
const scanned = await scanWikiPages(
|
|
8890
|
+
const scanned = await scanWikiPages(path49.join(root, CONCEPTS_DIR));
|
|
6254
8891
|
return scanned.filter(({ meta }) => meta.orphaned).map(({ slug }) => slug);
|
|
6255
8892
|
}
|
|
6256
8893
|
async function loadPageRecords(root, slugs) {
|
|
@@ -6263,7 +8900,7 @@ async function loadPageRecords(root, slugs) {
|
|
|
6263
8900
|
}
|
|
6264
8901
|
async function readPage(root, slug) {
|
|
6265
8902
|
for (const dir of PAGE_DIRS2) {
|
|
6266
|
-
const content = await safeReadFile(
|
|
8903
|
+
const content = await safeReadFile(path49.join(root, dir, `${slug}.md`));
|
|
6267
8904
|
if (!content) continue;
|
|
6268
8905
|
const { meta, body } = parseFrontmatter(content);
|
|
6269
8906
|
if (meta.orphaned) continue;
|
|
@@ -6278,8 +8915,8 @@ async function readPage(root, slug) {
|
|
|
6278
8915
|
}
|
|
6279
8916
|
|
|
6280
8917
|
// src/mcp/resources.ts
|
|
6281
|
-
import
|
|
6282
|
-
import { readdir as
|
|
8918
|
+
import path50 from "path";
|
|
8919
|
+
import { readdir as readdir14 } from "fs/promises";
|
|
6283
8920
|
import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
6284
8921
|
function jsonContent(uri, payload) {
|
|
6285
8922
|
return {
|
|
@@ -6312,7 +8949,7 @@ function registerIndexResource(server, root) {
|
|
|
6312
8949
|
mimeType: "text/markdown"
|
|
6313
8950
|
},
|
|
6314
8951
|
async (uri) => {
|
|
6315
|
-
const content = await safeReadFile(
|
|
8952
|
+
const content = await safeReadFile(path50.join(root, INDEX_FILE));
|
|
6316
8953
|
return { contents: [markdownContent(uri, content)] };
|
|
6317
8954
|
}
|
|
6318
8955
|
);
|
|
@@ -6379,23 +9016,23 @@ function registerQueryResource(server, root) {
|
|
|
6379
9016
|
);
|
|
6380
9017
|
}
|
|
6381
9018
|
async function listSources(root) {
|
|
6382
|
-
const sourcesPath =
|
|
9019
|
+
const sourcesPath = path50.join(root, SOURCES_DIR);
|
|
6383
9020
|
let files;
|
|
6384
9021
|
try {
|
|
6385
|
-
files = await
|
|
9022
|
+
files = await readdir14(sourcesPath);
|
|
6386
9023
|
} catch {
|
|
6387
9024
|
return [];
|
|
6388
9025
|
}
|
|
6389
9026
|
const records = [];
|
|
6390
9027
|
for (const file of files.filter((f) => f.endsWith(".md"))) {
|
|
6391
|
-
const content = await safeReadFile(
|
|
9028
|
+
const content = await safeReadFile(path50.join(sourcesPath, file));
|
|
6392
9029
|
const { meta } = parseFrontmatter(content);
|
|
6393
9030
|
records.push({ filename: file, ...meta });
|
|
6394
9031
|
}
|
|
6395
9032
|
return records;
|
|
6396
9033
|
}
|
|
6397
9034
|
async function loadPageWithMeta(root, dir, slug) {
|
|
6398
|
-
const filePath =
|
|
9035
|
+
const filePath = path50.join(root, dir, `${slug}.md`);
|
|
6399
9036
|
const content = await safeReadFile(filePath);
|
|
6400
9037
|
if (!content) {
|
|
6401
9038
|
throw new Error(`Page not found: ${dir}/${slug}.md`);
|
|
@@ -6404,10 +9041,10 @@ async function loadPageWithMeta(root, dir, slug) {
|
|
|
6404
9041
|
return { slug, meta, body: body.trim() };
|
|
6405
9042
|
}
|
|
6406
9043
|
async function listPagesUnder(root, dir, scheme) {
|
|
6407
|
-
const pagesPath =
|
|
9044
|
+
const pagesPath = path50.join(root, dir);
|
|
6408
9045
|
let files;
|
|
6409
9046
|
try {
|
|
6410
|
-
files = await
|
|
9047
|
+
files = await readdir14(pagesPath);
|
|
6411
9048
|
} catch {
|
|
6412
9049
|
return { resources: [] };
|
|
6413
9050
|
}
|
|
@@ -6540,6 +9177,55 @@ program.command("lint").description("Run rule-based quality checks against the w
|
|
|
6540
9177
|
process.exit(1);
|
|
6541
9178
|
}
|
|
6542
9179
|
});
|
|
9180
|
+
var evalCmd = program.command("eval").description("Evaluate wiki quality (health, citation coverage, LLM judge)").option("--suite <level>", "fast (deterministic) or full (+ LLM judge)", "fast").option("--out <format>", "terminal or json", "terminal").option("--sample <n>", "number of citations to judge in full suite", "20").action(async (opts) => {
|
|
9181
|
+
try {
|
|
9182
|
+
await evalCommand(opts);
|
|
9183
|
+
} catch (err) {
|
|
9184
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
9185
|
+
process.exit(1);
|
|
9186
|
+
}
|
|
9187
|
+
});
|
|
9188
|
+
var evalCacheCmd = evalCmd.command("cache").description("Manage the citation judgement cache");
|
|
9189
|
+
evalCacheCmd.command("clear").description("Delete the cache so all judgements re-run on the next full eval").action(async () => {
|
|
9190
|
+
try {
|
|
9191
|
+
await evalCacheClearCommand();
|
|
9192
|
+
} catch (err) {
|
|
9193
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
9194
|
+
process.exit(1);
|
|
9195
|
+
}
|
|
9196
|
+
});
|
|
9197
|
+
evalCacheCmd.command("show").description("Print a score distribution summary of cached citation judgements").action(async () => {
|
|
9198
|
+
try {
|
|
9199
|
+
await evalCacheShowCommand();
|
|
9200
|
+
} catch (err) {
|
|
9201
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
9202
|
+
process.exit(1);
|
|
9203
|
+
}
|
|
9204
|
+
});
|
|
9205
|
+
evalCmd.command("report").description("Re-display the most recent eval report without running a new eval").option("--out <format>", "terminal or json", "terminal").action(async (opts) => {
|
|
9206
|
+
try {
|
|
9207
|
+
await evalReportCommand(opts);
|
|
9208
|
+
} catch (err) {
|
|
9209
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
9210
|
+
process.exit(1);
|
|
9211
|
+
}
|
|
9212
|
+
});
|
|
9213
|
+
evalCmd.command("history").description("Show a trend table of past eval runs").option("--n <count>", "number of runs to show", "10").option("--out <format>", "terminal or json", "terminal").action(async (opts) => {
|
|
9214
|
+
try {
|
|
9215
|
+
await evalHistoryCommand(opts);
|
|
9216
|
+
} catch (err) {
|
|
9217
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
9218
|
+
process.exit(1);
|
|
9219
|
+
}
|
|
9220
|
+
});
|
|
9221
|
+
evalCmd.command("judgements").description("Browse cached citation judgements").option("--score <0|1|2>", "filter by support score (0=unsupported, 1=partial, 2=full)").option("--page <slug>", "filter by wiki page slug").option("--n <count>", "limit number of judgements shown").option("--out <format>", "terminal or json", "terminal").action(async (opts) => {
|
|
9222
|
+
try {
|
|
9223
|
+
await evalJudgementsCommand(opts);
|
|
9224
|
+
} catch (err) {
|
|
9225
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
9226
|
+
process.exit(1);
|
|
9227
|
+
}
|
|
9228
|
+
});
|
|
6543
9229
|
var schemaCmd = program.command("schema").description("Inspect or initialize the project's wiki schema config");
|
|
6544
9230
|
schemaCmd.command("init").description("Write a starter schema file to .llmwiki/schema.json").action(async () => {
|
|
6545
9231
|
try {
|
|
@@ -6568,6 +9254,28 @@ program.command("export").description("Export wiki content to portable formats (
|
|
|
6568
9254
|
process.exit(1);
|
|
6569
9255
|
}
|
|
6570
9256
|
});
|
|
9257
|
+
program.command("next").description("Show the recommended next action for this llmwiki project (read-only)").option("--json", "Emit a stable JSON envelope for agent consumption").action(
|
|
9258
|
+
async (options) => runExitCodeCommand(() => nextCommand({ json: options.json }))
|
|
9259
|
+
);
|
|
9260
|
+
program.command("context <prompt>").description(
|
|
9261
|
+
"Build an agent-ready evidence pack for <prompt> from the compiled wiki (read-only; provider credentials optional \u2014 semantic retrieval is used when available and falls back to lexical otherwise)"
|
|
9262
|
+
).option("--budget <tokens>", "Approximate output token budget (default 8000)").option("--format <format>", "Output format: json | markdown (default markdown)").option("--json", "Emit the stable v1 JSON envelope (overrides --format)").option("--depth <n>", "Graph neighborhood depth, default 1, max 2; 0 disables expansion").option("--top-pages <n>", "Max primary pages (default 5, max 20)").option("--top-chunks <n>", "Max semantic chunks (default 8, max 50)").option("--omit-root", "Emit project.root as null for privacy").option("--no-neighbors", "Suppress graph expansion (keeps neighbors/gaps as empty arrays)").option(
|
|
9263
|
+
"--include-sources",
|
|
9264
|
+
"Populate primary[].sourceWindows from claim-level citation spans (max 20 windows, 30 lines each)"
|
|
9265
|
+
).action(
|
|
9266
|
+
async (prompt, options) => runExitCodeCommand(() => contextCommand(prompt, options))
|
|
9267
|
+
);
|
|
9268
|
+
program.command("quickstart <source>").description(
|
|
9269
|
+
"Ingest a source and compile it into a wiki in one step. Recommends the next action when finished."
|
|
9270
|
+
).option("--review", "Generate review candidates instead of mutating wiki/").option("--no-open", "Skip the viewer handoff after a successful compile").option(
|
|
9271
|
+
"--provider <name>",
|
|
9272
|
+
"Override LLMWIKI_PROVIDER for this run only (e.g. anthropic, openai, ollama)"
|
|
9273
|
+
).option(
|
|
9274
|
+
"--lang <code>",
|
|
9275
|
+
'Target language for generated wiki content (e.g. "Chinese", "ja", "zh-CN"). Equivalent to setting LLMWIKI_OUTPUT_LANG.'
|
|
9276
|
+
).option("--json", "Emit the quickstart JSON envelope instead of human output (implies --no-open)").action(
|
|
9277
|
+
async (source2, options) => runExitCodeCommand(() => quickstartCommand(source2, options))
|
|
9278
|
+
);
|
|
6571
9279
|
program.command("serve").description("Start an MCP server exposing wiki tools and resources over stdio").option("--root <dir>", "Project root directory", process.cwd()).action(async (options) => {
|
|
6572
9280
|
try {
|
|
6573
9281
|
await startMCPServer({ root: options.root, version });
|
|
@@ -6576,44 +9284,21 @@ program.command("serve").description("Start an MCP server exposing wiki tools an
|
|
|
6576
9284
|
process.exit(1);
|
|
6577
9285
|
}
|
|
6578
9286
|
});
|
|
6579
|
-
function applyLanguageOption(lang) {
|
|
6580
|
-
if (lang && lang.trim().length > 0) {
|
|
6581
|
-
process.env.LLMWIKI_OUTPUT_LANG = lang.trim();
|
|
6582
|
-
}
|
|
6583
|
-
}
|
|
6584
|
-
var PROVIDER_KEY_VARS2 = {
|
|
6585
|
-
anthropic: "ANTHROPIC_API_KEY",
|
|
6586
|
-
openai: "OPENAI_API_KEY",
|
|
6587
|
-
ollama: null,
|
|
6588
|
-
minimax: "MINIMAX_API_KEY",
|
|
6589
|
-
copilot: "GITHUB_TOKEN"
|
|
6590
|
-
};
|
|
6591
9287
|
function requireProvider() {
|
|
6592
|
-
|
|
6593
|
-
|
|
6594
|
-
|
|
6595
|
-
|
|
6596
|
-
|
|
6597
|
-
`\x1B[31mError:\x1B[0m Anthropic credentials are required for the "anthropic" provider.
|
|
6598
|
-
Set one of: export ANTHROPIC_API_KEY=<your-key> OR export ANTHROPIC_AUTH_TOKEN=<your-token>`
|
|
6599
|
-
);
|
|
6600
|
-
process.exit(1);
|
|
6601
|
-
}
|
|
6602
|
-
return;
|
|
6603
|
-
}
|
|
6604
|
-
const keyVar = PROVIDER_KEY_VARS2[provider];
|
|
6605
|
-
if (keyVar === void 0) {
|
|
6606
|
-
console.error(
|
|
6607
|
-
`\x1B[31mError:\x1B[0m Unknown provider "${provider}".
|
|
6608
|
-
Supported: ${Object.keys(PROVIDER_KEY_VARS2).join(", ")}`
|
|
6609
|
-
);
|
|
9288
|
+
try {
|
|
9289
|
+
ensureProviderAvailable();
|
|
9290
|
+
} catch (err) {
|
|
9291
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
9292
|
+
console.error(`\x1B[31mError:\x1B[0m ${message}`);
|
|
6610
9293
|
process.exit(1);
|
|
6611
9294
|
}
|
|
6612
|
-
|
|
6613
|
-
|
|
6614
|
-
|
|
6615
|
-
|
|
6616
|
-
);
|
|
9295
|
+
}
|
|
9296
|
+
async function runExitCodeCommand(work) {
|
|
9297
|
+
try {
|
|
9298
|
+
const code = await work();
|
|
9299
|
+
if (code !== 0) process.exitCode = code;
|
|
9300
|
+
} catch (err) {
|
|
9301
|
+
console.error(`\x1B[31mError:\x1B[0m ${err instanceof Error ? err.message : err}`);
|
|
6617
9302
|
process.exit(1);
|
|
6618
9303
|
}
|
|
6619
9304
|
}
|