@zosmaai/pi-llm-wiki 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/README.md +16 -8
- package/dist/extensions/llm-wiki/lib/bootstrap.js +2 -0
- package/dist/extensions/llm-wiki/lib/indexing.js +24 -1
- package/dist/extensions/llm-wiki/lib/ingest-worker.js +3 -1
- package/dist/extensions/llm-wiki/lib/knowledge-document.js +11 -2
- package/dist/extensions/llm-wiki/lib/knowledge-links.js +41 -6
- package/dist/extensions/llm-wiki/lib/model-command.js +45 -8
- package/dist/extensions/llm-wiki/lib/qmd-indexing.js +1024 -0
- package/dist/extensions/llm-wiki/lib/qmd-mirror.js +418 -0
- package/dist/extensions/llm-wiki/lib/qmd-store.js +112 -0
- package/dist/extensions/llm-wiki/lib/recall.js +77 -3
- package/dist/extensions/llm-wiki/lib/runtime.js +25 -1
- package/dist/extensions/llm-wiki/lib/subagent.js +47 -7
- package/dist/extensions/llm-wiki/lib/tools.js +165 -5
- package/dist/extensions/llm-wiki/lib/utils.js +16 -2
- package/dist/extensions/llm-wiki/lib/wiki-service.js +104 -5
- package/dist/mcp/index.js +66 -2
- package/dist/mcp/operations.js +26 -2
- package/docs/api.md +43 -1
- package/docs/architecture.md +28 -0
- package/docs/commands.md +1 -0
- package/docs/qmd-compatibility.md +47 -0
- package/docs/retrieval-benchmark.md +47 -0
- package/docs/superpowers/benchmarks/phase-1-current-baseline.json +53 -0
- package/docs/superpowers/plans/2026-08-09-qmd-retrieval-phase-2-remediation.md +549 -0
- package/docs/superpowers/plans/2026-08-09-qmd-retrieval-phase-2-validated-indexing.md +1493 -0
- package/docs/superpowers/plans/2026-08-11-qmd-retrieval-phase-3-retrieval-modes-and-recall-cutover.md +678 -0
- package/docs/superpowers/plans/2026-09-05-wikilink-alias-pipe-table-only.md +257 -0
- package/extensions/llm-wiki/index.ts +14 -1
- package/extensions/llm-wiki/lib/bootstrap.ts +2 -0
- package/extensions/llm-wiki/lib/indexing.ts +24 -1
- package/extensions/llm-wiki/lib/ingest-worker.ts +10 -2
- package/extensions/llm-wiki/lib/knowledge-document.ts +20 -3
- package/extensions/llm-wiki/lib/knowledge-links.ts +39 -7
- package/extensions/llm-wiki/lib/model-command.ts +57 -12
- package/extensions/llm-wiki/lib/qmd-indexing.ts +1304 -0
- package/extensions/llm-wiki/lib/qmd-mirror.ts +496 -0
- package/extensions/llm-wiki/lib/qmd-store.ts +222 -0
- package/extensions/llm-wiki/lib/recall.ts +77 -3
- package/extensions/llm-wiki/lib/runtime.ts +57 -5
- package/extensions/llm-wiki/lib/subagent.ts +73 -10
- package/extensions/llm-wiki/lib/tools.ts +188 -4
- package/extensions/llm-wiki/lib/utils.ts +21 -2
- package/extensions/llm-wiki/lib/wiki-service.ts +160 -4
- package/mcp/index.ts +78 -1
- package/mcp/operations.ts +41 -2
- package/package.json +9 -6
- package/skills/llm-wiki/SKILL.md +7 -1
|
@@ -1,13 +1,42 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { runAgentLoop, } from "@earendil-works/pi-agent-core";
|
|
2
|
+
let cachedDefaultStreamFn;
|
|
3
|
+
/**
|
|
4
|
+
* The default pi-ai stream function (dispatches to registered API providers).
|
|
5
|
+
* It moved from the package root to the `./compat` subpath in pi-ai 0.85, so
|
|
6
|
+
* it is resolved lazily — a static import of either path breaks the other pi
|
|
7
|
+
* version at load time. Cached after the first resolution.
|
|
8
|
+
*/
|
|
9
|
+
async function resolveDefaultStreamFn() {
|
|
10
|
+
if (cachedDefaultStreamFn)
|
|
11
|
+
return cachedDefaultStreamFn;
|
|
12
|
+
const root = await import("@earendil-works/pi-ai");
|
|
13
|
+
const rootFn = root.streamSimple;
|
|
14
|
+
if (rootFn) {
|
|
15
|
+
cachedDefaultStreamFn = rootFn;
|
|
16
|
+
return rootFn;
|
|
17
|
+
}
|
|
18
|
+
// pi-ai >= 0.85 exposes streamSimple via the ./compat subpath. The
|
|
19
|
+
// specifier is a variable so static tooling (vite in vitest, jiti in pi)
|
|
20
|
+
// cannot resolve a subpath that does not exist in pi < 0.85; at runtime
|
|
21
|
+
// this branch is only reached when the root import lacks streamSimple.
|
|
22
|
+
const compatSpecifier = "@earendil-works/pi-ai/compat";
|
|
23
|
+
const compat = await import(compatSpecifier);
|
|
24
|
+
cachedDefaultStreamFn = compat.streamSimple;
|
|
25
|
+
return cachedDefaultStreamFn;
|
|
26
|
+
}
|
|
2
27
|
/**
|
|
3
28
|
* Run a sub-agent loop to completion.
|
|
4
29
|
*
|
|
5
30
|
* Returns nothing useful directly — by design, results are collected by the
|
|
6
31
|
* `tools` the caller passes (their `execute` accumulates into caller-owned
|
|
7
32
|
* state). This keeps the runner generic across every background task type.
|
|
33
|
+
*
|
|
34
|
+
* Rejections from the loop (provider errors, auth failures, a streamFn that
|
|
35
|
+
* throws) reject this promise, so `BackgroundRuntime.launchTask`'s try/catch
|
|
36
|
+
* degrades them to a warning toast.
|
|
8
37
|
*/
|
|
9
38
|
export async function runSubAgent(args) {
|
|
10
|
-
const { model, apiKey, headers, systemPrompt, userPrompt, tools, maxTokens, signal } = args;
|
|
39
|
+
const { model, apiKey, headers, systemPrompt, userPrompt, tools, maxTokens, signal, streamFn, env, } = args;
|
|
11
40
|
const text = userPrompt.trim();
|
|
12
41
|
if (!text)
|
|
13
42
|
return;
|
|
@@ -32,10 +61,21 @@ export async function runSubAgent(args) {
|
|
|
32
61
|
convertToLlm: (msgs) => msgs,
|
|
33
62
|
toolExecution: "sequential",
|
|
34
63
|
...(reasoning ? { reasoning: "high" } : {}),
|
|
64
|
+
// Provider-scoped env from auth resolution (issue #222). pi-agent-core
|
|
65
|
+
// spreads the config into the stream options, where pi-ai >= 0.85 honors
|
|
66
|
+
// it; the conditional spread keeps this compiling on pi < 0.85.
|
|
67
|
+
...(env ? { env } : {}),
|
|
35
68
|
};
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
69
|
+
// Drive the loop directly instead of agentLoop(): agentLoop() wraps the
|
|
70
|
+
// loop in a detached promise (`void runAgentLoop(...).then(...)` with no
|
|
71
|
+
// .catch), so a rejection from the stream path — e.g. "No API provider
|
|
72
|
+
// registered for api: X" for a model from an extension-registered provider
|
|
73
|
+
// — escaped as an uncaughtException and killed the whole pi process while
|
|
74
|
+
// this function's stream drain hung forever (issue #222). runAgentLoop is
|
|
75
|
+
// the same loop with the rejection propagating to THIS promise.
|
|
76
|
+
// pi >= 0.85 requires streamFn explicitly (its internal fallback throws
|
|
77
|
+
// unless the host configured a default), so we always pass one: the
|
|
78
|
+
// provider-specific function when available, else pi-ai's default.
|
|
79
|
+
const activeStreamFn = streamFn ?? (await resolveDefaultStreamFn());
|
|
80
|
+
await runAgentLoop(prompts, context, config, async () => { }, signal, activeStreamFn);
|
|
41
81
|
}
|
|
@@ -5,15 +5,16 @@ import { bootstrapVault } from "./bootstrap.js";
|
|
|
5
5
|
import { launchEmbedPages, reindexEmbeddings, resolveEmbedder } from "./embeddings.js";
|
|
6
6
|
import { scheduleReindex } from "./indexing.js";
|
|
7
7
|
import { runIngestSynthesis } from "./ingest-worker.js";
|
|
8
|
-
import { createKnowledgeDocument, serializeKnowledgeDocument, writeKnowledgeDocumentFile, } from "./knowledge-document.js";
|
|
8
|
+
import { createKnowledgeDocument, parseMarkdownFrontmatter, serializeKnowledgeDocument, writeKnowledgeDocumentFile, } from "./knowledge-document.js";
|
|
9
9
|
import { applyWikilinkGate, buildResolvedBacklinks, buildWikilinkIndex, } from "./knowledge-links.js";
|
|
10
10
|
import { repairLegacyKnowledgeDocuments } from "./legacy-repair.js";
|
|
11
11
|
import { appendEvent, rebuildMetadata, rebuildMetadataLight } from "./metadata.js";
|
|
12
|
+
import { readQmdIndexStatus, reindexQmdVault } from "./qmd-indexing.js";
|
|
12
13
|
import { captureFile, captureText, captureUrl } from "./source-packet.js";
|
|
13
14
|
import { loadTaskConfig, parseModelRef, resolveWikilinkValidation } from "./task-config.js";
|
|
14
15
|
import { detectVaultFormat, fmtDate, getVaultPaths, readJson, resolveVaultPaths, slugify, writeJson, } from "./utils.js";
|
|
15
16
|
import { assertWritableVault, compareCodePoint, discoverKnowledgeDocuments, inspectVaultFormat, inspectWritableVault, } from "./vault-format.js";
|
|
16
|
-
import { getWikiStatus, searchRegistry } from "./wiki-service.js";
|
|
17
|
+
import { getWikiStatus, reindexWiki, searchRegistry } from "./wiki-service.js";
|
|
17
18
|
/**
|
|
18
19
|
* All LLM Wiki custom tools.
|
|
19
20
|
*/
|
|
@@ -323,6 +324,8 @@ export function registerWikiIngest(pi, runtime) {
|
|
|
323
324
|
model: resolved.model,
|
|
324
325
|
apiKey: resolved.apiKey,
|
|
325
326
|
headers: resolved.headers,
|
|
327
|
+
streamFn: resolved.streamFn,
|
|
328
|
+
env: resolved.env,
|
|
326
329
|
paths,
|
|
327
330
|
sourceId: s.id,
|
|
328
331
|
manifest: s.manifest,
|
|
@@ -410,6 +413,10 @@ export function registerWikiIngest(pi, runtime) {
|
|
|
410
413
|
});
|
|
411
414
|
}
|
|
412
415
|
// ─── 4. wiki_ensure_page ────────────────────────────────
|
|
416
|
+
// Frontmatter fields wiki_ensure_page generates or derives itself (title also
|
|
417
|
+
// determines the filename). Model-supplied values for these are ignored rather
|
|
418
|
+
// than merged (issue #241).
|
|
419
|
+
const RESERVED_FRONTMATTER = new Set(["type", "title", "created", "updated", "sources", "id"]);
|
|
413
420
|
export function registerWikiEnsurePage(pi, runtime) {
|
|
414
421
|
pi.registerTool({
|
|
415
422
|
name: "wiki_ensure_page",
|
|
@@ -419,13 +426,16 @@ export function registerWikiEnsurePage(pi, runtime) {
|
|
|
419
426
|
promptGuidelines: [
|
|
420
427
|
"Use wiki_ensure_page before creating pages to avoid duplicates.",
|
|
421
428
|
"Search existing pages first with wiki_search.",
|
|
429
|
+
"Content may begin with a YAML frontmatter block; its fields are merged into the page frontmatter, and generated fields are reserved.",
|
|
422
430
|
],
|
|
423
431
|
parameters: Type.Object({
|
|
424
432
|
type: Type.String({
|
|
425
433
|
description: "Page type: entity | concept | synthesis | analysis | requirement | skill | case (built-in) or any user-defined type from llm-wiki.customTypes config",
|
|
426
434
|
}),
|
|
427
435
|
title: Type.String({ description: "Page title" }),
|
|
428
|
-
content: Type.Optional(Type.String({
|
|
436
|
+
content: Type.Optional(Type.String({
|
|
437
|
+
description: "Optional Markdown body. May begin with a YAML frontmatter block, whose fields are merged into the page frontmatter; generated fields (type, title, created, updated, sources) are reserved and ignored.",
|
|
438
|
+
})),
|
|
429
439
|
}),
|
|
430
440
|
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
431
441
|
const paths = getPaths(ctx.cwd);
|
|
@@ -465,6 +475,24 @@ export function registerWikiEnsurePage(pi, runtime) {
|
|
|
465
475
|
}
|
|
466
476
|
const today = fmtDate();
|
|
467
477
|
let body = params.content ?? buildPageBody(type, params.title);
|
|
478
|
+
// #241: models sometimes pass a YAML frontmatter block inside content.
|
|
479
|
+
// Consume it (with the same hardened parser used for page reads) and merge
|
|
480
|
+
// its fields into the generated frontmatter, instead of writing the block
|
|
481
|
+
// verbatim into the body (duplicate frontmatter, and the model's fields
|
|
482
|
+
// silently never took effect). Generated fields are reserved; a
|
|
483
|
+
// frontmatter-only content falls back to the template body.
|
|
484
|
+
const extraFrontmatter = {};
|
|
485
|
+
if (body.trimStart().startsWith("---")) {
|
|
486
|
+
const parsed = parseMarkdownFrontmatter(body, `${folder}/${slug}.md`);
|
|
487
|
+
if (parsed.ok) {
|
|
488
|
+
for (const [key, value] of Object.entries(parsed.mapping)) {
|
|
489
|
+
if (!RESERVED_FRONTMATTER.has(key)) {
|
|
490
|
+
extraFrontmatter[key] = value;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
body = parsed.body.trim() ? parsed.body : buildPageBody(type, params.title);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
468
496
|
// Pre-write wikilink gate (#172): validate/normalize caller-supplied content.
|
|
469
497
|
const mode = resolveWikilinkValidation(loadTaskConfig(ctx.cwd));
|
|
470
498
|
let wikilinkIssues = [];
|
|
@@ -494,6 +522,7 @@ export function registerWikiEnsurePage(pi, runtime) {
|
|
|
494
522
|
title: params.title,
|
|
495
523
|
created: today,
|
|
496
524
|
updated: today,
|
|
525
|
+
...extraFrontmatter,
|
|
497
526
|
}, body);
|
|
498
527
|
mkdirSync(join(paths.wiki, folder), { recursive: true });
|
|
499
528
|
writeKnowledgeDocumentFile(pagePath, doc);
|
|
@@ -740,8 +769,9 @@ export function registerWikiLint(pi, runtime) {
|
|
|
740
769
|
* Run the wiki health scan (issue #77 extracted it from the tool body so it can
|
|
741
770
|
* run off-thread via `dispatchReported`). Returns the human-readable summary.
|
|
742
771
|
*/
|
|
743
|
-
function runWikiLint(paths, autoFix) {
|
|
772
|
+
async function runWikiLint(paths, autoFix) {
|
|
744
773
|
assertWritableVault(paths);
|
|
774
|
+
const qmdStatus = await readQmdIndexStatus(paths);
|
|
745
775
|
let repair;
|
|
746
776
|
if (autoFix) {
|
|
747
777
|
let projection = rebuildMetadata(paths);
|
|
@@ -891,6 +921,20 @@ function runWikiLint(paths, autoFix) {
|
|
|
891
921
|
});
|
|
892
922
|
rebuildMetadataLight(paths);
|
|
893
923
|
}
|
|
924
|
+
const qmdFindings = [];
|
|
925
|
+
if (qmdStatus.state === "stale") {
|
|
926
|
+
const components = JSON.stringify(qmdStatus.repairComponents.length > 0 ? qmdStatus.repairComponents : ["lexical"]);
|
|
927
|
+
qmdFindings.push(`- QMD index stale (${qmdStatus.indexedManifestHash ? "manifest or model changed" : ""}): repair with \`wiki_reindex(scope="changed", components=${components}, vault="active")\``);
|
|
928
|
+
}
|
|
929
|
+
else if (qmdStatus.state === "recovering") {
|
|
930
|
+
qmdFindings.push(`- QMD swap interrupted (${qmdStatus.swapPhase ?? ""}): restart recovery via \`wiki_reindex(vault="active")\``);
|
|
931
|
+
}
|
|
932
|
+
else if (qmdStatus.state === "error") {
|
|
933
|
+
qmdFindings.push(`- QMD index error: ${qmdStatus.issues[0]?.message ?? "repair with wiki_reindex"} — \`wiki_reindex(scope="changed", components=${JSON.stringify(qmdStatus.repairComponents.length > 0 ? qmdStatus.repairComponents : ["lexical"])}, vault="active")\``);
|
|
934
|
+
}
|
|
935
|
+
else if (qmdStatus.state === "missing") {
|
|
936
|
+
qmdFindings.push("- QMD index not built yet (informational): run wiki_reindex to build it");
|
|
937
|
+
}
|
|
894
938
|
return [
|
|
895
939
|
"🧹 **LLM Wiki lint complete**",
|
|
896
940
|
"",
|
|
@@ -904,6 +948,10 @@ function runWikiLint(paths, autoFix) {
|
|
|
904
948
|
reportPath ? `📄 Report: \`${reportPath}\`` : "",
|
|
905
949
|
repair?.manifestPath ? `🛟 Repair manifest: \`${repair.manifestPath}\`` : "",
|
|
906
950
|
gaps.length ? `💡 ${gaps.length} knowledge gap(s) tracked` : "",
|
|
951
|
+
"",
|
|
952
|
+
"## QMD Index",
|
|
953
|
+
`- State: ${qmdStatus.state}`,
|
|
954
|
+
...qmdFindings,
|
|
907
955
|
]
|
|
908
956
|
.filter(Boolean)
|
|
909
957
|
.join("\n");
|
|
@@ -927,7 +975,7 @@ export function registerWikiStatus(pi) {
|
|
|
927
975
|
isError: true,
|
|
928
976
|
};
|
|
929
977
|
}
|
|
930
|
-
const status = getWikiStatus(paths);
|
|
978
|
+
const status = await getWikiStatus(paths);
|
|
931
979
|
const config = readJson(join(paths.dotWiki, "config.json"), {});
|
|
932
980
|
const backlinks = readJson(join(paths.meta, "backlinks.json"), {});
|
|
933
981
|
const orphanCount = Object.entries(backlinks).filter(([, inbound]) => inbound.length === 0).length;
|
|
@@ -954,6 +1002,10 @@ export function registerWikiStatus(pi) {
|
|
|
954
1002
|
`Gaps: ${gaps.gaps?.length || 0}`,
|
|
955
1003
|
`Health: ${health}`,
|
|
956
1004
|
`Last updated: ${status.lastUpdated || "Never"}`,
|
|
1005
|
+
`QMD index: ${status.qmd.state}`,
|
|
1006
|
+
`QMD documents: ${status.qmd.totalDocuments} (${status.qmd.canonicalDocuments} canonical, ${status.qmd.evidenceDocuments} evidence)`,
|
|
1007
|
+
`QMD embeddings pending: ${status.qmd.needsEmbedding}`,
|
|
1008
|
+
`QMD package: ${status.qmd.qmdVersion}`,
|
|
957
1009
|
...diagLines,
|
|
958
1010
|
];
|
|
959
1011
|
return {
|
|
@@ -968,6 +1020,7 @@ export function registerWikiStatus(pi) {
|
|
|
968
1020
|
gaps: gaps.gaps?.length || 0,
|
|
969
1021
|
health,
|
|
970
1022
|
blockingDiagnostics: status.blockingDiagnostics,
|
|
1023
|
+
qmd: status.qmd,
|
|
971
1024
|
},
|
|
972
1025
|
};
|
|
973
1026
|
},
|
|
@@ -1009,6 +1062,22 @@ export function registerWikiRebuildMeta(pi, runtime) {
|
|
|
1009
1062
|
return `⚠️ LLM Wiki: rebuild had issues — ${result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("; ")}`;
|
|
1010
1063
|
}
|
|
1011
1064
|
const warnings = result.diagnostics.filter((diagnostic) => diagnostic.severity === "warning");
|
|
1065
|
+
// After a successful projection, keep the generated QMD index in sync
|
|
1066
|
+
// (model-free lexical pass). A QMD failure is a warning, never a
|
|
1067
|
+
// projection failure.
|
|
1068
|
+
const qmdResult = await reindexQmdVault(paths, {
|
|
1069
|
+
scope: "changed",
|
|
1070
|
+
components: ["lexical"],
|
|
1071
|
+
force: false,
|
|
1072
|
+
});
|
|
1073
|
+
if (!qmdResult.ok) {
|
|
1074
|
+
warnings.push({
|
|
1075
|
+
severity: "warning",
|
|
1076
|
+
code: "qmd_index_error",
|
|
1077
|
+
path: paths.qmd,
|
|
1078
|
+
message: qmdResult.errors[0]?.message ?? "QMD indexing failed",
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1012
1081
|
if (warnings.length > 0) {
|
|
1013
1082
|
return `⚠️ LLM Wiki: metadata rebuilt with warnings — ${warnings
|
|
1014
1083
|
.map((diagnostic) => `${diagnostic.code}: ${diagnostic.message}`)
|
|
@@ -1091,6 +1160,97 @@ export function registerWikiReindexEmbeddings(pi, runtime) {
|
|
|
1091
1160
|
},
|
|
1092
1161
|
});
|
|
1093
1162
|
}
|
|
1163
|
+
export function registerWikiReindex(pi) {
|
|
1164
|
+
pi.registerTool({
|
|
1165
|
+
name: "wiki_reindex",
|
|
1166
|
+
label: "Wiki Reindex QMD",
|
|
1167
|
+
description: "Rebuild or repair the generated QMD index (meta/qmd) for the vault. " +
|
|
1168
|
+
"Lexical indexing is model-free; selecting vectors may download " +
|
|
1169
|
+
"approximately 2 GB of models on first use. Repair stale/error state. " +
|
|
1170
|
+
"Active recall still uses the legacy heuristic until Phase 3.",
|
|
1171
|
+
promptSnippet: "Rebuild the QMD search index",
|
|
1172
|
+
promptGuidelines: [
|
|
1173
|
+
"Use wiki_reindex to repair a stale, error, or recovering QMD index.",
|
|
1174
|
+
"Lexical-only reindexing never loads a model.",
|
|
1175
|
+
"Vector reindexing may download approximately 2 GB of models on first use.",
|
|
1176
|
+
],
|
|
1177
|
+
parameters: Type.Object({
|
|
1178
|
+
scope: Type.Optional(Type.Union([Type.Literal("changed"), Type.Literal("all")], { default: "changed" })),
|
|
1179
|
+
components: Type.Optional(Type.Array(Type.Union([Type.Literal("lexical"), Type.Literal("vectors")]), {
|
|
1180
|
+
minItems: 1,
|
|
1181
|
+
uniqueItems: true,
|
|
1182
|
+
default: ["lexical", "vectors"],
|
|
1183
|
+
})),
|
|
1184
|
+
force: Type.Optional(Type.Boolean({ default: false })),
|
|
1185
|
+
vault: Type.Optional(Type.Union([
|
|
1186
|
+
Type.Literal("active"),
|
|
1187
|
+
Type.Literal("personal"),
|
|
1188
|
+
Type.Literal("project"),
|
|
1189
|
+
Type.Literal("all"),
|
|
1190
|
+
], { default: "active" })),
|
|
1191
|
+
}),
|
|
1192
|
+
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
|
1193
|
+
const paths = getPaths(ctx.cwd);
|
|
1194
|
+
const vaultCheck = inspectWritableVault(paths);
|
|
1195
|
+
if (!vaultCheck.ok) {
|
|
1196
|
+
return {
|
|
1197
|
+
content: [
|
|
1198
|
+
{ type: "text", text: `Wiki vault error: ${vaultCheck.diagnostics[0].message}` },
|
|
1199
|
+
],
|
|
1200
|
+
details: {
|
|
1201
|
+
error: vaultCheck.diagnostics[0].code,
|
|
1202
|
+
diagnostics: vaultCheck.diagnostics,
|
|
1203
|
+
},
|
|
1204
|
+
isError: true,
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
const scope = params.scope ?? "changed";
|
|
1208
|
+
const components = params.components ?? ["lexical", "vectors"];
|
|
1209
|
+
const force = params.force === true;
|
|
1210
|
+
const vault = params.vault ?? "active";
|
|
1211
|
+
const lexicalOnly = components.length === 1 && components[0] === "lexical";
|
|
1212
|
+
// Warn before any vector work so the operator expects a large download.
|
|
1213
|
+
if (components.includes("vectors") && signal && signal.aborted) {
|
|
1214
|
+
return {
|
|
1215
|
+
content: [{ type: "text", text: "QMD reindex cancelled before it started." }],
|
|
1216
|
+
details: { cancelled: true },
|
|
1217
|
+
isError: true,
|
|
1218
|
+
};
|
|
1219
|
+
}
|
|
1220
|
+
const result = await reindexWiki(paths, {
|
|
1221
|
+
scope,
|
|
1222
|
+
components,
|
|
1223
|
+
force,
|
|
1224
|
+
vault,
|
|
1225
|
+
signal,
|
|
1226
|
+
});
|
|
1227
|
+
const ok = result.results.every((r) => r.result.ok);
|
|
1228
|
+
const lines = [
|
|
1229
|
+
ok ? "✅ QMD indexing complete" : "⚠️ QMD indexing completed with errors",
|
|
1230
|
+
...result.results.map((r) => {
|
|
1231
|
+
const st = r.result.status;
|
|
1232
|
+
return `- ${r.label} (${r.root}): state=${st.state}, documents=${st.totalDocuments}, indexed=${r.result.documents.indexed}, updated=${r.result.documents.updated}, removed=${r.result.documents.removed}, vectors=${r.result.vectors.generated}`;
|
|
1233
|
+
}),
|
|
1234
|
+
];
|
|
1235
|
+
if (lexicalOnly)
|
|
1236
|
+
lines.push("Model-free lexical indexing — no model was downloaded.");
|
|
1237
|
+
if (components.includes("vectors")) {
|
|
1238
|
+
lines.push("⚠️ Vector indexing may download approximately 2 GB of models on first use.");
|
|
1239
|
+
}
|
|
1240
|
+
for (const r of result.results) {
|
|
1241
|
+
for (const e of r.result.errors)
|
|
1242
|
+
lines.push(`- [${r.label}] ${e.code}: ${e.message}`);
|
|
1243
|
+
for (const w of r.result.warnings)
|
|
1244
|
+
lines.push(`- [${r.label}] ${w.code}: ${w.message}`);
|
|
1245
|
+
}
|
|
1246
|
+
return {
|
|
1247
|
+
content: [{ type: "text", text: lines.join("\n") }],
|
|
1248
|
+
details: { scope, components, ...result },
|
|
1249
|
+
...(ok ? {} : { isError: true }),
|
|
1250
|
+
};
|
|
1251
|
+
},
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1094
1254
|
export function registerWikiLogEvent(pi) {
|
|
1095
1255
|
pi.registerTool({
|
|
1096
1256
|
name: "wiki_log_event",
|
|
@@ -162,30 +162,44 @@ export function resolveVaultRoot(cwd) {
|
|
|
162
162
|
}
|
|
163
163
|
/** Get all vault paths for the new (.llm-wiki) layout. */
|
|
164
164
|
export function getVaultPaths(root) {
|
|
165
|
+
const meta = join(root, ".llm-wiki", "meta");
|
|
166
|
+
const qmd = join(meta, "qmd");
|
|
165
167
|
return {
|
|
166
168
|
root,
|
|
167
169
|
raw: join(root, ".llm-wiki", "raw"),
|
|
168
170
|
rawSources: join(root, ".llm-wiki", "raw", "sources"),
|
|
169
171
|
rawTrajectories: join(root, ".llm-wiki", "raw", "trajectories"),
|
|
170
172
|
wiki: join(root, ".llm-wiki", "wiki"),
|
|
171
|
-
meta
|
|
173
|
+
meta,
|
|
172
174
|
dotWiki: join(root, ".llm-wiki"),
|
|
173
175
|
outputs: join(root, ".llm-wiki", "outputs"),
|
|
174
176
|
discoveries: join(root, ".llm-wiki", ".discoveries"),
|
|
177
|
+
qmd,
|
|
178
|
+
qmdCurrent: join(qmd, "current"),
|
|
179
|
+
qmdDocuments: join(qmd, "documents"),
|
|
180
|
+
qmdManifest: join(qmd, "manifest.json"),
|
|
181
|
+
qmdSwap: join(qmd, "swap.json"),
|
|
175
182
|
};
|
|
176
183
|
}
|
|
177
184
|
/** Get all vault paths for the legacy (.wiki) layout. */
|
|
178
185
|
export function getLegacyVaultPaths(root) {
|
|
186
|
+
const meta = join(root, "meta");
|
|
187
|
+
const qmd = join(meta, "qmd");
|
|
179
188
|
return {
|
|
180
189
|
root,
|
|
181
190
|
raw: join(root, "raw"),
|
|
182
191
|
rawSources: join(root, "raw", "sources"),
|
|
183
192
|
rawTrajectories: join(root, "raw", "trajectories"),
|
|
184
193
|
wiki: join(root, "wiki"),
|
|
185
|
-
meta
|
|
194
|
+
meta,
|
|
186
195
|
dotWiki: join(root, ".wiki"),
|
|
187
196
|
outputs: join(root, "outputs"),
|
|
188
197
|
discoveries: join(root, ".discoveries"),
|
|
198
|
+
qmd,
|
|
199
|
+
qmdCurrent: join(qmd, "current"),
|
|
200
|
+
qmdDocuments: join(qmd, "documents"),
|
|
201
|
+
qmdManifest: join(qmd, "manifest.json"),
|
|
202
|
+
qmdSwap: join(qmd, "swap.json"),
|
|
189
203
|
};
|
|
190
204
|
}
|
|
191
205
|
/**
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { awaitQmdIndexQueue, readQmdIndexStatus, reindexQmdVault, } from "./qmd-indexing.js";
|
|
4
|
+
import { QMD_PACKAGE_VERSION, resolveQmdModels } from "./qmd-store.js";
|
|
5
|
+
import { getPersonalWikiPaths, isPersonalVault, readJson } from "./utils.js";
|
|
6
|
+
import { compareCodePoint, discoverKnowledgeDocuments, inspectVaultFormat, inspectWritableVault, } from "./vault-format.js";
|
|
5
7
|
/**
|
|
6
8
|
* Search the registry for matching concepts.
|
|
7
9
|
*
|
|
8
|
-
* Matches ID,
|
|
10
|
+
* Matches ID, title, type, state, status, category, domain, tags, aliases, and recall triggers.
|
|
9
11
|
* Preserves unknown types as strings.
|
|
10
12
|
*/
|
|
11
13
|
export function searchRegistry(paths, query, typeFilter) {
|
|
@@ -55,6 +57,16 @@ function matchesField(id, entry, query) {
|
|
|
55
57
|
.toLowerCase()
|
|
56
58
|
.includes(query))
|
|
57
59
|
return true;
|
|
60
|
+
// Match state
|
|
61
|
+
if (String(entry.state || "")
|
|
62
|
+
.toLowerCase()
|
|
63
|
+
.includes(query))
|
|
64
|
+
return true;
|
|
65
|
+
// Match status
|
|
66
|
+
if (String(entry.status || "")
|
|
67
|
+
.toLowerCase()
|
|
68
|
+
.includes(query))
|
|
69
|
+
return true;
|
|
58
70
|
// Match category/domain
|
|
59
71
|
if (String(entry.category || "")
|
|
60
72
|
.toLowerCase()
|
|
@@ -96,9 +108,10 @@ function matchesField(id, entry, query) {
|
|
|
96
108
|
/**
|
|
97
109
|
* Get a status snapshot of the wiki.
|
|
98
110
|
*
|
|
99
|
-
* Reports resolved knowledge_format, page counts,
|
|
111
|
+
* Reports resolved knowledge_format, page counts, blocking diagnostics, and
|
|
112
|
+
* generated QMD index status (read without opening any QMD store).
|
|
100
113
|
*/
|
|
101
|
-
export function getWikiStatus(paths) {
|
|
114
|
+
export async function getWikiStatus(paths) {
|
|
102
115
|
const vaultState = inspectVaultFormat(paths);
|
|
103
116
|
const diagnostics = [...vaultState.diagnostics];
|
|
104
117
|
// Also check discovery for current concept health
|
|
@@ -124,5 +137,91 @@ export function getWikiStatus(paths) {
|
|
|
124
137
|
byType,
|
|
125
138
|
blockingDiagnostics: diagnostics.filter((d) => d.severity === "error"),
|
|
126
139
|
lastUpdated: registry?.last_updated || "",
|
|
140
|
+
qmd: await readQmdIndexStatus(paths),
|
|
127
141
|
};
|
|
128
142
|
}
|
|
143
|
+
function blockedReindexResult(scope, components, code, message) {
|
|
144
|
+
return {
|
|
145
|
+
ok: false,
|
|
146
|
+
scope,
|
|
147
|
+
components,
|
|
148
|
+
documents: { indexed: 0, updated: 0, unchanged: 0, removed: 0 },
|
|
149
|
+
vectors: { generated: 0, skipped: 0, errors: 0 },
|
|
150
|
+
elapsedMs: 0,
|
|
151
|
+
status: {
|
|
152
|
+
state: "error",
|
|
153
|
+
qmdVersion: QMD_PACKAGE_VERSION,
|
|
154
|
+
models: resolveQmdModels(),
|
|
155
|
+
totalDocuments: 0,
|
|
156
|
+
canonicalDocuments: 0,
|
|
157
|
+
evidenceDocuments: 0,
|
|
158
|
+
needsEmbedding: 0,
|
|
159
|
+
hasVectorIndex: false,
|
|
160
|
+
repairComponents: [],
|
|
161
|
+
issues: [{ code, message }],
|
|
162
|
+
},
|
|
163
|
+
warnings: [],
|
|
164
|
+
errors: [{ code, message }],
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Reindex QMD stores for selected vaults, sequentially. Validates each vault
|
|
169
|
+
* immediately before work. One vault's failure does not prevent the others.
|
|
170
|
+
*/
|
|
171
|
+
export async function reindexWiki(activePaths, input) {
|
|
172
|
+
const scope = input.scope ?? "changed";
|
|
173
|
+
const components = input.components ?? ["lexical", "vectors"];
|
|
174
|
+
const force = input.force ?? false;
|
|
175
|
+
const vault = input.vault ?? "active";
|
|
176
|
+
const signal = input.signal;
|
|
177
|
+
const onProgress = input.onProgress;
|
|
178
|
+
const personalPaths = getPersonalWikiPaths();
|
|
179
|
+
const activeIsPersonal = isPersonalVault(activePaths);
|
|
180
|
+
const targets = [];
|
|
181
|
+
switch (vault) {
|
|
182
|
+
case "active":
|
|
183
|
+
targets.push({ paths: activePaths, label: activeIsPersonal ? "personal" : "active" });
|
|
184
|
+
break;
|
|
185
|
+
case "personal":
|
|
186
|
+
targets.push({ paths: personalPaths, label: "personal" });
|
|
187
|
+
break;
|
|
188
|
+
case "project":
|
|
189
|
+
if (!activeIsPersonal)
|
|
190
|
+
targets.push({ paths: activePaths, label: "project" });
|
|
191
|
+
break;
|
|
192
|
+
case "all":
|
|
193
|
+
if (!activeIsPersonal)
|
|
194
|
+
targets.push({ paths: activePaths, label: "project" });
|
|
195
|
+
targets.push({ paths: personalPaths, label: "personal" });
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
198
|
+
const seen = new Set();
|
|
199
|
+
const results = [];
|
|
200
|
+
for (const target of targets) {
|
|
201
|
+
if (seen.has(target.paths.root))
|
|
202
|
+
continue;
|
|
203
|
+
seen.add(target.paths.root);
|
|
204
|
+
const check = inspectWritableVault(target.paths);
|
|
205
|
+
if (!check.ok) {
|
|
206
|
+
results.push({
|
|
207
|
+
root: target.paths.root,
|
|
208
|
+
label: target.label,
|
|
209
|
+
result: blockedReindexResult(scope, components, check.diagnostics[0]?.code ?? "config_invalid_knowledge_format", check.diagnostics[0]?.message ?? "Vault is not writable"),
|
|
210
|
+
});
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
const result = await reindexQmdVault(target.paths, {
|
|
214
|
+
scope,
|
|
215
|
+
components,
|
|
216
|
+
force,
|
|
217
|
+
signal,
|
|
218
|
+
onProgress: (p) => onProgress?.({ vault: target.paths.root, progress: p }),
|
|
219
|
+
});
|
|
220
|
+
results.push({ root: target.paths.root, label: target.label, result });
|
|
221
|
+
}
|
|
222
|
+
return { vault, results };
|
|
223
|
+
}
|
|
224
|
+
/** Test-only: drain/await in-process QMD reindex queue work for a vault root. */
|
|
225
|
+
export function awaitWikiQmdIndexQueue(root) {
|
|
226
|
+
return awaitQmdIndexQueue(root);
|
|
227
|
+
}
|
package/dist/mcp/index.js
CHANGED
|
@@ -13,10 +13,11 @@ import { join } from "node:path";
|
|
|
13
13
|
import { McpServer } from "@modelcontextprotocol/server";
|
|
14
14
|
import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
|
|
15
15
|
import * as z from "zod/v4";
|
|
16
|
+
import { recoverQmdIndex } from "../extensions/llm-wiki/lib/qmd-indexing.js";
|
|
16
17
|
import { loadTaskConfig, resolveWikilinkValidation, } from "../extensions/llm-wiki/lib/task-config.js";
|
|
17
18
|
import { getVaultPaths, resolveVaultPaths } from "../extensions/llm-wiki/lib/utils.js";
|
|
18
19
|
import { createExecApi } from "./exec.js";
|
|
19
|
-
import { bootstrapOperation, captureSourceOperation, recallOperation, retroOperation, searchOperation, statusOperation, } from "./operations.js";
|
|
20
|
+
import { bootstrapOperation, captureSourceOperation, recallOperation, reindexOperation, retroOperation, searchOperation, statusOperation, } from "./operations.js";
|
|
20
21
|
const execApi = createExecApi();
|
|
21
22
|
// ─── Vault Detection ────────────────────────────────────
|
|
22
23
|
/** Resolve vault paths, same as Pi extension. */
|
|
@@ -48,7 +49,7 @@ const server = new McpServer({
|
|
|
48
49
|
// ---- wiki_bootstrap ----
|
|
49
50
|
//
|
|
50
51
|
// Registered first, and the only tool not gated on an existing vault: the
|
|
51
|
-
// other
|
|
52
|
+
// other tools fail closed with a message naming this one, which an MCP-only
|
|
52
53
|
// client could not act on while it was extension-only (issue #130).
|
|
53
54
|
server.registerTool("wiki_bootstrap", {
|
|
54
55
|
description: "Create an LLM Wiki vault at this server's wiki root (WIKI_ROOT, or the working directory). Writes config, schema, templates and metadata scaffolding. Run this first when no vault exists; safe to re-run on an existing vault, where it updates the config and rebuilds metadata without touching pages.",
|
|
@@ -182,6 +183,56 @@ server.registerTool("wiki_status", {
|
|
|
182
183
|
],
|
|
183
184
|
};
|
|
184
185
|
});
|
|
186
|
+
// ---- wiki_reindex ----
|
|
187
|
+
server.registerTool("wiki_reindex", {
|
|
188
|
+
description: "Rebuild or repair the generated QMD search index (meta/qmd) for the vault. " +
|
|
189
|
+
"Lexical indexing is model-free; selecting vectors may download approximately 2 GB " +
|
|
190
|
+
"of models on first use. Options: scope (changed|all), components (lexical|vectors), " +
|
|
191
|
+
"force, vault (active|personal|project|all).",
|
|
192
|
+
inputSchema: z.object({
|
|
193
|
+
scope: z.enum(["changed", "all"]).optional().default("changed").describe("changed or all"),
|
|
194
|
+
components: z
|
|
195
|
+
.array(z.enum(["lexical", "vectors"]))
|
|
196
|
+
.min(1)
|
|
197
|
+
.optional()
|
|
198
|
+
.describe("Index components (lexical|vectors)"),
|
|
199
|
+
force: z.boolean().optional().describe("Force full rebuild of selected components"),
|
|
200
|
+
vault: z
|
|
201
|
+
.enum(["active", "personal", "project", "all"])
|
|
202
|
+
.optional()
|
|
203
|
+
.default("active")
|
|
204
|
+
.describe("Which vaults to reindex"),
|
|
205
|
+
}),
|
|
206
|
+
}, async ({ scope, components, force, vault }) => {
|
|
207
|
+
if (!hasVault()) {
|
|
208
|
+
return {
|
|
209
|
+
content: [
|
|
210
|
+
{
|
|
211
|
+
type: "text",
|
|
212
|
+
text: "No wiki vault found. Set WIKI_ROOT or run wiki_bootstrap first.",
|
|
213
|
+
},
|
|
214
|
+
],
|
|
215
|
+
isError: true,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
const paths = getPaths();
|
|
219
|
+
const result = await reindexOperation(paths, {
|
|
220
|
+
scope,
|
|
221
|
+
components,
|
|
222
|
+
force,
|
|
223
|
+
vault,
|
|
224
|
+
});
|
|
225
|
+
const ok = result.results.every((r) => r.result.ok);
|
|
226
|
+
return {
|
|
227
|
+
content: [
|
|
228
|
+
{
|
|
229
|
+
type: "text",
|
|
230
|
+
text: JSON.stringify(result, null, 2),
|
|
231
|
+
},
|
|
232
|
+
],
|
|
233
|
+
...(ok ? {} : { isError: true }),
|
|
234
|
+
};
|
|
235
|
+
});
|
|
185
236
|
// ---- wiki_retro ----
|
|
186
237
|
server.registerTool("wiki_retro", {
|
|
187
238
|
description: "Save an atomic insight from a completed task into the wiki. Creates a source page.",
|
|
@@ -276,6 +327,19 @@ async function main() {
|
|
|
276
327
|
const transport = new StdioServerTransport();
|
|
277
328
|
await server.connect(transport);
|
|
278
329
|
console.error("🧠 LLM Wiki MCP Server running on stdio");
|
|
330
|
+
// Fire-and-forget QMD index recovery AFTER the transport is connected so
|
|
331
|
+
// clients are never blocked. A busy/live lock just logs a warning and MCP
|
|
332
|
+
// continues with current state untouched.
|
|
333
|
+
if (hasVault()) {
|
|
334
|
+
const paths = getPaths();
|
|
335
|
+
recoverQmdIndex(paths)
|
|
336
|
+
.then((result) => {
|
|
337
|
+
if (!result.ok) {
|
|
338
|
+
console.error(`[llm-wiki] QMD recovery skipped: ${result.diagnostics[0]?.message ?? "locked"}`);
|
|
339
|
+
}
|
|
340
|
+
})
|
|
341
|
+
.catch((err) => console.error(`[llm-wiki] QMD recovery failed: ${err.message}`));
|
|
342
|
+
}
|
|
279
343
|
}
|
|
280
344
|
main().catch((err) => {
|
|
281
345
|
console.error("MCP Server error:", err);
|