@zosmaai/pi-llm-wiki 0.12.1 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/README.md +16 -8
- package/dist/extensions/llm-wiki/lib/bootstrap.js +2 -0
- package/dist/extensions/llm-wiki/lib/indexing.js +24 -1
- package/dist/extensions/llm-wiki/lib/ingest-worker.js +3 -1
- package/dist/extensions/llm-wiki/lib/knowledge-links.js +41 -6
- package/dist/extensions/llm-wiki/lib/model-command.js +45 -8
- package/dist/extensions/llm-wiki/lib/qmd-indexing.js +1024 -0
- package/dist/extensions/llm-wiki/lib/qmd-mirror.js +418 -0
- package/dist/extensions/llm-wiki/lib/qmd-store.js +112 -0
- package/dist/extensions/llm-wiki/lib/recall.js +77 -3
- package/dist/extensions/llm-wiki/lib/runtime.js +25 -1
- package/dist/extensions/llm-wiki/lib/subagent.js +47 -7
- package/dist/extensions/llm-wiki/lib/tools.js +165 -5
- package/dist/extensions/llm-wiki/lib/utils.js +16 -2
- package/dist/extensions/llm-wiki/lib/wiki-service.js +104 -5
- package/dist/mcp/index.js +66 -2
- package/dist/mcp/operations.js +26 -2
- package/docs/api.md +43 -1
- package/docs/architecture.md +28 -0
- package/docs/commands.md +1 -0
- package/docs/qmd-compatibility.md +47 -0
- package/docs/retrieval-benchmark.md +47 -0
- package/docs/superpowers/benchmarks/phase-1-current-baseline.json +53 -0
- package/docs/superpowers/plans/2026-08-09-qmd-retrieval-phase-2-remediation.md +549 -0
- package/docs/superpowers/plans/2026-08-09-qmd-retrieval-phase-2-validated-indexing.md +1493 -0
- package/docs/superpowers/plans/2026-08-11-qmd-retrieval-phase-3-retrieval-modes-and-recall-cutover.md +678 -0
- package/docs/superpowers/plans/2026-09-05-wikilink-alias-pipe-table-only.md +257 -0
- package/extensions/llm-wiki/index.ts +14 -1
- package/extensions/llm-wiki/lib/bootstrap.ts +2 -0
- package/extensions/llm-wiki/lib/indexing.ts +24 -1
- package/extensions/llm-wiki/lib/ingest-worker.ts +10 -2
- package/extensions/llm-wiki/lib/knowledge-document.ts +8 -1
- package/extensions/llm-wiki/lib/knowledge-links.ts +39 -7
- package/extensions/llm-wiki/lib/model-command.ts +57 -12
- package/extensions/llm-wiki/lib/qmd-indexing.ts +1304 -0
- package/extensions/llm-wiki/lib/qmd-mirror.ts +496 -0
- package/extensions/llm-wiki/lib/qmd-store.ts +222 -0
- package/extensions/llm-wiki/lib/recall.ts +77 -3
- package/extensions/llm-wiki/lib/runtime.ts +57 -5
- package/extensions/llm-wiki/lib/subagent.ts +73 -10
- package/extensions/llm-wiki/lib/tools.ts +188 -4
- package/extensions/llm-wiki/lib/utils.ts +21 -2
- package/extensions/llm-wiki/lib/wiki-service.ts +160 -4
- package/mcp/index.ts +78 -1
- package/mcp/operations.ts +41 -2
- package/package.json +9 -6
- package/skills/llm-wiki/SKILL.md +7 -1
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { serializeKnowledgeDocument } from "./knowledge-document.js";
|
|
5
|
+
import { discoverKnowledgeDocuments } from "./vault-format.js";
|
|
6
|
+
export const QMD_MANIFEST_VERSION = 1;
|
|
7
|
+
const CANONICAL_TYPES = new Set([
|
|
8
|
+
"concept",
|
|
9
|
+
"entity",
|
|
10
|
+
"analysis",
|
|
11
|
+
"synthesis",
|
|
12
|
+
"requirement",
|
|
13
|
+
"skill",
|
|
14
|
+
"case",
|
|
15
|
+
]);
|
|
16
|
+
/** Classify a page type into a QMD collection role. Unknown types are evidence. */
|
|
17
|
+
export function roleForDocumentType(type) {
|
|
18
|
+
return CANONICAL_TYPES.has(type.trim().toLowerCase()) ? "canonical" : "evidence";
|
|
19
|
+
}
|
|
20
|
+
/** Build the deterministic generated mirror path for a role + page id. */
|
|
21
|
+
export function manifestKey(role, pageId) {
|
|
22
|
+
return `${["documents", role, ...pageId.split("/")].join("/")}.md`;
|
|
23
|
+
}
|
|
24
|
+
export function hashQmdContent(content) {
|
|
25
|
+
return createHash("sha256").update(content, "utf8").digest("hex");
|
|
26
|
+
}
|
|
27
|
+
export function hashQmdManifest(manifest) {
|
|
28
|
+
const entries = Object.fromEntries(Object.entries(manifest.entries).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0));
|
|
29
|
+
return hashQmdContent(JSON.stringify({ version: manifest.version, vaultId: manifest.vaultId, entries }));
|
|
30
|
+
}
|
|
31
|
+
async function atomicWrite(path, content) {
|
|
32
|
+
await mkdir(dirname(path), { recursive: true });
|
|
33
|
+
const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`;
|
|
34
|
+
await writeFile(temporary, content, { encoding: "utf8", flag: "wx" });
|
|
35
|
+
await rename(temporary, path);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Resolve a manifest key to a physical path, rejecting anything that could
|
|
39
|
+
* escape `paths.qmd` or traverse via `..`. Never trust paths from JSON.
|
|
40
|
+
*/
|
|
41
|
+
function manifestKeyToPath(paths, key) {
|
|
42
|
+
if (isAbsolute(key)) {
|
|
43
|
+
throw new Error("qmd_manifest_invalid: manifest key is absolute");
|
|
44
|
+
}
|
|
45
|
+
const parts = key.split("/");
|
|
46
|
+
if (parts.some((part) => part === ".." || part === "" || part.includes("\\"))) {
|
|
47
|
+
throw new Error("qmd_manifest_invalid: manifest key contains unsafe path segments");
|
|
48
|
+
}
|
|
49
|
+
if (parts[0] !== "documents") {
|
|
50
|
+
throw new Error("qmd_manifest_invalid: manifest key outside documents");
|
|
51
|
+
}
|
|
52
|
+
const resolved = resolve(paths.qmd, key);
|
|
53
|
+
if (!resolved.startsWith(resolve(paths.qmd) + sep)) {
|
|
54
|
+
throw new Error("qmd_manifest_invalid: manifest key escapes qmd directory");
|
|
55
|
+
}
|
|
56
|
+
return resolved;
|
|
57
|
+
}
|
|
58
|
+
/** Load and validate a previously published manifest for this vault. */
|
|
59
|
+
export async function readQmdManifest(paths, expectedVaultId) {
|
|
60
|
+
let raw;
|
|
61
|
+
try {
|
|
62
|
+
raw = await readFile(paths.qmdManifest, "utf8");
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return { version: QMD_MANIFEST_VERSION, vaultId: expectedVaultId, entries: {} };
|
|
66
|
+
}
|
|
67
|
+
let data;
|
|
68
|
+
try {
|
|
69
|
+
data = JSON.parse(raw);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
throw new Error("qmd_manifest_invalid: manifest is not valid JSON");
|
|
73
|
+
}
|
|
74
|
+
if (typeof data !== "object" ||
|
|
75
|
+
data === null ||
|
|
76
|
+
data.version !== QMD_MANIFEST_VERSION) {
|
|
77
|
+
throw new Error("qmd_manifest_invalid: unsupported manifest version");
|
|
78
|
+
}
|
|
79
|
+
const vaultId = data.vaultId;
|
|
80
|
+
if (typeof vaultId !== "string" || vaultId !== expectedVaultId) {
|
|
81
|
+
throw new Error("qmd_manifest_invalid: manifest vaultId mismatch");
|
|
82
|
+
}
|
|
83
|
+
const rawEntries = data.entries;
|
|
84
|
+
if (typeof rawEntries !== "object" || rawEntries === null) {
|
|
85
|
+
throw new Error("qmd_manifest_invalid: manifest entries missing");
|
|
86
|
+
}
|
|
87
|
+
const entries = {};
|
|
88
|
+
const wikiRoot = resolve(paths.wiki) + sep;
|
|
89
|
+
for (const [key, value] of Object.entries(rawEntries)) {
|
|
90
|
+
// Reject absolute keys and traversal keys before trusting any entry.
|
|
91
|
+
if (isAbsolute(key) || key.includes("\\") || key.split("/").includes("..")) {
|
|
92
|
+
throw new Error("qmd_manifest_invalid: unsafe manifest key");
|
|
93
|
+
}
|
|
94
|
+
const entry = value;
|
|
95
|
+
if (typeof entry !== "object" ||
|
|
96
|
+
entry === null ||
|
|
97
|
+
typeof entry.sourcePath !== "string" ||
|
|
98
|
+
typeof entry.pageId !== "string" ||
|
|
99
|
+
typeof entry.contentHash !== "string" ||
|
|
100
|
+
typeof entry.type !== "string" ||
|
|
101
|
+
(entry.role !== "canonical" && entry.role !== "evidence")) {
|
|
102
|
+
throw new Error("qmd_manifest_invalid: malformed manifest entry");
|
|
103
|
+
}
|
|
104
|
+
// The key is deterministic: documents/<role>/<pageId>.md. The entry must
|
|
105
|
+
// match both the role and page id encoded in its key.
|
|
106
|
+
if (!key.startsWith("documents/")) {
|
|
107
|
+
throw new Error("qmd_manifest_invalid: manifest key outside documents");
|
|
108
|
+
}
|
|
109
|
+
const rest = key.slice("documents/".length);
|
|
110
|
+
const slash = rest.indexOf("/");
|
|
111
|
+
if (slash <= 0 || !rest.endsWith(".md")) {
|
|
112
|
+
throw new Error("qmd_manifest_invalid: manifest key lacks role/page id");
|
|
113
|
+
}
|
|
114
|
+
const keyRole = rest.slice(0, slash);
|
|
115
|
+
const keyPageId = rest.slice(slash + 1, -3);
|
|
116
|
+
if (keyRole !== entry.role || keyPageId !== entry.pageId || keyPageId === "") {
|
|
117
|
+
throw new Error("qmd_manifest_invalid: manifest key/entry mismatch");
|
|
118
|
+
}
|
|
119
|
+
if (entry.vaultId !== vaultId) {
|
|
120
|
+
throw new Error("qmd_manifest_invalid: entry vaultId mismatch");
|
|
121
|
+
}
|
|
122
|
+
if (entry.type.trim() === "") {
|
|
123
|
+
throw new Error("qmd_manifest_invalid: entry type is empty");
|
|
124
|
+
}
|
|
125
|
+
if (!/^[0-9a-f]{64}$/.test(entry.contentHash)) {
|
|
126
|
+
throw new Error("qmd_manifest_invalid: entry contentHash is not a sha256 hex");
|
|
127
|
+
}
|
|
128
|
+
// Source paths are authoritative page files under paths.wiki, therefore
|
|
129
|
+
// under paths.root and outside paths.qmd.
|
|
130
|
+
const source = resolve(entry.sourcePath);
|
|
131
|
+
if (!isAbsolute(entry.sourcePath) || !source.startsWith(wikiRoot)) {
|
|
132
|
+
throw new Error("qmd_manifest_invalid: entry sourcePath outside wiki");
|
|
133
|
+
}
|
|
134
|
+
entries[key] = entry;
|
|
135
|
+
}
|
|
136
|
+
return { version: QMD_MANIFEST_VERSION, vaultId, entries };
|
|
137
|
+
}
|
|
138
|
+
function mirrorDiagnostic(code, path, message) {
|
|
139
|
+
return { severity: "warning", code, path, message };
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Reconcile the generated QMD mirror against parser-valid authoritative pages.
|
|
143
|
+
* Publishes a validated manifest and only writes/removes generated files.
|
|
144
|
+
*/
|
|
145
|
+
export async function reconcileQmdMirror(paths, vaultId, scope) {
|
|
146
|
+
const diagnostics = [];
|
|
147
|
+
const discovery = discoverKnowledgeDocuments(paths);
|
|
148
|
+
diagnostics.push(...discovery.diagnostics);
|
|
149
|
+
let prior;
|
|
150
|
+
try {
|
|
151
|
+
prior = await readQmdManifest(paths, vaultId);
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
prior = { version: QMD_MANIFEST_VERSION, vaultId, entries: {} };
|
|
155
|
+
diagnostics.push(mirrorDiagnostic("qmd_manifest_invalid", paths.qmdManifest, error.message));
|
|
156
|
+
}
|
|
157
|
+
// Build the desired manifest from parser-valid documents. Serialize each
|
|
158
|
+
// document once; the mirror file and its content hash use that exact string.
|
|
159
|
+
const desired = { version: QMD_MANIFEST_VERSION, vaultId, entries: {} };
|
|
160
|
+
const serializedByKey = new Map();
|
|
161
|
+
for (const doc of discovery.documents) {
|
|
162
|
+
const role = roleForDocumentType(doc.frontmatter.type);
|
|
163
|
+
const key = manifestKey(role, doc.id);
|
|
164
|
+
const serialized = serializeKnowledgeDocument(doc);
|
|
165
|
+
serializedByKey.set(key, serialized);
|
|
166
|
+
desired.entries[key] = {
|
|
167
|
+
sourcePath: doc.absolutePath,
|
|
168
|
+
vaultId,
|
|
169
|
+
pageId: doc.id,
|
|
170
|
+
contentHash: hashQmdContent(serialized),
|
|
171
|
+
role,
|
|
172
|
+
type: String(doc.frontmatter.type),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
const counts = { indexed: 0, updated: 0, unchanged: 0, removed: 0 };
|
|
176
|
+
// Determine which prior entries are unsafe (deleted, malformed, or role-moved)
|
|
177
|
+
// and must be removed from the manifest before files change.
|
|
178
|
+
const removedEntries = {};
|
|
179
|
+
for (const [key, entry] of Object.entries(prior.entries)) {
|
|
180
|
+
const desiredEntry = desired.entries[key];
|
|
181
|
+
if (!desiredEntry || desiredEntry.pageId !== entry.pageId) {
|
|
182
|
+
removedEntries[key] = entry;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
if (Object.keys(removedEntries).length > 0) {
|
|
186
|
+
const intermediate = {
|
|
187
|
+
version: QMD_MANIFEST_VERSION,
|
|
188
|
+
vaultId,
|
|
189
|
+
entries: Object.fromEntries(Object.entries(prior.entries).filter(([key]) => !removedEntries[key])),
|
|
190
|
+
};
|
|
191
|
+
await atomicWrite(paths.qmdManifest, JSON.stringify(intermediate, null, 2));
|
|
192
|
+
}
|
|
193
|
+
// Write new/changed mirror files.
|
|
194
|
+
for (const [key, entry] of Object.entries(desired.entries)) {
|
|
195
|
+
const priorEntry = prior.entries[key];
|
|
196
|
+
const mirrorPath = manifestKeyToPath(paths, key);
|
|
197
|
+
const content = serializedByKey.get(key) ?? "";
|
|
198
|
+
const changed = !priorEntry || priorEntry.contentHash !== entry.contentHash;
|
|
199
|
+
if (scope === "all" || changed) {
|
|
200
|
+
// Full scope still rewrites every file; count unchanged pages by their
|
|
201
|
+
// content identity so the totals reconcile with the final manifest.
|
|
202
|
+
if (changed) {
|
|
203
|
+
await atomicWrite(mirrorPath, content);
|
|
204
|
+
if (!priorEntry)
|
|
205
|
+
counts.indexed++;
|
|
206
|
+
else
|
|
207
|
+
counts.updated++;
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
counts.unchanged++;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
else {
|
|
214
|
+
counts.unchanged++;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
// Remove orphaned generated files that are no longer in the desired manifest.
|
|
218
|
+
await removeOrphans(paths, desired);
|
|
219
|
+
// Final desired manifest.
|
|
220
|
+
await atomicWrite(paths.qmdManifest, JSON.stringify(desired, null, 2));
|
|
221
|
+
counts.removed = Object.keys(removedEntries).length;
|
|
222
|
+
// Remove generated mirror files for removed entries.
|
|
223
|
+
for (const key of Object.keys(removedEntries)) {
|
|
224
|
+
try {
|
|
225
|
+
await rm(manifestKeyToPath(paths, key), { force: true });
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
// Already absent.
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
const finalManifest = desired;
|
|
232
|
+
return {
|
|
233
|
+
manifest: finalManifest,
|
|
234
|
+
manifestHash: hashQmdManifest(finalManifest),
|
|
235
|
+
counts,
|
|
236
|
+
diagnostics,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Remove generated mirror files under documents/ that are not referenced by the
|
|
241
|
+
* desired manifest, plus empty directories. Does not follow symlinks.
|
|
242
|
+
*/
|
|
243
|
+
async function removeOrphans(paths, desired) {
|
|
244
|
+
const documentsRoot = join(paths.qmd, "documents");
|
|
245
|
+
const referenced = new Set(Object.keys(desired.entries));
|
|
246
|
+
async function walk(dir) {
|
|
247
|
+
let entries;
|
|
248
|
+
try {
|
|
249
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
let isEmpty = true;
|
|
255
|
+
for (const entry of entries) {
|
|
256
|
+
const full = join(dir, entry.name);
|
|
257
|
+
if (entry.isDirectory()) {
|
|
258
|
+
await walk(full);
|
|
259
|
+
let stillEmpty;
|
|
260
|
+
try {
|
|
261
|
+
stillEmpty = (await readdir(full)).length === 0;
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
stillEmpty = true;
|
|
265
|
+
}
|
|
266
|
+
if (stillEmpty) {
|
|
267
|
+
try {
|
|
268
|
+
await rm(full, { recursive: false, force: true });
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
// Ignore
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
else {
|
|
275
|
+
isEmpty = false;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
else if (entry.name.endsWith(".md")) {
|
|
279
|
+
// Manifest keys are rooted at paths.qmd (e.g. "documents/canonical/...").
|
|
280
|
+
const rel = relative(paths.qmd, full).split(sep).join("/");
|
|
281
|
+
if (!referenced.has(rel)) {
|
|
282
|
+
try {
|
|
283
|
+
await rm(full, { force: true });
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
// Ignore
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
isEmpty = false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
isEmpty = false;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
if (isEmpty && dir !== documentsRoot) {
|
|
298
|
+
try {
|
|
299
|
+
await rm(dir, { recursive: false, force: true });
|
|
300
|
+
}
|
|
301
|
+
catch {
|
|
302
|
+
// Ignore
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
try {
|
|
307
|
+
await stat(documentsRoot);
|
|
308
|
+
await walk(documentsRoot);
|
|
309
|
+
}
|
|
310
|
+
catch {
|
|
311
|
+
// No documents dir yet.
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Safety-only path used when metadata projection fails: may only REMOVE mirror
|
|
316
|
+
* entries for missing or rejected pages. Never adds or updates valid documents.
|
|
317
|
+
*/
|
|
318
|
+
export async function invalidateUnsafeQmdEntries(paths, vaultId) {
|
|
319
|
+
const diagnostics = [];
|
|
320
|
+
const discovery = discoverKnowledgeDocuments(paths);
|
|
321
|
+
diagnostics.push(...discovery.diagnostics);
|
|
322
|
+
let prior;
|
|
323
|
+
let priorUnsafe = false;
|
|
324
|
+
try {
|
|
325
|
+
prior = await readQmdManifest(paths, vaultId);
|
|
326
|
+
}
|
|
327
|
+
catch {
|
|
328
|
+
// Fail closed in the removal direction: a corrupt prior manifest cannot be
|
|
329
|
+
// trusted, so treat every previously generated mirror entry as unsafe and
|
|
330
|
+
// remove only files we can enumerate safely from disk.
|
|
331
|
+
prior = { version: QMD_MANIFEST_VERSION, vaultId, entries: {} };
|
|
332
|
+
priorUnsafe = true;
|
|
333
|
+
}
|
|
334
|
+
if (priorUnsafe) {
|
|
335
|
+
// Remove every generated mirror file under documents/ that can be
|
|
336
|
+
// enumerated safely, so stale deleted-page candidates cannot survive.
|
|
337
|
+
const documentsRoot = join(paths.qmd, "documents");
|
|
338
|
+
const removedFiles = [];
|
|
339
|
+
async function collect(dir) {
|
|
340
|
+
let entries;
|
|
341
|
+
try {
|
|
342
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
343
|
+
}
|
|
344
|
+
catch {
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
for (const entry of entries) {
|
|
348
|
+
const full = join(dir, entry.name);
|
|
349
|
+
if (entry.isDirectory()) {
|
|
350
|
+
await collect(full);
|
|
351
|
+
}
|
|
352
|
+
else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
353
|
+
removedFiles.push(full);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
await collect(documentsRoot);
|
|
358
|
+
for (const file of removedFiles) {
|
|
359
|
+
try {
|
|
360
|
+
await rm(file, { force: true });
|
|
361
|
+
}
|
|
362
|
+
catch {
|
|
363
|
+
// Already absent.
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
await atomicWrite(paths.qmdManifest, JSON.stringify({ version: QMD_MANIFEST_VERSION, vaultId, entries: {} }, null, 2));
|
|
367
|
+
return {
|
|
368
|
+
manifest: { version: QMD_MANIFEST_VERSION, vaultId, entries: {} },
|
|
369
|
+
manifestHash: hashQmdManifest({ version: QMD_MANIFEST_VERSION, vaultId, entries: {} }),
|
|
370
|
+
counts: { indexed: 0, updated: 0, unchanged: 0, removed: removedFiles.length },
|
|
371
|
+
diagnostics: [
|
|
372
|
+
...diagnostics,
|
|
373
|
+
{
|
|
374
|
+
severity: "warning",
|
|
375
|
+
code: "qmd_manifest_invalid",
|
|
376
|
+
path: paths.qmdManifest,
|
|
377
|
+
message: "Corrupt QMD manifest during invalidation; removed generated mirror entries fail-closed",
|
|
378
|
+
},
|
|
379
|
+
],
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
// The set of still-valid mirror entries (present and parser-valid now).
|
|
383
|
+
const validHashes = new Map();
|
|
384
|
+
for (const doc of discovery.documents) {
|
|
385
|
+
const role = roleForDocumentType(doc.frontmatter.type);
|
|
386
|
+
const key = manifestKey(role, doc.id);
|
|
387
|
+
const serialized = serializeKnowledgeDocument(doc);
|
|
388
|
+
validHashes.set(key, hashQmdContent(serialized));
|
|
389
|
+
}
|
|
390
|
+
const removed = {};
|
|
391
|
+
const kept = {};
|
|
392
|
+
for (const [key, entry] of Object.entries(prior.entries)) {
|
|
393
|
+
if (!validHashes.has(key)) {
|
|
394
|
+
removed[key] = entry;
|
|
395
|
+
}
|
|
396
|
+
else {
|
|
397
|
+
kept[key] = entry;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
if (Object.keys(removed).length > 0) {
|
|
401
|
+
await atomicWrite(paths.qmdManifest, JSON.stringify({ version: QMD_MANIFEST_VERSION, vaultId, entries: kept }, null, 2));
|
|
402
|
+
for (const key of Object.keys(removed)) {
|
|
403
|
+
try {
|
|
404
|
+
await rm(manifestKeyToPath(paths, key), { force: true });
|
|
405
|
+
}
|
|
406
|
+
catch {
|
|
407
|
+
// Already absent.
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
const finalManifest = { version: QMD_MANIFEST_VERSION, vaultId, entries: kept };
|
|
412
|
+
return {
|
|
413
|
+
manifest: finalManifest,
|
|
414
|
+
manifestHash: hashQmdManifest(finalManifest),
|
|
415
|
+
counts: { indexed: 0, updated: 0, unchanged: 0, removed: Object.keys(removed).length },
|
|
416
|
+
diagnostics,
|
|
417
|
+
};
|
|
418
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { createStore } from "@tobilu/qmd";
|
|
3
|
+
/**
|
|
4
|
+
* Package-private normalized adapter over the pinned @tobilu/qmd SDK.
|
|
5
|
+
*
|
|
6
|
+
* This is the ONLY production module allowed to import @tobilu/qmd. It hides
|
|
7
|
+
* SDK-specific types, collection config, model identity, and close behavior so
|
|
8
|
+
* the rest of the extension never touches QMD internals or tables directly.
|
|
9
|
+
*/
|
|
10
|
+
export const QMD_PACKAGE_VERSION = "2.5.3";
|
|
11
|
+
export const QMD_DEFAULT_MODELS = {
|
|
12
|
+
embed: "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf",
|
|
13
|
+
generate: "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf",
|
|
14
|
+
rerank: "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf",
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Mirror collection a hit came from: the SDK reports virtual paths
|
|
18
|
+
* "qmd://<collection>/<path>.md", and the mirror layout is
|
|
19
|
+
* ".../documents/<role>/<pageId>.md". Either form resolves the role.
|
|
20
|
+
*/
|
|
21
|
+
function roleFromFile(file) {
|
|
22
|
+
return /(?:^|\/)documents\/canonical\/|^qmd:\/\/canonical\//.test(file)
|
|
23
|
+
? "canonical"
|
|
24
|
+
: "evidence";
|
|
25
|
+
}
|
|
26
|
+
/** Normalize raw within-store scores to 0..1 against this list's max. */
|
|
27
|
+
function withNormalizedScores(hits) {
|
|
28
|
+
const max = Math.max(...hits.map((h) => h.raw), 1e-9);
|
|
29
|
+
return hits.map(({ raw, ...rest }) => ({ ...rest, score: raw / max }));
|
|
30
|
+
}
|
|
31
|
+
const mapLexHit = (r) => ({
|
|
32
|
+
collection: roleFromFile(r.filepath),
|
|
33
|
+
file: r.filepath,
|
|
34
|
+
title: r.title,
|
|
35
|
+
source: r.source,
|
|
36
|
+
raw: r.score,
|
|
37
|
+
});
|
|
38
|
+
const mapHybridHit = (r) => ({
|
|
39
|
+
collection: roleFromFile(r.file),
|
|
40
|
+
file: r.file,
|
|
41
|
+
title: r.title,
|
|
42
|
+
source: "fts",
|
|
43
|
+
body: r.bestChunk,
|
|
44
|
+
raw: r.score,
|
|
45
|
+
});
|
|
46
|
+
/**
|
|
47
|
+
* Open a QMD index store over the mirror documents directory, with two
|
|
48
|
+
* non-overlapping collections (canonical and evidence).
|
|
49
|
+
*/
|
|
50
|
+
export async function openQmdIndexStore(input) {
|
|
51
|
+
const store = await createStore({
|
|
52
|
+
dbPath: input.dbPath,
|
|
53
|
+
config: {
|
|
54
|
+
global_context: "Validated LLM Wiki knowledge",
|
|
55
|
+
collections: {
|
|
56
|
+
canonical: {
|
|
57
|
+
path: join(input.documentsPath, "canonical"),
|
|
58
|
+
pattern: "**/*.md",
|
|
59
|
+
context: { "/": "Reusable conclusions, entities, requirements, and procedures" },
|
|
60
|
+
},
|
|
61
|
+
evidence: {
|
|
62
|
+
path: join(input.documentsPath, "evidence"),
|
|
63
|
+
pattern: "**/*.md",
|
|
64
|
+
context: { "/": "Source evidence, observations, trajectories, and unpromoted notes" },
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
return {
|
|
70
|
+
update: (onProgress) => store.update({ onProgress }),
|
|
71
|
+
embed: ({ force, onProgress }) => store.embed({ force, chunkStrategy: "regex", onProgress }),
|
|
72
|
+
status: async () => {
|
|
73
|
+
const status = await store.getStatus();
|
|
74
|
+
const counts = Object.fromEntries(status.collections.map((collection) => [collection.name, collection.documents]));
|
|
75
|
+
return {
|
|
76
|
+
totalDocuments: status.totalDocuments,
|
|
77
|
+
needsEmbedding: status.needsEmbedding,
|
|
78
|
+
hasVectorIndex: status.hasVectorIndex,
|
|
79
|
+
canonicalDocuments: counts.canonical ?? 0,
|
|
80
|
+
evidenceDocuments: counts.evidence ?? 0,
|
|
81
|
+
};
|
|
82
|
+
},
|
|
83
|
+
close: () => store.close(),
|
|
84
|
+
searchLex: async (query, limit = 40) => withNormalizedScores((await store.searchLex(query, { limit })).map(mapLexHit)),
|
|
85
|
+
searchTyped: async (query, limit = 10) => withNormalizedScores((await store.search({
|
|
86
|
+
queries: [
|
|
87
|
+
{ type: "lex", query },
|
|
88
|
+
{ type: "vec", query },
|
|
89
|
+
],
|
|
90
|
+
rerank: false,
|
|
91
|
+
candidateLimit: 40,
|
|
92
|
+
limit,
|
|
93
|
+
explain: true,
|
|
94
|
+
})).map(mapHybridHit)),
|
|
95
|
+
searchExpanded: async (query, intent, limit = 10) => withNormalizedScores((await store.search({
|
|
96
|
+
query,
|
|
97
|
+
intent,
|
|
98
|
+
rerank: true,
|
|
99
|
+
candidateLimit: 40,
|
|
100
|
+
limit,
|
|
101
|
+
explain: true,
|
|
102
|
+
})).map(mapHybridHit)),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
/** Resolve model identities from env, defaulting to pinned models. No downloads. */
|
|
106
|
+
export function resolveQmdModels(env = process.env) {
|
|
107
|
+
return {
|
|
108
|
+
embed: env.QMD_EMBED_MODEL?.trim() || QMD_DEFAULT_MODELS.embed,
|
|
109
|
+
generate: env.QMD_GENERATE_MODEL?.trim() || QMD_DEFAULT_MODELS.generate,
|
|
110
|
+
rerank: env.QMD_RERANK_MODEL?.trim() || QMD_DEFAULT_MODELS.rerank,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
@@ -78,11 +78,14 @@ function queryTerms(query) {
|
|
|
78
78
|
if (compact && compact !== normalized)
|
|
79
79
|
terms.push(compact);
|
|
80
80
|
for (const part of normalized.split(/\s+/)) {
|
|
81
|
-
if (part.length >= 2)
|
|
81
|
+
if (part.length >= 2 && !STOPWORDS.has(part))
|
|
82
82
|
terms.push(part);
|
|
83
83
|
}
|
|
84
84
|
const latinRuns = normalized.match(/[a-z0-9]{2,}/g) ?? [];
|
|
85
|
-
|
|
85
|
+
for (const run of latinRuns) {
|
|
86
|
+
if (!STOPWORDS.has(run))
|
|
87
|
+
terms.push(run);
|
|
88
|
+
}
|
|
86
89
|
const cjkRuns = normalized.match(/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+/gu) ?? [];
|
|
87
90
|
for (const run of cjkRuns) {
|
|
88
91
|
for (let size = 2; size <= 3; size++) {
|
|
@@ -95,10 +98,36 @@ function queryTerms(query) {
|
|
|
95
98
|
}
|
|
96
99
|
return unique(terms).slice(0, 30);
|
|
97
100
|
}
|
|
101
|
+
/** Matches any CJK (Han / Hiragana / Katakana) character. */
|
|
102
|
+
const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
|
|
103
|
+
/**
|
|
104
|
+
* Whether `haystack` contains `term`.
|
|
105
|
+
*
|
|
106
|
+
* Root-cause fix for issue #223. Previously this was a raw substring
|
|
107
|
+
* containment, so a 2-char query token like "to" or "so" matched thousands of
|
|
108
|
+
* unrelated words ("to" ⊂ "history"/"story"; "so" ⊂ "person"/"wilson"), and
|
|
109
|
+
* weighted field stacking pushed that junk past the auto-injection gate.
|
|
110
|
+
*
|
|
111
|
+
* Now:
|
|
112
|
+
* - CJK terms (Han/Kana) have no whitespace word boundaries, so they still
|
|
113
|
+
* match by compact substring — a CJK bigram matches inside a longer glued
|
|
114
|
+
* CJK run, preserving the multilingual recall the vaults depend on.
|
|
115
|
+
* - Latin/ASCII terms require a WHOLE-WORD match: a spaceless haystack is a
|
|
116
|
+
* single word (term must equal it), and a spaced haystack must contain the
|
|
117
|
+
* term as a whole whitespace-delimited token. "to" still matches the word
|
|
118
|
+
* "to" and "Go" still matches "Go", but no longer "history"/"story"/"person".
|
|
119
|
+
*/
|
|
98
120
|
function includesTerm(haystack, term) {
|
|
99
121
|
if (!haystack || !term)
|
|
100
122
|
return false;
|
|
101
|
-
|
|
123
|
+
const termCompact = compactText(term);
|
|
124
|
+
if (CJK_RE.test(termCompact)) {
|
|
125
|
+
return compactText(haystack).includes(termCompact);
|
|
126
|
+
}
|
|
127
|
+
if (!/\s/.test(haystack)) {
|
|
128
|
+
return compactText(haystack) === termCompact;
|
|
129
|
+
}
|
|
130
|
+
return haystack.split(/\s+/).some((t) => t === term || t === termCompact);
|
|
102
131
|
}
|
|
103
132
|
function scoreField(value, terms, weight) {
|
|
104
133
|
const text = normalizeText(value);
|
|
@@ -164,6 +193,51 @@ const STOPWORDS = new Set([
|
|
|
164
193
|
"type",
|
|
165
194
|
"used",
|
|
166
195
|
"using",
|
|
196
|
+
// Common English function words (issue #223). These must not become scoring
|
|
197
|
+
// terms, or a natural-language prompt full of "to"/"so"/"the"/"did" pushes
|
|
198
|
+
// unrelated pages past the auto-injection gate. 2-letter acronyms that are
|
|
199
|
+
// NOT stopwords ("go", "pi", "sso") still match, as intended.
|
|
200
|
+
"to",
|
|
201
|
+
"so",
|
|
202
|
+
"did",
|
|
203
|
+
"do",
|
|
204
|
+
"does",
|
|
205
|
+
"and",
|
|
206
|
+
"or",
|
|
207
|
+
"but",
|
|
208
|
+
"a",
|
|
209
|
+
"an",
|
|
210
|
+
"i",
|
|
211
|
+
"it",
|
|
212
|
+
"its",
|
|
213
|
+
"is",
|
|
214
|
+
"are",
|
|
215
|
+
"was",
|
|
216
|
+
"be",
|
|
217
|
+
"am",
|
|
218
|
+
"for",
|
|
219
|
+
"on",
|
|
220
|
+
"at",
|
|
221
|
+
"by",
|
|
222
|
+
"as",
|
|
223
|
+
"if",
|
|
224
|
+
"in",
|
|
225
|
+
"of",
|
|
226
|
+
"up",
|
|
227
|
+
"out",
|
|
228
|
+
"not",
|
|
229
|
+
"no",
|
|
230
|
+
"can",
|
|
231
|
+
"has",
|
|
232
|
+
"had",
|
|
233
|
+
"me",
|
|
234
|
+
"my",
|
|
235
|
+
"you",
|
|
236
|
+
"your",
|
|
237
|
+
"we",
|
|
238
|
+
"our",
|
|
239
|
+
"us",
|
|
240
|
+
"too",
|
|
167
241
|
]);
|
|
168
242
|
/**
|
|
169
243
|
* Split a page's body into chunks by headings.
|
|
@@ -68,7 +68,31 @@ export class Runtime {
|
|
|
68
68
|
const provider = model.provider ?? "unknown";
|
|
69
69
|
return { ok: false, reason: `no API key for provider "${provider}"` };
|
|
70
70
|
}
|
|
71
|
-
|
|
71
|
+
// Extension-registered providers (issue #222): pi-ai's default stream path
|
|
72
|
+
// may not be able to resolve their api (e.g. claude-bridge on pi 0.85+), so
|
|
73
|
+
// surface the provider's own streamSimple for the sub-agent stream. On pi
|
|
74
|
+
// < 0.85 the registry method is absent (?.) and the default path — which
|
|
75
|
+
// already knows extension streamSimples — is used instead.
|
|
76
|
+
const modelProvider = model.provider;
|
|
77
|
+
const registered = modelProvider
|
|
78
|
+
? ctx.modelRegistry.getRegisteredProviderConfig?.(modelProvider)
|
|
79
|
+
: undefined;
|
|
80
|
+
const streamFn = registered?.streamSimple && registered.api === model.api
|
|
81
|
+
? registered.streamSimple
|
|
82
|
+
: undefined;
|
|
83
|
+
// Auth can redirect the endpoint (e.g. GitHub Copilot business vs
|
|
84
|
+
// individual accounts) and/or carry provider-scoped env values (pi >=
|
|
85
|
+
// 0.85). Streaming against the catalogue values yields 421 Misdirected
|
|
86
|
+
// Request and the synthesis silently produces nothing (issue #222).
|
|
87
|
+
const authedModel = auth.baseUrl ? { ...model, baseUrl: auth.baseUrl } : model;
|
|
88
|
+
return {
|
|
89
|
+
ok: true,
|
|
90
|
+
model: authedModel,
|
|
91
|
+
apiKey: auth.apiKey ?? "",
|
|
92
|
+
headers: auth.headers,
|
|
93
|
+
env: auth.env,
|
|
94
|
+
streamFn,
|
|
95
|
+
};
|
|
72
96
|
}
|
|
73
97
|
/**
|
|
74
98
|
* Fire-and-forget a background task.
|