@tangle-network/agent-knowledge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -0
- package/dist/chunk-TXNYP4WI.js +147 -0
- package/dist/chunk-TXNYP4WI.js.map +1 -0
- package/dist/chunk-XDXIBHPS.js +703 -0
- package/dist/chunk-XDXIBHPS.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +245 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +153 -0
- package/dist/index.js +219 -0
- package/dist/index.js.map +1 -0
- package/dist/types-C1FBNRIN.d.ts +135 -0
- package/dist/viz/index.d.ts +37 -0
- package/dist/viz/index.js +11 -0
- package/dist/viz/index.js.map +1 -0
- package/docs/architecture.md +52 -0
- package/package.json +67 -0
|
@@ -0,0 +1,703 @@
|
|
|
1
|
+
// src/ids.ts
|
|
2
|
+
import { createHash } from "crypto";
|
|
3
|
+
function sha256(text) {
|
|
4
|
+
return createHash("sha256").update(text).digest("hex");
|
|
5
|
+
}
|
|
6
|
+
function slugify(input) {
|
|
7
|
+
const out = input.trim().toLowerCase().normalize("NFKD").replace(/[\u0300-\u036f]/g, "").replace(/[^a-z0-9\u4e00-\u9fff]+/g, "-").replace(/^-+|-+$/g, "");
|
|
8
|
+
return out || "untitled";
|
|
9
|
+
}
|
|
10
|
+
function stableId(prefix, content) {
|
|
11
|
+
return `${prefix}_${sha256(content).slice(0, 16)}`;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// src/frontmatter.ts
|
|
15
|
+
function parseFrontmatter(content) {
|
|
16
|
+
const normalized = content.replace(/\r\n/g, "\n");
|
|
17
|
+
if (!normalized.startsWith("---\n")) return { frontmatter: {}, body: normalized };
|
|
18
|
+
const end = normalized.indexOf("\n---", 4);
|
|
19
|
+
if (end < 0) return { frontmatter: {}, body: normalized };
|
|
20
|
+
const raw = normalized.slice(4, end);
|
|
21
|
+
const after = normalized.slice(end).replace(/^\n---\s*\n?/, "");
|
|
22
|
+
return { frontmatter: parseSimpleYaml(raw), body: after };
|
|
23
|
+
}
|
|
24
|
+
function formatFrontmatter(frontmatter, body) {
|
|
25
|
+
const lines = Object.entries(frontmatter).filter(([, value]) => value !== void 0).flatMap(([key, value]) => formatYamlField(key, value));
|
|
26
|
+
if (lines.length === 0) return body;
|
|
27
|
+
return `---
|
|
28
|
+
${lines.join("\n")}
|
|
29
|
+
---
|
|
30
|
+
${body.replace(/^\n+/, "")}`;
|
|
31
|
+
}
|
|
32
|
+
function parseSimpleYaml(raw) {
|
|
33
|
+
const out = {};
|
|
34
|
+
const lines = raw.split("\n");
|
|
35
|
+
for (let i = 0; i < lines.length; i++) {
|
|
36
|
+
const line = lines[i];
|
|
37
|
+
const scalar = /^([A-Za-z0-9_-]+):\s*(.*)$/.exec(line);
|
|
38
|
+
if (!scalar) continue;
|
|
39
|
+
const key = scalar[1];
|
|
40
|
+
const rest = scalar[2].trim();
|
|
41
|
+
if (rest === "") {
|
|
42
|
+
const items = [];
|
|
43
|
+
while (i + 1 < lines.length) {
|
|
44
|
+
const item = /^\s*-\s*(.+?)\s*$/.exec(lines[i + 1]);
|
|
45
|
+
if (!item) break;
|
|
46
|
+
items.push(unquote(item[1]));
|
|
47
|
+
i++;
|
|
48
|
+
}
|
|
49
|
+
out[key] = items;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (rest.startsWith("[") && rest.endsWith("]")) {
|
|
53
|
+
out[key] = rest.slice(1, -1).split(",").map((part) => unquote(part.trim())).filter(Boolean);
|
|
54
|
+
} else if (rest === "true" || rest === "false") {
|
|
55
|
+
out[key] = rest === "true";
|
|
56
|
+
} else if (/^-?\d+(?:\.\d+)?$/.test(rest)) {
|
|
57
|
+
out[key] = Number(rest);
|
|
58
|
+
} else {
|
|
59
|
+
out[key] = unquote(rest);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
function formatYamlField(key, value) {
|
|
65
|
+
if (Array.isArray(value)) {
|
|
66
|
+
return [key + ":", ...value.map((item) => ` - ${String(item)}`)];
|
|
67
|
+
}
|
|
68
|
+
if (typeof value === "string") return [`${key}: ${value}`];
|
|
69
|
+
if (typeof value === "number" || typeof value === "boolean") return [`${key}: ${String(value)}`];
|
|
70
|
+
return [`${key}: ${JSON.stringify(value)}`];
|
|
71
|
+
}
|
|
72
|
+
function unquote(value) {
|
|
73
|
+
return value.replace(/^['"]|['"]$/g, "");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// src/wikilinks.ts
|
|
77
|
+
var WIKILINK_REGEX = /\[\[([^\]|]+?)(?:\|[^\]]+?)?\]\]/g;
|
|
78
|
+
function extractWikilinks(content) {
|
|
79
|
+
const links = [];
|
|
80
|
+
const regex = new RegExp(WIKILINK_REGEX.source, "g");
|
|
81
|
+
let match;
|
|
82
|
+
while ((match = regex.exec(content)) !== null) {
|
|
83
|
+
links.push(match[1].trim());
|
|
84
|
+
}
|
|
85
|
+
return [...new Set(links)];
|
|
86
|
+
}
|
|
87
|
+
function normalizeLinkTarget(target) {
|
|
88
|
+
return target.trim().replace(/\.md$/i, "").toLowerCase().replace(/\s+/g, "-");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// src/write-protocol.ts
|
|
92
|
+
var OPENER_LINE = /^---\s*FILE:\s*(.+?)\s*---\s*$/i;
|
|
93
|
+
var CLOSER_LINE = /^---\s*END\s+FILE\s*---\s*$/i;
|
|
94
|
+
var FENCE_LINE = /^\s{0,3}(```+|~~~+)/;
|
|
95
|
+
function isSafeKnowledgePath(path, allowedPrefixes = ["knowledge/"]) {
|
|
96
|
+
if (typeof path !== "string" || path.trim() === "") return false;
|
|
97
|
+
if (/[\x00-\x1f]/.test(path)) return false;
|
|
98
|
+
if (path.startsWith("/") || path.startsWith("\\")) return false;
|
|
99
|
+
if (/^[a-zA-Z]:/.test(path)) return false;
|
|
100
|
+
const normalized = path.replace(/\\/g, "/");
|
|
101
|
+
if (normalized.split("/").some((part) => part === "..")) return false;
|
|
102
|
+
return allowedPrefixes.some((prefix) => normalized.startsWith(prefix));
|
|
103
|
+
}
|
|
104
|
+
function parseKnowledgeWriteBlocks(text, allowedPrefixes = ["knowledge/"]) {
|
|
105
|
+
const lines = text.replace(/\r\n/g, "\n").split("\n");
|
|
106
|
+
const blocks = [];
|
|
107
|
+
const warnings = [];
|
|
108
|
+
let i = 0;
|
|
109
|
+
while (i < lines.length) {
|
|
110
|
+
const opener = OPENER_LINE.exec(lines[i]);
|
|
111
|
+
if (!opener) {
|
|
112
|
+
i++;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
const path = opener[1].trim();
|
|
116
|
+
i++;
|
|
117
|
+
const contentLines = [];
|
|
118
|
+
let fenceMarker = null;
|
|
119
|
+
let fenceLen = 0;
|
|
120
|
+
let closed = false;
|
|
121
|
+
while (i < lines.length) {
|
|
122
|
+
const line = lines[i];
|
|
123
|
+
const fence = FENCE_LINE.exec(line);
|
|
124
|
+
if (fence) {
|
|
125
|
+
const run = fence[1];
|
|
126
|
+
const char = run[0];
|
|
127
|
+
if (fenceMarker === null) {
|
|
128
|
+
fenceMarker = char;
|
|
129
|
+
fenceLen = run.length;
|
|
130
|
+
} else if (char === fenceMarker && run.length >= fenceLen) {
|
|
131
|
+
fenceMarker = null;
|
|
132
|
+
fenceLen = 0;
|
|
133
|
+
}
|
|
134
|
+
contentLines.push(line);
|
|
135
|
+
i++;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (fenceMarker === null && CLOSER_LINE.test(line)) {
|
|
139
|
+
closed = true;
|
|
140
|
+
i++;
|
|
141
|
+
break;
|
|
142
|
+
}
|
|
143
|
+
contentLines.push(line);
|
|
144
|
+
i++;
|
|
145
|
+
}
|
|
146
|
+
if (!closed) {
|
|
147
|
+
warnings.push(`FILE block "${path || "(empty)"}" was not closed before end of stream.`);
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
if (!isSafeKnowledgePath(path, allowedPrefixes)) {
|
|
151
|
+
warnings.push(`FILE block with unsafe path "${path}" rejected.`);
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
blocks.push({ path, content: contentLines.join("\n") });
|
|
155
|
+
}
|
|
156
|
+
return { blocks, warnings };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// src/adapters.ts
|
|
160
|
+
var textSourceAdapter = {
|
|
161
|
+
id: "text",
|
|
162
|
+
canLoad: (input) => Boolean(input.text) || /\.(md|txt|json|csv)$/i.test(input.uri),
|
|
163
|
+
load: (input) => ({
|
|
164
|
+
title: input.uri.split("/").pop(),
|
|
165
|
+
mediaType: mediaTypeFor(input.uri),
|
|
166
|
+
text: decodeText(input),
|
|
167
|
+
anchors: anchorsForText(input.uri, decodeText(input)),
|
|
168
|
+
metadata: input.metadata
|
|
169
|
+
})
|
|
170
|
+
};
|
|
171
|
+
function mediaTypeFor(uri) {
|
|
172
|
+
const lower = uri.toLowerCase();
|
|
173
|
+
if (lower.endsWith(".md")) return "text/markdown";
|
|
174
|
+
if (lower.endsWith(".txt")) return "text/plain";
|
|
175
|
+
if (lower.endsWith(".json")) return "application/json";
|
|
176
|
+
if (lower.endsWith(".csv")) return "text/csv";
|
|
177
|
+
if (lower.endsWith(".pdf")) return "application/pdf";
|
|
178
|
+
return "application/octet-stream";
|
|
179
|
+
}
|
|
180
|
+
function decodeText(input) {
|
|
181
|
+
return input.text ?? (input.bytes ? new TextDecoder().decode(input.bytes).slice(0, 2e5) : void 0);
|
|
182
|
+
}
|
|
183
|
+
function anchorsForText(uri, text) {
|
|
184
|
+
if (!text) return [];
|
|
185
|
+
const lines = text.split("\n");
|
|
186
|
+
const anchors = [{ id: "all", sourceId: "", label: "Full source", lineStart: 1, lineEnd: lines.length }];
|
|
187
|
+
for (let i = 0; i < lines.length; i += 50) {
|
|
188
|
+
anchors.push({
|
|
189
|
+
id: `l${i + 1}`,
|
|
190
|
+
sourceId: "",
|
|
191
|
+
label: `${uri}:${i + 1}`,
|
|
192
|
+
lineStart: i + 1,
|
|
193
|
+
lineEnd: Math.min(lines.length, i + 50)
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
return anchors;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// src/proposals.ts
|
|
200
|
+
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
201
|
+
import { dirname, join } from "path";
|
|
202
|
+
async function applyKnowledgeWriteBlocks(root, proposalText) {
|
|
203
|
+
const parsed = parseKnowledgeWriteBlocks(proposalText);
|
|
204
|
+
const written = [];
|
|
205
|
+
for (const block of parsed.blocks) {
|
|
206
|
+
const path = join(root, block.path);
|
|
207
|
+
await mkdir(dirname(path), { recursive: true });
|
|
208
|
+
await writeFile(path, block.content.endsWith("\n") ? block.content : `${block.content}
|
|
209
|
+
`, "utf8");
|
|
210
|
+
written.push(block.path);
|
|
211
|
+
}
|
|
212
|
+
return { written, warnings: parsed.warnings };
|
|
213
|
+
}
|
|
214
|
+
async function applyKnowledgeWriteBlocksFile(root, proposalPath) {
|
|
215
|
+
return applyKnowledgeWriteBlocks(root, await readFile(proposalPath, "utf8"));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// src/store.ts
|
|
219
|
+
import { mkdir as mkdir2, readFile as readFile2, readdir, stat, writeFile as writeFile2 } from "fs/promises";
|
|
220
|
+
import { dirname as dirname2, join as join2, relative } from "path";
|
|
221
|
+
function layoutFor(root) {
|
|
222
|
+
return {
|
|
223
|
+
root,
|
|
224
|
+
knowledgeDir: join2(root, "knowledge"),
|
|
225
|
+
rawSourcesDir: join2(root, "raw", "sources"),
|
|
226
|
+
sourceRegistryPath: join2(root, ".agent-knowledge", "sources.json"),
|
|
227
|
+
indexPath: join2(root, "knowledge", "index.md"),
|
|
228
|
+
logPath: join2(root, "knowledge", "log.md"),
|
|
229
|
+
cacheDir: join2(root, ".agent-knowledge")
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
async function initKnowledgeBase(root) {
|
|
233
|
+
const layout = layoutFor(root);
|
|
234
|
+
await mkdir2(layout.knowledgeDir, { recursive: true });
|
|
235
|
+
await mkdir2(layout.rawSourcesDir, { recursive: true });
|
|
236
|
+
await mkdir2(layout.cacheDir, { recursive: true });
|
|
237
|
+
await writeIfMissing(layout.indexPath, "# Knowledge Index\n\n");
|
|
238
|
+
await writeIfMissing(layout.logPath, "# Knowledge Log\n\n");
|
|
239
|
+
await writeIfMissing(layout.sourceRegistryPath, '{\n "generatedAt": "1970-01-01T00:00:00.000Z",\n "sources": []\n}\n');
|
|
240
|
+
return layout;
|
|
241
|
+
}
|
|
242
|
+
async function loadKnowledgePages(root) {
|
|
243
|
+
const layout = layoutFor(root);
|
|
244
|
+
const files = await listMarkdownFiles(layout.knowledgeDir);
|
|
245
|
+
const pages = [];
|
|
246
|
+
for (const file of files) {
|
|
247
|
+
const content = await readFile2(file, "utf8");
|
|
248
|
+
const { frontmatter, body } = parseFrontmatter(content);
|
|
249
|
+
const rel = relative(root, file).replace(/\\/g, "/");
|
|
250
|
+
const title = stringField(frontmatter.title) ?? firstHeading(body) ?? rel.split("/").pop().replace(/\.md$/, "");
|
|
251
|
+
const sourceIds = arrayField(frontmatter.sources);
|
|
252
|
+
const tags = arrayField(frontmatter.tags);
|
|
253
|
+
pages.push({
|
|
254
|
+
id: stringField(frontmatter.id) ?? slugify(rel.replace(/^knowledge\//, "").replace(/\.md$/, "")),
|
|
255
|
+
path: rel,
|
|
256
|
+
title,
|
|
257
|
+
text: body,
|
|
258
|
+
frontmatter,
|
|
259
|
+
sourceIds,
|
|
260
|
+
tags,
|
|
261
|
+
outLinks: extractWikilinks(body).map(normalizeLinkTarget)
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
pages.sort((a, b) => a.path.localeCompare(b.path));
|
|
265
|
+
return pages;
|
|
266
|
+
}
|
|
267
|
+
async function writeJson(path, value) {
|
|
268
|
+
await mkdir2(dirname2(path), { recursive: true });
|
|
269
|
+
await writeFile2(path, JSON.stringify(value, null, 2) + "\n", "utf8");
|
|
270
|
+
}
|
|
271
|
+
async function writeIfMissing(path, content) {
|
|
272
|
+
try {
|
|
273
|
+
await stat(path);
|
|
274
|
+
} catch {
|
|
275
|
+
await mkdir2(dirname2(path), { recursive: true });
|
|
276
|
+
await writeFile2(path, content, "utf8");
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
async function listMarkdownFiles(root) {
|
|
280
|
+
try {
|
|
281
|
+
const entries = await readdir(root, { withFileTypes: true });
|
|
282
|
+
const out = [];
|
|
283
|
+
for (const entry of entries) {
|
|
284
|
+
const full = join2(root, entry.name);
|
|
285
|
+
if (entry.isDirectory()) out.push(...await listMarkdownFiles(full));
|
|
286
|
+
else if (entry.isFile() && entry.name.endsWith(".md")) out.push(full);
|
|
287
|
+
}
|
|
288
|
+
return out;
|
|
289
|
+
} catch {
|
|
290
|
+
return [];
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
function stringField(value) {
|
|
294
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
295
|
+
}
|
|
296
|
+
function arrayField(value) {
|
|
297
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
298
|
+
}
|
|
299
|
+
function firstHeading(body) {
|
|
300
|
+
return /^#\s+(.+)$/m.exec(body)?.[1]?.trim();
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// src/sources.ts
|
|
304
|
+
import { copyFile, mkdir as mkdir3, readFile as readFile3, readdir as readdir2, stat as stat2, writeFile as writeFile3 } from "fs/promises";
|
|
305
|
+
import { basename, dirname as dirname3, join as join3, relative as relative2 } from "path";
|
|
306
|
+
async function loadSourceRegistry(root) {
|
|
307
|
+
const path = sourceRegistryPath(root);
|
|
308
|
+
try {
|
|
309
|
+
const parsed = JSON.parse(await readFile3(path, "utf8"));
|
|
310
|
+
return {
|
|
311
|
+
generatedAt: typeof parsed.generatedAt === "string" ? parsed.generatedAt : (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
312
|
+
sources: Array.isArray(parsed.sources) ? parsed.sources : []
|
|
313
|
+
};
|
|
314
|
+
} catch {
|
|
315
|
+
return { generatedAt: (/* @__PURE__ */ new Date(0)).toISOString(), sources: [] };
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
async function writeSourceRegistry(root, registry) {
|
|
319
|
+
const path = sourceRegistryPath(root);
|
|
320
|
+
await mkdir3(dirname3(path), { recursive: true });
|
|
321
|
+
await writeFile3(path, JSON.stringify(registry, null, 2) + "\n", "utf8");
|
|
322
|
+
}
|
|
323
|
+
async function addSourcePath(root, sourcePath, options = {}) {
|
|
324
|
+
const s = await stat2(sourcePath);
|
|
325
|
+
if (s.isDirectory()) {
|
|
326
|
+
const out = [];
|
|
327
|
+
for (const file of await listFiles(sourcePath)) {
|
|
328
|
+
out.push(...await addSourcePath(root, file, options));
|
|
329
|
+
}
|
|
330
|
+
return out;
|
|
331
|
+
}
|
|
332
|
+
const layout = layoutFor(root);
|
|
333
|
+
await mkdir3(layout.rawSourcesDir, { recursive: true });
|
|
334
|
+
const bytes = await readFile3(sourcePath);
|
|
335
|
+
const contentHash = sha256(bytes.toString("base64"));
|
|
336
|
+
const fileName = basename(sourcePath);
|
|
337
|
+
const adapters = options.adapters ?? [textSourceAdapter];
|
|
338
|
+
const adapter = adapters.find((candidate) => candidate.canLoad({ uri: sourcePath, bytes }));
|
|
339
|
+
const loaded = adapter ? await adapter.load({ uri: sourcePath, bytes }) : {};
|
|
340
|
+
const id = stableId("src", `${contentHash}:${fileName}`);
|
|
341
|
+
const targetRel = join3("raw", "sources", `${slugify(fileName.replace(/\.[^.]+$/, ""))}-${contentHash.slice(0, 8)}${ext(fileName)}`).replace(/\\/g, "/");
|
|
342
|
+
const targetAbs = join3(root, targetRel);
|
|
343
|
+
if (options.copyIntoRaw ?? true) {
|
|
344
|
+
await mkdir3(dirname3(targetAbs), { recursive: true });
|
|
345
|
+
await copyFile(sourcePath, targetAbs);
|
|
346
|
+
}
|
|
347
|
+
const existing = await loadSourceRegistry(root);
|
|
348
|
+
const record = {
|
|
349
|
+
id,
|
|
350
|
+
uri: targetRel,
|
|
351
|
+
title: loaded.title ?? fileName,
|
|
352
|
+
mediaType: loaded.mediaType ?? mediaTypeFor2(fileName),
|
|
353
|
+
contentHash,
|
|
354
|
+
text: loaded.text ?? textPreview(fileName, bytes),
|
|
355
|
+
anchors: (loaded.anchors ?? []).map((anchor) => ({ ...anchor, sourceId: id })),
|
|
356
|
+
createdAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
|
|
357
|
+
metadata: {
|
|
358
|
+
...loaded.metadata ?? {},
|
|
359
|
+
originalPath: sourcePath,
|
|
360
|
+
sizeBytes: bytes.length,
|
|
361
|
+
projectRelativePath: relative2(root, sourcePath).replace(/\\/g, "/")
|
|
362
|
+
}
|
|
363
|
+
};
|
|
364
|
+
const next = {
|
|
365
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
366
|
+
sources: [record, ...existing.sources.filter((source) => source.id !== id)]
|
|
367
|
+
};
|
|
368
|
+
await writeSourceRegistry(root, next);
|
|
369
|
+
return [record];
|
|
370
|
+
}
|
|
371
|
+
function sourceRegistryPath(root) {
|
|
372
|
+
return join3(layoutFor(root).cacheDir, "sources.json");
|
|
373
|
+
}
|
|
374
|
+
async function listFiles(root) {
|
|
375
|
+
const entries = await readdir2(root, { withFileTypes: true });
|
|
376
|
+
const out = [];
|
|
377
|
+
for (const entry of entries) {
|
|
378
|
+
const full = join3(root, entry.name);
|
|
379
|
+
if (entry.isDirectory()) out.push(...await listFiles(full));
|
|
380
|
+
else if (entry.isFile()) out.push(full);
|
|
381
|
+
}
|
|
382
|
+
return out;
|
|
383
|
+
}
|
|
384
|
+
function ext(fileName) {
|
|
385
|
+
const idx = fileName.lastIndexOf(".");
|
|
386
|
+
return idx >= 0 ? fileName.slice(idx) : "";
|
|
387
|
+
}
|
|
388
|
+
function mediaTypeFor2(fileName) {
|
|
389
|
+
const lower = fileName.toLowerCase();
|
|
390
|
+
if (lower.endsWith(".md")) return "text/markdown";
|
|
391
|
+
if (lower.endsWith(".txt")) return "text/plain";
|
|
392
|
+
if (lower.endsWith(".json")) return "application/json";
|
|
393
|
+
if (lower.endsWith(".csv")) return "text/csv";
|
|
394
|
+
if (lower.endsWith(".pdf")) return "application/pdf";
|
|
395
|
+
return "application/octet-stream";
|
|
396
|
+
}
|
|
397
|
+
function textPreview(fileName, bytes) {
|
|
398
|
+
const mediaType = mediaTypeFor2(fileName);
|
|
399
|
+
if (!mediaType.startsWith("text/") && mediaType !== "application/json") return void 0;
|
|
400
|
+
return bytes.toString("utf8").slice(0, 2e5);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// src/graph.ts
|
|
404
|
+
function buildKnowledgeGraph(pages) {
|
|
405
|
+
const byId = /* @__PURE__ */ new Map();
|
|
406
|
+
const bySlug = /* @__PURE__ */ new Map();
|
|
407
|
+
for (const page of pages) {
|
|
408
|
+
byId.set(page.id, page);
|
|
409
|
+
bySlug.set(normalizeLinkTarget(page.id), page);
|
|
410
|
+
bySlug.set(normalizeLinkTarget(page.title), page);
|
|
411
|
+
bySlug.set(normalizeLinkTarget(page.path.split("/").pop().replace(/\.md$/, "")), page);
|
|
412
|
+
}
|
|
413
|
+
const incoming = /* @__PURE__ */ new Map();
|
|
414
|
+
const outgoing = /* @__PURE__ */ new Map();
|
|
415
|
+
const edgesByKey = /* @__PURE__ */ new Map();
|
|
416
|
+
for (const page of pages) {
|
|
417
|
+
outgoing.set(page.id, 0);
|
|
418
|
+
incoming.set(page.id, 0);
|
|
419
|
+
}
|
|
420
|
+
for (const page of pages) {
|
|
421
|
+
for (const raw of page.outLinks) {
|
|
422
|
+
const target = bySlug.get(normalizeLinkTarget(raw));
|
|
423
|
+
if (!target || target.id === page.id) continue;
|
|
424
|
+
const key = `${page.id}->${target.id}`;
|
|
425
|
+
const edge = edgesByKey.get(key);
|
|
426
|
+
if (edge) edge.weight += 1;
|
|
427
|
+
else edgesByKey.set(key, { source: page.id, target: target.id, weight: 1, reasons: ["wikilink"] });
|
|
428
|
+
outgoing.set(page.id, (outgoing.get(page.id) ?? 0) + 1);
|
|
429
|
+
incoming.set(target.id, (incoming.get(target.id) ?? 0) + 1);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
addSourceOverlapEdges(pages, edgesByKey);
|
|
433
|
+
const nodes = pages.map((page) => ({
|
|
434
|
+
id: page.id,
|
|
435
|
+
title: page.title,
|
|
436
|
+
path: page.path,
|
|
437
|
+
tags: page.tags,
|
|
438
|
+
sourceIds: page.sourceIds,
|
|
439
|
+
outDegree: outgoing.get(page.id) ?? 0,
|
|
440
|
+
inDegree: incoming.get(page.id) ?? 0
|
|
441
|
+
}));
|
|
442
|
+
return { nodes, edges: [...edgesByKey.values()].sort((a, b) => b.weight - a.weight) };
|
|
443
|
+
}
|
|
444
|
+
function addSourceOverlapEdges(pages, edges) {
|
|
445
|
+
for (let i = 0; i < pages.length; i++) {
|
|
446
|
+
for (let j = i + 1; j < pages.length; j++) {
|
|
447
|
+
const a = pages[i];
|
|
448
|
+
const b = pages[j];
|
|
449
|
+
const overlap = a.sourceIds.filter((source) => b.sourceIds.includes(source));
|
|
450
|
+
if (overlap.length === 0) continue;
|
|
451
|
+
const key = `${a.id}->${b.id}`;
|
|
452
|
+
const edge = edges.get(key);
|
|
453
|
+
if (edge) {
|
|
454
|
+
edge.weight += overlap.length * 0.5;
|
|
455
|
+
edge.reasons.push("shared-source");
|
|
456
|
+
} else {
|
|
457
|
+
edges.set(key, {
|
|
458
|
+
source: a.id,
|
|
459
|
+
target: b.id,
|
|
460
|
+
weight: overlap.length * 0.5,
|
|
461
|
+
reasons: ["shared-source"]
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// src/search.ts
|
|
469
|
+
var RRF_K = 60;
|
|
470
|
+
var STOP_WORDS = /* @__PURE__ */ new Set(["the", "is", "a", "an", "what", "how", "are", "was", "were", "to", "for", "of", "with", "by", "in", "on", "and"]);
|
|
471
|
+
function searchKnowledge(index, query, limit = 10) {
|
|
472
|
+
const trimmed = query.trim();
|
|
473
|
+
if (trimmed === "") return [];
|
|
474
|
+
const tokenRanked = rankByTokens(index.pages, trimmed);
|
|
475
|
+
const graphRanked = rankByGraph(index.pages, tokenRanked);
|
|
476
|
+
const scores = reciprocalRankFusion([tokenRanked.map((p) => p.id), graphRanked.map((p) => p.id)]);
|
|
477
|
+
const byId = new Map(index.pages.map((page) => [page.id, page]));
|
|
478
|
+
return [...scores.entries()].map(([id, score]) => ({ page: byId.get(id), score })).filter((item) => Boolean(item.page)).sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)).slice(0, limit).map((item, i) => ({
|
|
479
|
+
page: item.page,
|
|
480
|
+
score: item.score,
|
|
481
|
+
rank: i + 1,
|
|
482
|
+
snippet: buildSnippet(item.page.text, trimmed),
|
|
483
|
+
reasons: reasonsFor(item.page, trimmed)
|
|
484
|
+
}));
|
|
485
|
+
}
|
|
486
|
+
function tokenizeQuery(query) {
|
|
487
|
+
const raw = query.toLowerCase().split(/[\s,,。!?、;:""''()()\-_/\\·~~…]+/).filter((token) => token.length > 1 && !STOP_WORDS.has(token));
|
|
488
|
+
const tokens = [];
|
|
489
|
+
for (const token of raw) {
|
|
490
|
+
if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(token) && token.length > 2) {
|
|
491
|
+
const chars = [...token];
|
|
492
|
+
for (let i = 0; i < chars.length - 1; i++) tokens.push(chars[i] + chars[i + 1]);
|
|
493
|
+
tokens.push(...chars);
|
|
494
|
+
}
|
|
495
|
+
tokens.push(token);
|
|
496
|
+
}
|
|
497
|
+
return [...new Set(tokens)];
|
|
498
|
+
}
|
|
499
|
+
function reciprocalRankFusion(rankLists, k = RRF_K) {
|
|
500
|
+
const scores = /* @__PURE__ */ new Map();
|
|
501
|
+
for (const list of rankLists) {
|
|
502
|
+
list.forEach((id, idx) => {
|
|
503
|
+
scores.set(id, (scores.get(id) ?? 0) + 1 / (k + idx + 1));
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
return scores;
|
|
507
|
+
}
|
|
508
|
+
function rankByTokens(pages, query) {
|
|
509
|
+
const tokens = tokenizeQuery(query);
|
|
510
|
+
const effective = tokens.length > 0 ? tokens : [query.toLowerCase()];
|
|
511
|
+
return pages.map((page) => ({ page, score: tokenScore(page, query, effective) })).filter((item) => item.score > 0).sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)).map((item) => item.page);
|
|
512
|
+
}
|
|
513
|
+
function rankByGraph(pages, tokenRanked) {
|
|
514
|
+
if (tokenRanked.length === 0) return [];
|
|
515
|
+
const seeds = new Set(tokenRanked.slice(0, 5).map((page) => page.id));
|
|
516
|
+
return pages.map((page) => ({
|
|
517
|
+
page,
|
|
518
|
+
score: page.outLinks.filter((link) => seeds.has(link)).length + page.sourceIds.filter((source) => tokenRanked.some((seed) => seed.sourceIds.includes(source))).length
|
|
519
|
+
})).filter((item) => item.score > 0).sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)).map((item) => item.page);
|
|
520
|
+
}
|
|
521
|
+
function tokenScore(page, query, tokens) {
|
|
522
|
+
const title = page.title.toLowerCase();
|
|
523
|
+
const path = page.path.toLowerCase();
|
|
524
|
+
const body = page.text.toLowerCase();
|
|
525
|
+
const phrase = query.toLowerCase();
|
|
526
|
+
let score = 0;
|
|
527
|
+
if (path.endsWith(`${phrase}.md`) || title === phrase) score += 200;
|
|
528
|
+
if (title.includes(phrase)) score += 50;
|
|
529
|
+
if (body.includes(phrase)) score += 20;
|
|
530
|
+
for (const token of tokens) {
|
|
531
|
+
if (title.includes(token)) score += 5;
|
|
532
|
+
if (body.includes(token)) score += 1;
|
|
533
|
+
if (path.includes(token)) score += 3;
|
|
534
|
+
}
|
|
535
|
+
return score;
|
|
536
|
+
}
|
|
537
|
+
function buildSnippet(text, query) {
|
|
538
|
+
const compact = text.replace(/\s+/g, " ").trim();
|
|
539
|
+
const idx = compact.toLowerCase().indexOf(query.toLowerCase());
|
|
540
|
+
if (idx < 0) return compact.slice(0, 180);
|
|
541
|
+
return compact.slice(Math.max(0, idx - 80), Math.min(compact.length, idx + query.length + 100));
|
|
542
|
+
}
|
|
543
|
+
function reasonsFor(page, query) {
|
|
544
|
+
const lower = `${page.title}
|
|
545
|
+
${page.text}`.toLowerCase();
|
|
546
|
+
const reasons = [];
|
|
547
|
+
if (lower.includes(query.toLowerCase())) reasons.push("phrase");
|
|
548
|
+
if (page.sourceIds.length > 0) reasons.push("sourced");
|
|
549
|
+
if (page.outLinks.length > 0) reasons.push("linked");
|
|
550
|
+
return reasons;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// src/indexer.ts
|
|
554
|
+
import { join as join4 } from "path";
|
|
555
|
+
async function buildKnowledgeIndex(root) {
|
|
556
|
+
const [pages, sourceRegistry] = await Promise.all([
|
|
557
|
+
loadKnowledgePages(root),
|
|
558
|
+
loadSourceRegistry(root)
|
|
559
|
+
]);
|
|
560
|
+
const index = {
|
|
561
|
+
root,
|
|
562
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
563
|
+
sources: sourceRegistry.sources,
|
|
564
|
+
pages,
|
|
565
|
+
graph: buildKnowledgeGraph(pages)
|
|
566
|
+
};
|
|
567
|
+
return index;
|
|
568
|
+
}
|
|
569
|
+
async function writeKnowledgeIndex(root) {
|
|
570
|
+
const index = await buildKnowledgeIndex(root);
|
|
571
|
+
await writeJson(join4(layoutFor(root).cacheDir, "index.json"), index);
|
|
572
|
+
return index;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// src/lint.ts
|
|
576
|
+
function lintKnowledgeIndex(index) {
|
|
577
|
+
const findings = [];
|
|
578
|
+
const byTarget = /* @__PURE__ */ new Set();
|
|
579
|
+
const titles = /* @__PURE__ */ new Map();
|
|
580
|
+
const sourceIds = new Set(index.sources.map((source) => source.id));
|
|
581
|
+
const anchorIds = new Map(index.sources.map((source) => [source.id, new Set((source.anchors ?? []).map((anchor) => anchor.id))]));
|
|
582
|
+
for (const page of index.pages) {
|
|
583
|
+
byTarget.add(normalizeLinkTarget(page.id));
|
|
584
|
+
byTarget.add(normalizeLinkTarget(page.title));
|
|
585
|
+
byTarget.add(normalizeLinkTarget(page.path.split("/").pop().replace(/\.md$/, "")));
|
|
586
|
+
const titleKey = page.title.toLowerCase();
|
|
587
|
+
titles.set(titleKey, [...titles.get(titleKey) ?? [], page.path]);
|
|
588
|
+
}
|
|
589
|
+
const inbound = /* @__PURE__ */ new Map();
|
|
590
|
+
for (const page of index.pages) inbound.set(page.id, 0);
|
|
591
|
+
for (const page of index.pages) {
|
|
592
|
+
if (page.outLinks.length === 0 && !isStructural(page.path)) {
|
|
593
|
+
findings.push({ type: "no-outlinks", severity: "info", page: page.path, message: "Page has no wikilinks to other knowledge pages." });
|
|
594
|
+
}
|
|
595
|
+
for (const link of page.outLinks) {
|
|
596
|
+
if (!byTarget.has(normalizeLinkTarget(link))) {
|
|
597
|
+
findings.push({ type: "broken-link", severity: "warning", page: page.path, message: `Broken wikilink [[${link}]].` });
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
for (const edge of index.graph.edges) inbound.set(edge.target, (inbound.get(edge.target) ?? 0) + 1);
|
|
602
|
+
for (const page of index.pages) {
|
|
603
|
+
if (!isStructural(page.path) && (inbound.get(page.id) ?? 0) === 0) {
|
|
604
|
+
findings.push({ type: "orphan", severity: "info", page: page.path, message: "No other page links to this page." });
|
|
605
|
+
}
|
|
606
|
+
if (/\bclaim\b/i.test(page.text) && page.sourceIds.length === 0) {
|
|
607
|
+
findings.push({ type: "uncited-claim", severity: "warning", page: page.path, message: "Page appears to contain claims but has no sources frontmatter." });
|
|
608
|
+
}
|
|
609
|
+
for (const sourceId of page.sourceIds) {
|
|
610
|
+
if (!sourceIds.has(sourceId)) {
|
|
611
|
+
findings.push({ type: "missing-source", severity: "error", page: page.path, message: `Page cites unknown source "${sourceId}".`, metadata: { sourceId } });
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
for (const ref of extractSourceRefs(page.text)) {
|
|
615
|
+
if (!sourceIds.has(ref.sourceId)) {
|
|
616
|
+
findings.push({ type: "missing-source", severity: "error", page: page.path, message: `Page cites unknown source "${ref.sourceId}".`, metadata: ref });
|
|
617
|
+
} else if (ref.anchorId && !anchorIds.get(ref.sourceId)?.has(ref.anchorId)) {
|
|
618
|
+
findings.push({ type: "missing-source", severity: "error", page: page.path, message: `Page cites unknown source anchor "${ref.sourceId}#${ref.anchorId}".`, metadata: ref });
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
for (const [title, paths] of titles) {
|
|
623
|
+
if (title && paths.length > 1) {
|
|
624
|
+
findings.push({ type: "duplicate-title", severity: "warning", message: `Duplicate title "${title}" in ${paths.join(", ")}.`, metadata: { paths } });
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
return findings;
|
|
628
|
+
}
|
|
629
|
+
function isStructural(path) {
|
|
630
|
+
return path.endsWith("/index.md") || path.endsWith("/log.md") || path === "knowledge/index.md" || path === "knowledge/log.md";
|
|
631
|
+
}
|
|
632
|
+
function extractSourceRefs(text) {
|
|
633
|
+
const refs = [];
|
|
634
|
+
const regex = /\[\^([A-Za-z0-9_-]+)(?:#([A-Za-z0-9_.:-]+))?\]/g;
|
|
635
|
+
let match;
|
|
636
|
+
while ((match = regex.exec(text)) !== null) {
|
|
637
|
+
refs.push({ sourceId: match[1], anchorId: match[2] });
|
|
638
|
+
}
|
|
639
|
+
return refs;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
// src/inspect.ts
|
|
643
|
+
function inspectKnowledgeIndex(index) {
|
|
644
|
+
const findings = lintKnowledgeIndex(index);
|
|
645
|
+
const degree = new Map(index.graph.nodes.map((node) => [node.id, node.inDegree + node.outDegree]));
|
|
646
|
+
return {
|
|
647
|
+
pageCount: index.pages.length,
|
|
648
|
+
sourceCount: index.sources.length,
|
|
649
|
+
edgeCount: index.graph.edges.length,
|
|
650
|
+
findingCount: findings.length,
|
|
651
|
+
blockingFindingCount: findings.filter((finding) => finding.severity === "error").length,
|
|
652
|
+
topPages: [...index.pages].sort((a, b) => (degree.get(b.id) ?? 0) - (degree.get(a.id) ?? 0)).slice(0, 10).map((page) => ({ path: page.path, title: page.title, degree: degree.get(page.id) ?? 0, sources: page.sourceIds.length })),
|
|
653
|
+
findings
|
|
654
|
+
};
|
|
655
|
+
}
|
|
656
|
+
function explainKnowledgeTarget(index, target) {
|
|
657
|
+
const page = index.pages.find((candidate) => candidate.path === target || candidate.id === target || candidate.title.toLowerCase() === target.toLowerCase());
|
|
658
|
+
const inbound = page ? index.graph.edges.filter((edge) => edge.target === page.id).map((edge) => index.pages.find((candidate) => candidate.id === edge.source)?.path ?? edge.source) : [];
|
|
659
|
+
const related = page ? searchKnowledge(index, `${page.title} ${page.tags.join(" ")}`, 6).filter((result) => result.page.id !== page.id).map((result) => ({ path: result.page.path, title: result.page.title, score: result.score })) : searchKnowledge(index, target, 6).map((result) => ({ path: result.page.path, title: result.page.title, score: result.score }));
|
|
660
|
+
return {
|
|
661
|
+
target,
|
|
662
|
+
page,
|
|
663
|
+
sources: page ? index.sources.filter((source) => page.sourceIds.includes(source.id)).map((source) => ({ id: source.id, title: source.title, uri: source.uri })) : [],
|
|
664
|
+
links: page?.outLinks ?? [],
|
|
665
|
+
inbound,
|
|
666
|
+
related
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
export {
|
|
671
|
+
sha256,
|
|
672
|
+
slugify,
|
|
673
|
+
stableId,
|
|
674
|
+
parseFrontmatter,
|
|
675
|
+
formatFrontmatter,
|
|
676
|
+
WIKILINK_REGEX,
|
|
677
|
+
extractWikilinks,
|
|
678
|
+
normalizeLinkTarget,
|
|
679
|
+
isSafeKnowledgePath,
|
|
680
|
+
parseKnowledgeWriteBlocks,
|
|
681
|
+
textSourceAdapter,
|
|
682
|
+
mediaTypeFor,
|
|
683
|
+
applyKnowledgeWriteBlocks,
|
|
684
|
+
applyKnowledgeWriteBlocksFile,
|
|
685
|
+
layoutFor,
|
|
686
|
+
initKnowledgeBase,
|
|
687
|
+
loadKnowledgePages,
|
|
688
|
+
writeJson,
|
|
689
|
+
loadSourceRegistry,
|
|
690
|
+
writeSourceRegistry,
|
|
691
|
+
addSourcePath,
|
|
692
|
+
sourceRegistryPath,
|
|
693
|
+
buildKnowledgeGraph,
|
|
694
|
+
searchKnowledge,
|
|
695
|
+
tokenizeQuery,
|
|
696
|
+
reciprocalRankFusion,
|
|
697
|
+
buildKnowledgeIndex,
|
|
698
|
+
writeKnowledgeIndex,
|
|
699
|
+
lintKnowledgeIndex,
|
|
700
|
+
inspectKnowledgeIndex,
|
|
701
|
+
explainKnowledgeTarget
|
|
702
|
+
};
|
|
703
|
+
//# sourceMappingURL=chunk-XDXIBHPS.js.map
|