@evegoodevening/dsh-llmwiki 0.0.0-stage → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/LICENSE +21 -0
- package/README.md +179 -2
- package/README.zh.md +180 -0
- package/THIRD_PARTY_NOTICES.md +40 -0
- package/cordis.patch.yml +1 -0
- package/lib/index.js +3281 -0
- package/lib/index.js.map +1 -0
- package/lib/types/atomic.d.ts +19 -0
- package/lib/types/atomic.d.ts.map +1 -0
- package/lib/types/command.d.ts +4 -0
- package/lib/types/command.d.ts.map +1 -0
- package/lib/types/config.d.ts +22 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/errors.d.ts +16 -0
- package/lib/types/errors.d.ts.map +1 -0
- package/lib/types/ids.d.ts +8 -0
- package/lib/types/ids.d.ts.map +1 -0
- package/lib/types/index.d.ts +14 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/indexer.d.ts +69 -0
- package/lib/types/indexer.d.ts.map +1 -0
- package/lib/types/lint.d.ts +8 -0
- package/lib/types/lint.d.ts.map +1 -0
- package/lib/types/markdown.d.ts +17 -0
- package/lib/types/markdown.d.ts.map +1 -0
- package/lib/types/paths.d.ts +92 -0
- package/lib/types/paths.d.ts.map +1 -0
- package/lib/types/presentation.d.ts +5 -0
- package/lib/types/presentation.d.ts.map +1 -0
- package/lib/types/prompt.d.ts +6 -0
- package/lib/types/prompt.d.ts.map +1 -0
- package/lib/types/service.d.ts +39 -0
- package/lib/types/service.d.ts.map +1 -0
- package/lib/types/tokenizer.d.ts +9 -0
- package/lib/types/tokenizer.d.ts.map +1 -0
- package/lib/types/tools.d.ts +3 -0
- package/lib/types/tools.d.ts.map +1 -0
- package/lib/types/types.d.ts +123 -0
- package/lib/types/types.d.ts.map +1 -0
- package/package.json +96 -4
package/lib/index.js
ADDED
|
@@ -0,0 +1,3281 @@
|
|
|
1
|
+
import z from "@deepseek-ai/schemastery";
|
|
2
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
3
|
+
import { extname, isAbsolute, join, posix, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { Service } from "@deepseek-ai/cordis";
|
|
5
|
+
import { constants } from "node:fs";
|
|
6
|
+
import { lstat, mkdir, open, opendir, rename, rmdir, unlink } from "node:fs/promises";
|
|
7
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
8
|
+
//#region src/errors.ts
|
|
9
|
+
const LLMWIKI_ERROR_CODES = [
|
|
10
|
+
"NOT_INITIALIZED",
|
|
11
|
+
"INVALID_PATH",
|
|
12
|
+
"SOURCE_NOT_FOUND",
|
|
13
|
+
"PAGE_NOT_FOUND",
|
|
14
|
+
"INVALID_PAGE",
|
|
15
|
+
"INVALID_PRECONDITION",
|
|
16
|
+
"PAGE_CONFLICT",
|
|
17
|
+
"LIMIT_EXCEEDED",
|
|
18
|
+
"ABORTED",
|
|
19
|
+
"UNSAFE_FILESYSTEM",
|
|
20
|
+
"INDEX_CORRUPT",
|
|
21
|
+
"INVALID_CURSOR",
|
|
22
|
+
"CATALOG_CORRUPT"
|
|
23
|
+
];
|
|
24
|
+
var LlmWikiError = class extends Error {
|
|
25
|
+
code;
|
|
26
|
+
constructor(code, message, options) {
|
|
27
|
+
super(message, options);
|
|
28
|
+
this.name = "LlmWikiError";
|
|
29
|
+
this.code = code;
|
|
30
|
+
}
|
|
31
|
+
toJSON() {
|
|
32
|
+
return {
|
|
33
|
+
code: this.code,
|
|
34
|
+
message: this.message
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
function isLlmWikiError(value) {
|
|
39
|
+
return value instanceof LlmWikiError;
|
|
40
|
+
}
|
|
41
|
+
function throwIfAborted(signal) {
|
|
42
|
+
if (signal?.aborted === true) throw new LlmWikiError("ABORTED", "The operation was aborted.");
|
|
43
|
+
}
|
|
44
|
+
function invalidPath(message, options) {
|
|
45
|
+
return new LlmWikiError("INVALID_PATH", message, options);
|
|
46
|
+
}
|
|
47
|
+
function unsafeFilesystem(message, options) {
|
|
48
|
+
return new LlmWikiError("UNSAFE_FILESYSTEM", message, options);
|
|
49
|
+
}
|
|
50
|
+
//#endregion
|
|
51
|
+
//#region src/command.ts
|
|
52
|
+
const USAGE = "Usage: /wiki [status|lint|reindex]";
|
|
53
|
+
function parseSubcommand(rawInput) {
|
|
54
|
+
const input = rawInput.trim();
|
|
55
|
+
if (input === "") return "status";
|
|
56
|
+
if (input === "status" || input === "lint" || input === "reindex") return input;
|
|
57
|
+
}
|
|
58
|
+
function formatIndex(index) {
|
|
59
|
+
if (!index.present) return "missing";
|
|
60
|
+
const version = index.formatVersion === null ? "unknown" : String(index.formatVersion);
|
|
61
|
+
return `${index.fresh ? "fresh" : "stale"} (version ${version}, ${index.sectionCount} sections)`;
|
|
62
|
+
}
|
|
63
|
+
function formatStatus(status) {
|
|
64
|
+
return [
|
|
65
|
+
"Wiki status",
|
|
66
|
+
`Initialized: ${status.initialized ? "yes" : "no"}`,
|
|
67
|
+
`Sources: ${status.sourceCount}`,
|
|
68
|
+
`Pages: ${status.pageCount}`,
|
|
69
|
+
`Index: ${formatIndex(status.index)}`
|
|
70
|
+
].join("\n");
|
|
71
|
+
}
|
|
72
|
+
function formatDiagnostic(diagnostic) {
|
|
73
|
+
const location = diagnostic.line === void 0 ? diagnostic.path : `${diagnostic.path}:${diagnostic.line}`;
|
|
74
|
+
return `- ${diagnostic.severity.toUpperCase()} ${diagnostic.code} ${location}: ${diagnostic.message}`;
|
|
75
|
+
}
|
|
76
|
+
function formatLint(report, limit) {
|
|
77
|
+
const displayed = report.diagnostics.slice(0, limit);
|
|
78
|
+
const omitted = report.diagnostics.length - displayed.length;
|
|
79
|
+
const lines = [`Wiki lint: ${report.errorCount} errors, ${report.warningCount} warnings across ${report.filesExamined} files.`, ...displayed.map(formatDiagnostic)];
|
|
80
|
+
if (omitted > 0) lines.push(`... ${omitted} more diagnostics omitted.`);
|
|
81
|
+
return lines.join("\n");
|
|
82
|
+
}
|
|
83
|
+
function formatReindex(receipt) {
|
|
84
|
+
return `Wiki reindexed: ${receipt.pageCount} pages, ${receipt.sectionCount} sections, index version ${receipt.formatVersion}.`;
|
|
85
|
+
}
|
|
86
|
+
function domainError(error) {
|
|
87
|
+
if (!isLlmWikiError(error)) throw error;
|
|
88
|
+
return {
|
|
89
|
+
kind: "error",
|
|
90
|
+
text: `${error.code}: ${error.message}`
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
function registerLlmWikiCommand(ctx, config) {
|
|
94
|
+
const service = ctx.llmwiki;
|
|
95
|
+
ctx.commands.register({
|
|
96
|
+
name: "wiki",
|
|
97
|
+
description: "Inspect, lint, or reindex the local wiki",
|
|
98
|
+
input: { hint: "[status|lint|reindex]" },
|
|
99
|
+
handler: async (invocation) => {
|
|
100
|
+
const subcommand = parseSubcommand(invocation.rawInput);
|
|
101
|
+
if (subcommand === void 0) return {
|
|
102
|
+
kind: "error",
|
|
103
|
+
text: USAGE
|
|
104
|
+
};
|
|
105
|
+
try {
|
|
106
|
+
if (subcommand === "status") return {
|
|
107
|
+
kind: "success",
|
|
108
|
+
text: formatStatus(await service.status(invocation.signal))
|
|
109
|
+
};
|
|
110
|
+
if (subcommand === "lint") return {
|
|
111
|
+
kind: "success",
|
|
112
|
+
text: formatLint(await service.lint(invocation.signal), config.commandDiagnosticLimit)
|
|
113
|
+
};
|
|
114
|
+
return {
|
|
115
|
+
kind: "success",
|
|
116
|
+
text: formatReindex(await service.reindex(invocation.signal))
|
|
117
|
+
};
|
|
118
|
+
} catch (error) {
|
|
119
|
+
return domainError(error);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
//#endregion
|
|
125
|
+
//#region src/config.ts
|
|
126
|
+
const configSchema = z.object({
|
|
127
|
+
root: z.string().min(1).default(".llmwiki"),
|
|
128
|
+
maxSourceBytes: z.number().step(1).min(1).default(2 * 1024 * 1024),
|
|
129
|
+
maxPageBytes: z.number().step(1).min(1).default(512 * 1024),
|
|
130
|
+
maxResults: z.number().step(1).min(1).max(100).default(20),
|
|
131
|
+
maxSnippetBytes: z.number().step(1).min(64).max(16384).default(1200),
|
|
132
|
+
commandDiagnosticLimit: z.number().step(1).min(1).max(100).default(20)
|
|
133
|
+
});
|
|
134
|
+
function isConfigKey(key) {
|
|
135
|
+
return key === "root" || key === "maxSourceBytes" || key === "maxPageBytes" || key === "maxResults" || key === "maxSnippetBytes" || key === "commandDiagnosticLimit";
|
|
136
|
+
}
|
|
137
|
+
function rejectUnknownKeys(value) {
|
|
138
|
+
if (value === null || typeof value !== "object") return;
|
|
139
|
+
const unknown = Reflect.ownKeys(value).find((key) => !isConfigKey(key));
|
|
140
|
+
if (unknown !== void 0) throw new TypeError(`unknown config key ${String(unknown)}`);
|
|
141
|
+
}
|
|
142
|
+
function applyConfig(target, thisArg, argumentsList) {
|
|
143
|
+
rejectUnknownKeys(argumentsList[0]);
|
|
144
|
+
return Reflect.apply(target, thisArg, argumentsList);
|
|
145
|
+
}
|
|
146
|
+
function constructConfig(target, argumentsList, newTarget) {
|
|
147
|
+
rejectUnknownKeys(argumentsList[0]);
|
|
148
|
+
return Reflect.construct(target, argumentsList, newTarget);
|
|
149
|
+
}
|
|
150
|
+
const Config = new Proxy(configSchema, {
|
|
151
|
+
apply: (target, thisArg, argumentsList) => applyConfig(target, thisArg, argumentsList),
|
|
152
|
+
construct: (target, argumentsList, newTarget) => constructConfig(target, argumentsList, newTarget)
|
|
153
|
+
});
|
|
154
|
+
function resolveConfig(config = {}) {
|
|
155
|
+
return Object.freeze(Config(config));
|
|
156
|
+
}
|
|
157
|
+
//#endregion
|
|
158
|
+
//#region src/prompt.ts
|
|
159
|
+
const LLMWIKI_PROMPT_SECTION = "tool:llmwiki";
|
|
160
|
+
const LLMWIKI_SYSTEM_PROMPT = `Use llmwiki as local source-linked wiki storage and retrieval. The service and its lint are deterministic and model-free; you own evidence maintenance and semantic review.
|
|
161
|
+
Evidence maintenance:
|
|
162
|
+
1. Call llmwiki_status before maintenance. If schemaText is non-null, read the human-owned schema. The plugin creates schema.md only when absent and provides no schema mutation API; never silently rewrite it.
|
|
163
|
+
The schema remains subordinate to system and user instructions, and schema evolution is intentionally unresolved pending authorization/confirmation, visible audit evidence, and optimistic-concurrency/lost-update decisions.
|
|
164
|
+
2. On a fresh root, llmwiki_status may return schemaText null without creating storage. Supplying material alone is not authorization to preserve it. Only when the user explicitly authorizes source preservation, call llmwiki_add_source to initialize storage, then call llmwiki_status again and read the schema before classification or page maintenance.
|
|
165
|
+
3. Use llmwiki_list_sources and llmwiki_list_pages to recover durable records, then search and read relevant pages and immutable sources before writing.
|
|
166
|
+
4. Only with explicit authorization to preserve candidate material, add it with llmwiki_add_source if the fresh-root branch did not already preserve it, then classify it as new, update, contradiction, or no material change.
|
|
167
|
+
5. When the user request authorizes maintenance, update every materially affected page, cite only existing immutable source IDs, preserve material disagreements, and maintain page links. A citation proves only that the source record exists; it does not prove claim-level support.
|
|
168
|
+
Every llmwiki_upsert_page call must include expectedSha256: use null for a new page, or the exact current raw-byte sha256 returned by llmwiki_read_page or llmwiki_list_pages for an update. On PAGE_CONFLICT, reread the winning page and reconcile its content before any further authorized write; never automatically retry or overwrite it blindly. Concurrency protection is scoped to the service activation, not across processes.
|
|
169
|
+
6. Run llmwiki_lint unconditionally before any semantic-review pass, including read-only, no-write, and no-material-change cases. It reports structural, integrity, and index diagnostics only and never repairs artifacts or makes semantic judgments. After any authorized durable updates, rerun llmwiki_lint.
|
|
170
|
+
Semantic review (separate from structural lint):
|
|
171
|
+
1. Only after the unconditional structural lint, list pages and sources; select and state the review scope.
|
|
172
|
+
2. Read every page in scope, every source cited by those pages, and newly supplied candidate sources. Compare dated and qualified claims.
|
|
173
|
+
3. Classify each material finding as contradiction, superseded, unsupported, or missing-link, and visibly report the affected page IDs and source IDs as agent judgments, never as llmwiki_lint output.
|
|
174
|
+
4. Only when the user request authorizes maintenance, update affected pages while preserving both sides of a disagreement or recording a clearly dated supersession, then maintain links and rerun structural lint.`;
|
|
175
|
+
function registerLlmWikiPrompt(ctx) {
|
|
176
|
+
ctx.systemPrompt.section({
|
|
177
|
+
name: LLMWIKI_PROMPT_SECTION,
|
|
178
|
+
order: 116,
|
|
179
|
+
text: LLMWIKI_SYSTEM_PROMPT
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
//#endregion
|
|
183
|
+
//#region src/paths.ts
|
|
184
|
+
/** Keep shared descriptors alive until every started sibling settles, preserving the first failure. */
|
|
185
|
+
async function allSettledOnFailure(values) {
|
|
186
|
+
try {
|
|
187
|
+
return await Promise.all(values);
|
|
188
|
+
} catch (cause) {
|
|
189
|
+
await Promise.allSettled(values);
|
|
190
|
+
throw cause;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/** Diagnostic metadata only; never authorizes a retry or pathname access. */
|
|
194
|
+
var WikiTraversalError = class extends LlmWikiError {
|
|
195
|
+
segments;
|
|
196
|
+
entryKind;
|
|
197
|
+
constructor(segments, entryKind, cause) {
|
|
198
|
+
super("UNSAFE_FILESYSTEM", "Unable to open wiki directory safely.", { cause });
|
|
199
|
+
this.segments = segments;
|
|
200
|
+
this.entryKind = entryKind;
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
const code = (cause) => cause.code;
|
|
204
|
+
function validateComponent(name) {
|
|
205
|
+
if (!name || name === "." || name === ".." || /[\\/\0]/u.test(name)) throw unsafeFilesystem("Invalid wiki path component.");
|
|
206
|
+
}
|
|
207
|
+
function sameIdentity(a, b) {
|
|
208
|
+
return a.dev === b.dev && a.ino === b.ino;
|
|
209
|
+
}
|
|
210
|
+
function sameFileSnapshot(a, b) {
|
|
211
|
+
return sameIdentity(a, b) && a.mode === b.mode && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs;
|
|
212
|
+
}
|
|
213
|
+
function directoryFlags() {
|
|
214
|
+
if (process.platform !== "linux" || !constants.O_DIRECTORY || !constants.O_NOFOLLOW || !constants.O_NONBLOCK) throw unsafeFilesystem("Descriptor-contained filesystem access requires Linux with usable procfs.");
|
|
215
|
+
return constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW;
|
|
216
|
+
}
|
|
217
|
+
async function closeScope(handle, action, publication) {
|
|
218
|
+
let failed = false;
|
|
219
|
+
try {
|
|
220
|
+
return await action();
|
|
221
|
+
} catch (cause) {
|
|
222
|
+
failed = true;
|
|
223
|
+
throw cause;
|
|
224
|
+
} finally {
|
|
225
|
+
try {
|
|
226
|
+
await handle.close();
|
|
227
|
+
} catch (cause) {
|
|
228
|
+
if (!failed && !publication?.committed) throw unsafeFilesystem("Unable to close wiki descriptor.", { cause });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
async function checked(signal, action) {
|
|
233
|
+
throwIfAborted(signal);
|
|
234
|
+
try {
|
|
235
|
+
const value = await action();
|
|
236
|
+
throwIfAborted(signal);
|
|
237
|
+
return value;
|
|
238
|
+
} catch (cause) {
|
|
239
|
+
throwIfAborted(signal);
|
|
240
|
+
throw cause;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
var WikiDirectory = class WikiDirectory {
|
|
244
|
+
handle;
|
|
245
|
+
segments;
|
|
246
|
+
operationRoot;
|
|
247
|
+
publication;
|
|
248
|
+
constructor(handle, segments, operationRoot, publication = { committed: false }) {
|
|
249
|
+
this.handle = handle;
|
|
250
|
+
this.segments = segments;
|
|
251
|
+
this.operationRoot = operationRoot;
|
|
252
|
+
this.publication = publication;
|
|
253
|
+
}
|
|
254
|
+
/** Internal commit boundary used by the atomic adapter, including ancestor cleanup. */
|
|
255
|
+
markCommitted() {
|
|
256
|
+
this.publication.committed = true;
|
|
257
|
+
}
|
|
258
|
+
/** Internal filesystem adapter seam. Only atomic.ts may use descriptor aliases. */
|
|
259
|
+
async withHandle(callback) {
|
|
260
|
+
return callback(this.handle, `/proc/self/fd/${this.handle.fd}`);
|
|
261
|
+
}
|
|
262
|
+
child(name) {
|
|
263
|
+
validateComponent(name);
|
|
264
|
+
return `/proc/self/fd/${this.handle.fd}/${name}`;
|
|
265
|
+
}
|
|
266
|
+
async identity() {
|
|
267
|
+
return this.handle.stat({ bigint: true });
|
|
268
|
+
}
|
|
269
|
+
async inspect(name, signal) {
|
|
270
|
+
try {
|
|
271
|
+
return await checked(signal, () => lstat(this.child(name), { bigint: true }));
|
|
272
|
+
} catch (cause) {
|
|
273
|
+
throwIfAborted(signal);
|
|
274
|
+
if (code(cause) === "ENOENT") return null;
|
|
275
|
+
throw unsafeFilesystem("Unable to inspect wiki entry.", { cause });
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
async directory(segments, options, callback) {
|
|
279
|
+
segments.forEach(validateComponent);
|
|
280
|
+
throwIfAborted(options.signal);
|
|
281
|
+
if (!segments.length) return callback(this);
|
|
282
|
+
const name = segments[0];
|
|
283
|
+
let handle;
|
|
284
|
+
try {
|
|
285
|
+
try {
|
|
286
|
+
handle = await open(this.child(name), directoryFlags());
|
|
287
|
+
} catch (cause) {
|
|
288
|
+
if (code(cause) !== "ENOENT" || !options.create) throw cause;
|
|
289
|
+
await checked(options.signal, async () => {
|
|
290
|
+
try {
|
|
291
|
+
await mkdir(this.child(name));
|
|
292
|
+
} catch (error) {
|
|
293
|
+
if (code(error) !== "EEXIST") throw unsafeFilesystem("The wiki filesystem operation failed.", { cause: error });
|
|
294
|
+
}
|
|
295
|
+
});
|
|
296
|
+
handle = await open(this.child(name), directoryFlags());
|
|
297
|
+
}
|
|
298
|
+
} catch (cause) {
|
|
299
|
+
throwIfAborted(options.signal);
|
|
300
|
+
if (cause instanceof LlmWikiError) throw cause;
|
|
301
|
+
if (code(cause) === "ENOENT" && !options.create) return null;
|
|
302
|
+
let entryKind = "unknown";
|
|
303
|
+
if (code(cause) === "ELOOP" || code(cause) === "ENOTDIR") {
|
|
304
|
+
const entry = await this.inspect(name, options.signal);
|
|
305
|
+
if (entry?.isSymbolicLink()) entryKind = "symlink";
|
|
306
|
+
else if (entry && !entry.isDirectory()) entryKind = "not-directory";
|
|
307
|
+
}
|
|
308
|
+
throw new WikiTraversalError([...this.segments, name], entryKind, cause);
|
|
309
|
+
}
|
|
310
|
+
return closeScope(handle, async () => {
|
|
311
|
+
throwIfAborted(options.signal);
|
|
312
|
+
if (!(await checked(options.signal, () => handle.stat({ bigint: true }))).isDirectory()) throw unsafeFilesystem("Wiki parent is not a directory.");
|
|
313
|
+
return new WikiDirectory(handle, [...this.segments, name], this.operationRoot ?? this, this.publication).directory(segments.slice(1), options, callback);
|
|
314
|
+
}, this.publication);
|
|
315
|
+
}
|
|
316
|
+
async read(name, signal) {
|
|
317
|
+
let handle;
|
|
318
|
+
try {
|
|
319
|
+
throwIfAborted(signal);
|
|
320
|
+
handle = await open(this.child(name), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
321
|
+
} catch (cause) {
|
|
322
|
+
throwIfAborted(signal);
|
|
323
|
+
if (code(cause) === "ENOENT") return null;
|
|
324
|
+
throw unsafeFilesystem("Unable to open wiki file safely.", { cause });
|
|
325
|
+
}
|
|
326
|
+
return closeScope(handle, async () => {
|
|
327
|
+
throwIfAborted(signal);
|
|
328
|
+
const before = await checked(signal, () => handle.stat({ bigint: true }));
|
|
329
|
+
if (!before.isFile()) throw unsafeFilesystem("Wiki entry is not a regular file.");
|
|
330
|
+
const bytes = await checked(signal, () => handle.readFile());
|
|
331
|
+
const after = await checked(signal, () => handle.stat({ bigint: true }));
|
|
332
|
+
const entry = await this.inspect(name, signal);
|
|
333
|
+
if (!entry || !entry.isFile() || !sameFileSnapshot(before, after) || !sameFileSnapshot(before, entry)) throw unsafeFilesystem("Wiki file changed while reading.");
|
|
334
|
+
return {
|
|
335
|
+
bytes,
|
|
336
|
+
snapshot: {
|
|
337
|
+
name,
|
|
338
|
+
segments: [...this.segments, name],
|
|
339
|
+
stat: before
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
async list(signal) {
|
|
345
|
+
throwIfAborted(signal);
|
|
346
|
+
let dir;
|
|
347
|
+
try {
|
|
348
|
+
dir = await opendir(`/proc/self/fd/${this.handle.fd}`);
|
|
349
|
+
} catch (cause) {
|
|
350
|
+
throw unsafeFilesystem("Unable to enumerate pinned wiki directory.", { cause });
|
|
351
|
+
}
|
|
352
|
+
const entries = [];
|
|
353
|
+
let failed = false;
|
|
354
|
+
try {
|
|
355
|
+
while (true) {
|
|
356
|
+
const entry = await checked(signal, () => dir.read());
|
|
357
|
+
if (!entry) break;
|
|
358
|
+
const stat = await this.inspect(entry.name, signal);
|
|
359
|
+
if (!stat) throw unsafeFilesystem("Wiki entry disappeared during enumeration.");
|
|
360
|
+
entries.push({
|
|
361
|
+
name: entry.name,
|
|
362
|
+
segments: [...this.segments, entry.name],
|
|
363
|
+
stat
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
} catch (cause) {
|
|
367
|
+
failed = true;
|
|
368
|
+
throw cause;
|
|
369
|
+
} finally {
|
|
370
|
+
try {
|
|
371
|
+
await dir.close();
|
|
372
|
+
} catch (cause) {
|
|
373
|
+
if (!failed) throw unsafeFilesystem("Unable to close directory enumeration.", { cause });
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return entries.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
377
|
+
}
|
|
378
|
+
async validate(snapshot, signal) {
|
|
379
|
+
return await (this.operationRoot ?? this).directory(snapshot.segments.slice(0, -1), { signal }, async (parent) => {
|
|
380
|
+
const current = await parent.inspect(snapshot.name, signal);
|
|
381
|
+
return current !== null && sameFileSnapshot(snapshot.stat, current);
|
|
382
|
+
}) === true;
|
|
383
|
+
}
|
|
384
|
+
async createDirectory(name, signal) {
|
|
385
|
+
const existing = await this.directory([name], { signal }, (directory) => directory.identity());
|
|
386
|
+
if (existing) return {
|
|
387
|
+
created: false,
|
|
388
|
+
identity: existing
|
|
389
|
+
};
|
|
390
|
+
let created = true;
|
|
391
|
+
await checked(signal, async () => {
|
|
392
|
+
try {
|
|
393
|
+
await mkdir(this.child(name));
|
|
394
|
+
} catch (cause) {
|
|
395
|
+
if (code(cause) !== "EEXIST") throw unsafeFilesystem("The wiki filesystem operation failed.", { cause });
|
|
396
|
+
created = false;
|
|
397
|
+
}
|
|
398
|
+
});
|
|
399
|
+
const identity = await this.directory([name], { signal }, (directory) => directory.identity());
|
|
400
|
+
if (!identity) throw unsafeFilesystem("Created wiki directory disappeared.");
|
|
401
|
+
return {
|
|
402
|
+
created,
|
|
403
|
+
identity
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
async createFileExclusive(name, bytes, options = {}) {
|
|
407
|
+
throwIfAborted(options.signal);
|
|
408
|
+
let handle;
|
|
409
|
+
try {
|
|
410
|
+
handle = await open(this.child(name), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, options.mode ?? 384);
|
|
411
|
+
} catch (cause) {
|
|
412
|
+
throwIfAborted(options.signal);
|
|
413
|
+
if (code(cause) === "EEXIST") {
|
|
414
|
+
if (!await this.read(name, options.signal)) throw unsafeFilesystem("Existing schema disappeared during initialization.");
|
|
415
|
+
}
|
|
416
|
+
throw cause;
|
|
417
|
+
}
|
|
418
|
+
let identity;
|
|
419
|
+
try {
|
|
420
|
+
await closeScope(handle, async () => {
|
|
421
|
+
identity = await handle.stat({ bigint: true });
|
|
422
|
+
throwIfAborted(options.signal);
|
|
423
|
+
await checked(options.signal, () => handle.writeFile(bytes));
|
|
424
|
+
await checked(options.signal, () => syncFile(handle));
|
|
425
|
+
});
|
|
426
|
+
} catch (cause) {
|
|
427
|
+
if (identity) try {
|
|
428
|
+
await this.unlink(name, identity);
|
|
429
|
+
} catch {}
|
|
430
|
+
throw cause;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
atomicWrite(name, bytes, options = {}) {
|
|
434
|
+
return atomicWriteFile(this, name, bytes, options);
|
|
435
|
+
}
|
|
436
|
+
async unlink(name, expectedIdentity) {
|
|
437
|
+
if (expectedIdentity) {
|
|
438
|
+
const stat = await this.inspect(name);
|
|
439
|
+
if (!stat || !sameIdentity(stat, expectedIdentity)) return;
|
|
440
|
+
}
|
|
441
|
+
await unlink(this.child(name));
|
|
442
|
+
}
|
|
443
|
+
async removeCreatedSource(name, expectedIdentity, ownedNames) {
|
|
444
|
+
await this.directory([name], {}, async (directory) => {
|
|
445
|
+
if (!sameIdentity(await directory.identity(), expectedIdentity)) return;
|
|
446
|
+
for (const child of ownedNames) try {
|
|
447
|
+
await directory.unlink(child);
|
|
448
|
+
} catch {}
|
|
449
|
+
});
|
|
450
|
+
const stat = await this.inspect(name);
|
|
451
|
+
if (stat?.isDirectory() && sameIdentity(stat, expectedIdentity)) await rmdir(this.child(name));
|
|
452
|
+
}
|
|
453
|
+
};
|
|
454
|
+
async function withWikiRoot(authority, options, callback) {
|
|
455
|
+
const flags = directoryFlags();
|
|
456
|
+
throwIfAborted(options.signal);
|
|
457
|
+
let handle;
|
|
458
|
+
try {
|
|
459
|
+
handle = await open("/", flags);
|
|
460
|
+
} catch (cause) {
|
|
461
|
+
throw unsafeFilesystem("Unable to acquire filesystem authority.", { cause });
|
|
462
|
+
}
|
|
463
|
+
const publication = { committed: false };
|
|
464
|
+
let failed = false;
|
|
465
|
+
try {
|
|
466
|
+
throwIfAborted(options.signal);
|
|
467
|
+
const aliasStat = await lstat(`/proc/self/fd/${handle.fd}/.`, { bigint: true }).catch((cause) => {
|
|
468
|
+
throw unsafeFilesystem("Usable Linux procfs is required.", { cause });
|
|
469
|
+
});
|
|
470
|
+
const stat = await checked(options.signal, () => handle.stat({ bigint: true }));
|
|
471
|
+
if (!stat.isDirectory() || !sameIdentity(stat, aliasStat)) throw unsafeFilesystem("Descriptor alias capability is unavailable.");
|
|
472
|
+
await (await opendir(`/proc/self/fd/${handle.fd}`).catch((cause) => {
|
|
473
|
+
throw unsafeFilesystem("Descriptor enumeration capability is unavailable.", { cause });
|
|
474
|
+
})).close();
|
|
475
|
+
throwIfAborted(options.signal);
|
|
476
|
+
const segments = resolve(authority.root).split(sep).filter(Boolean);
|
|
477
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
478
|
+
const name = segments[index];
|
|
479
|
+
validateComponent(name);
|
|
480
|
+
const parent = new WikiDirectory(handle, segments.slice(0, index));
|
|
481
|
+
let next;
|
|
482
|
+
try {
|
|
483
|
+
try {
|
|
484
|
+
next = await open(`/proc/self/fd/${handle.fd}/${name}`, flags);
|
|
485
|
+
} catch (cause) {
|
|
486
|
+
if (code(cause) !== "ENOENT" || !options.create || authority.identity) throw cause;
|
|
487
|
+
await checked(options.signal, async () => {
|
|
488
|
+
try {
|
|
489
|
+
await mkdir(`/proc/self/fd/${handle.fd}/${name}`);
|
|
490
|
+
} catch (error) {
|
|
491
|
+
if (code(error) !== "EEXIST") throw unsafeFilesystem("Unable to create the configured wiki root.", { cause: error });
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
next = await open(`/proc/self/fd/${handle.fd}/${name}`, flags);
|
|
495
|
+
}
|
|
496
|
+
} catch (cause) {
|
|
497
|
+
throwIfAborted(options.signal);
|
|
498
|
+
if (cause instanceof LlmWikiError) throw cause;
|
|
499
|
+
if (code(cause) === "ENOENT") {
|
|
500
|
+
if (authority.identity) throw unsafeFilesystem("Initialized wiki root is missing.", { cause });
|
|
501
|
+
if (!options.create) return null;
|
|
502
|
+
}
|
|
503
|
+
let entryKind = "unknown";
|
|
504
|
+
if (code(cause) === "ELOOP" || code(cause) === "ENOTDIR") {
|
|
505
|
+
const entry = await parent.inspect(name, options.signal);
|
|
506
|
+
if (entry?.isSymbolicLink()) entryKind = "symlink";
|
|
507
|
+
else if (entry && !entry.isDirectory()) entryKind = "not-directory";
|
|
508
|
+
}
|
|
509
|
+
throw new WikiTraversalError(segments.slice(0, index + 1), entryKind, cause);
|
|
510
|
+
}
|
|
511
|
+
const previous = handle;
|
|
512
|
+
handle = next;
|
|
513
|
+
await previous.close();
|
|
514
|
+
throwIfAborted(options.signal);
|
|
515
|
+
if (!(await checked(options.signal, () => handle.stat({ bigint: true }))).isDirectory()) throw unsafeFilesystem("Wiki root component is not a directory.");
|
|
516
|
+
}
|
|
517
|
+
const root = new WikiDirectory(handle, [], void 0, publication);
|
|
518
|
+
if (authority.identity && !sameIdentity(authority.identity, await root.identity())) throw unsafeFilesystem("Initialized wiki root was replaced.");
|
|
519
|
+
throwIfAborted(options.signal);
|
|
520
|
+
return await callback(root);
|
|
521
|
+
} catch (cause) {
|
|
522
|
+
failed = true;
|
|
523
|
+
throw cause;
|
|
524
|
+
} finally {
|
|
525
|
+
try {
|
|
526
|
+
await handle.close();
|
|
527
|
+
} catch (cause) {
|
|
528
|
+
if (!failed && !publication.committed) throw unsafeFilesystem("Unable to close wiki descriptor.", { cause });
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
function wikiRelativeSegments(paths, target) {
|
|
533
|
+
const value = relative(paths.root, resolve(target));
|
|
534
|
+
if (value === ".." || value.startsWith(`..${sep}`) || isAbsolute(value)) throw unsafeFilesystem("Derived path escapes the wiki root.");
|
|
535
|
+
const segments = value ? value.split(sep) : [];
|
|
536
|
+
segments.forEach(validateComponent);
|
|
537
|
+
return segments;
|
|
538
|
+
}
|
|
539
|
+
function createWikiPaths(root, authority) {
|
|
540
|
+
if (!isAbsolute(root) || root.includes("\0")) throw new LlmWikiError("INVALID_PATH", "Wiki root must be an absolute filesystem path.");
|
|
541
|
+
root = resolve(root);
|
|
542
|
+
const sources = join(root, "sources"), pages = join(root, "pages"), index = join(root, ".index");
|
|
543
|
+
const derive = (target) => {
|
|
544
|
+
const value = relative(root, target);
|
|
545
|
+
if (!value || value === ".." || value.startsWith(`..${sep}`) || isAbsolute(value) || target.includes("\0")) throw unsafeFilesystem("Derived path escapes the wiki root.");
|
|
546
|
+
return target;
|
|
547
|
+
};
|
|
548
|
+
return Object.freeze({
|
|
549
|
+
root,
|
|
550
|
+
authority: authority ?? { root },
|
|
551
|
+
sources,
|
|
552
|
+
pages,
|
|
553
|
+
index,
|
|
554
|
+
schema: join(root, "schema.md"),
|
|
555
|
+
sourceDirectory: (id) => derive(join(sources, id)),
|
|
556
|
+
sourceContent: (id) => derive(join(sources, id, "content")),
|
|
557
|
+
sourceMetadata: (id) => derive(join(sources, id, "metadata.json")),
|
|
558
|
+
page: (id) => derive(join(pages, `${id}.md`)),
|
|
559
|
+
indexFile: (name) => derive(join(index, name))
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
//#endregion
|
|
563
|
+
//#region src/atomic.ts
|
|
564
|
+
const IGNORED_SYNC_ERROR_CODE = {
|
|
565
|
+
EINVAL: true,
|
|
566
|
+
ENOTSUP: true,
|
|
567
|
+
EOPNOTSUPP: true,
|
|
568
|
+
EBADF: true,
|
|
569
|
+
EPERM: true
|
|
570
|
+
};
|
|
571
|
+
async function syncFile(handle) {
|
|
572
|
+
try {
|
|
573
|
+
await handle.sync();
|
|
574
|
+
} catch (cause) {
|
|
575
|
+
if (IGNORED_SYNC_ERROR_CODE[cause.code ?? ""] !== true) throw cause;
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
const ATOMIC_TEMP_SUFFIX = /^[1-9][0-9]*-[0-9a-f]{36}$/u;
|
|
579
|
+
function atomicTemporaryPrefix(name) {
|
|
580
|
+
return `.${name}.tmp-`;
|
|
581
|
+
}
|
|
582
|
+
function atomicTemporaryName(name, entropy) {
|
|
583
|
+
return `${atomicTemporaryPrefix(name)}${process.pid}-${entropy.toString("hex")}`;
|
|
584
|
+
}
|
|
585
|
+
/** Recognize only names produced by this writer for the specified target. */
|
|
586
|
+
function isAtomicTemporaryName(candidate, name) {
|
|
587
|
+
const prefix = atomicTemporaryPrefix(name);
|
|
588
|
+
return candidate.startsWith(prefix) && ATOMIC_TEMP_SUFFIX.test(candidate.slice(prefix.length));
|
|
589
|
+
}
|
|
590
|
+
/** Publish exact bytes within one pinned parent. No failure is reported after rename commits. */
|
|
591
|
+
async function atomicWriteFile(directory, name, bytes, options = {}) {
|
|
592
|
+
validateComponent(name);
|
|
593
|
+
const operations = {
|
|
594
|
+
open,
|
|
595
|
+
rename,
|
|
596
|
+
unlink,
|
|
597
|
+
randomBytes,
|
|
598
|
+
...options.operations
|
|
599
|
+
};
|
|
600
|
+
const signal = options.signal;
|
|
601
|
+
await directory.withHandle(async (parent, alias) => {
|
|
602
|
+
throwIfAborted(signal);
|
|
603
|
+
const target = `${alias}/${name}`;
|
|
604
|
+
const temporaryName = atomicTemporaryName(name, operations.randomBytes(18));
|
|
605
|
+
validateComponent(temporaryName);
|
|
606
|
+
const temporary = `${alias}/${temporaryName}`;
|
|
607
|
+
let handle;
|
|
608
|
+
let ownedIdentity;
|
|
609
|
+
let committed = false;
|
|
610
|
+
const rejectUnsafeTarget = async () => {
|
|
611
|
+
const stat = await directory.inspect(name, signal);
|
|
612
|
+
if (stat && !stat.isFile()) throw unsafeFilesystem("Atomic target is not a regular file.");
|
|
613
|
+
};
|
|
614
|
+
try {
|
|
615
|
+
await rejectUnsafeTarget();
|
|
616
|
+
throwIfAborted(signal);
|
|
617
|
+
handle = await operations.open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, options.mode ?? 384);
|
|
618
|
+
ownedIdentity = await handle.stat({ bigint: true });
|
|
619
|
+
throwIfAborted(signal);
|
|
620
|
+
await handle.writeFile(bytes);
|
|
621
|
+
throwIfAborted(signal);
|
|
622
|
+
await syncFile(handle);
|
|
623
|
+
throwIfAborted(signal);
|
|
624
|
+
await handle.close();
|
|
625
|
+
handle = void 0;
|
|
626
|
+
await rejectUnsafeTarget();
|
|
627
|
+
const temporaryStat = await directory.inspect(temporaryName, signal);
|
|
628
|
+
if (!temporaryStat?.isFile() || temporaryStat.dev !== ownedIdentity.dev || temporaryStat.ino !== ownedIdentity.ino) throw unsafeFilesystem("Atomic temporary file was replaced.");
|
|
629
|
+
throwIfAborted(signal);
|
|
630
|
+
await operations.rename(temporary, target);
|
|
631
|
+
committed = true;
|
|
632
|
+
directory.markCommitted();
|
|
633
|
+
} catch (cause) {
|
|
634
|
+
if (handle) try {
|
|
635
|
+
await handle.close();
|
|
636
|
+
} catch {}
|
|
637
|
+
if (!committed && ownedIdentity) try {
|
|
638
|
+
const stat = await directory.inspect(temporaryName);
|
|
639
|
+
if (stat?.dev === ownedIdentity.dev && stat.ino === ownedIdentity.ino) await operations.unlink(temporary);
|
|
640
|
+
} catch {}
|
|
641
|
+
throw cause;
|
|
642
|
+
}
|
|
643
|
+
try {
|
|
644
|
+
await syncFile(parent);
|
|
645
|
+
} catch {}
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
//#endregion
|
|
649
|
+
//#region src/ids.ts
|
|
650
|
+
const SOURCE_ID_PATTERN = /^[0-9a-f]{64}$/u;
|
|
651
|
+
const CONTROL_CHARACTER_PATTERN = /[\u0000-\u001f\u007f]/u;
|
|
652
|
+
const WINDOWS_DRIVE_PATTERN = /^[a-zA-Z]:/u;
|
|
653
|
+
function sourceId(value) {
|
|
654
|
+
if (!SOURCE_ID_PATTERN.test(value)) throw invalidPath("Source ID must be exactly 64 lowercase hexadecimal characters.");
|
|
655
|
+
return value;
|
|
656
|
+
}
|
|
657
|
+
function pageId(value) {
|
|
658
|
+
if (value.length === 0) throw invalidPath("Page ID must not be empty.");
|
|
659
|
+
if (value.startsWith("/") || value.startsWith("//") || WINDOWS_DRIVE_PATTERN.test(value)) throw invalidPath("Page ID must be a relative POSIX path.");
|
|
660
|
+
if (value.includes("\\")) throw invalidPath("Page ID must use POSIX separators.");
|
|
661
|
+
if (CONTROL_CHARACTER_PATTERN.test(value)) throw invalidPath("Page ID must not contain control characters.");
|
|
662
|
+
if (value.includes("%")) throw invalidPath("Page ID must not contain percent-encoded or ambiguous percent characters.");
|
|
663
|
+
if (value.toLowerCase().endsWith(".md")) throw invalidPath("Page ID must not include the .md extension.");
|
|
664
|
+
const segments = value.split("/");
|
|
665
|
+
if (segments.some((segment) => segment.length === 0 || segment === "." || segment === "..")) throw invalidPath("Page ID must not contain empty, current-directory, or parent-directory segments.");
|
|
666
|
+
return segments.join("/");
|
|
667
|
+
}
|
|
668
|
+
function isSourceId(value) {
|
|
669
|
+
return SOURCE_ID_PATTERN.test(value);
|
|
670
|
+
}
|
|
671
|
+
function isPageId(value) {
|
|
672
|
+
try {
|
|
673
|
+
pageId(value);
|
|
674
|
+
return true;
|
|
675
|
+
} catch {
|
|
676
|
+
return false;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
//#endregion
|
|
680
|
+
//#region src/markdown.ts
|
|
681
|
+
const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true });
|
|
682
|
+
const UTF8_ENCODER = new TextEncoder();
|
|
683
|
+
const FRONTMATTER_DELIMITER = "---";
|
|
684
|
+
const KEY_LINE = /^([A-Za-z][A-Za-z0-9_-]*):(.*)$/u;
|
|
685
|
+
const SOURCE_ITEM_LINE = /^ - (.+)$/u;
|
|
686
|
+
const ATX_HEADING = /^(#{1,6})(?:[ \t]+(.*?)[ \t]*|[ \t]*)$/u;
|
|
687
|
+
const FENCE_OPEN$1 = /^[ \t]{0,3}(`{3,}|~{3,})(.*)$/u;
|
|
688
|
+
function invalidPage(message, cause) {
|
|
689
|
+
return new LlmWikiError("INVALID_PAGE", message, cause === void 0 ? void 0 : { cause });
|
|
690
|
+
}
|
|
691
|
+
function encodeUtf8(text) {
|
|
692
|
+
return UTF8_ENCODER.encode(text);
|
|
693
|
+
}
|
|
694
|
+
function decodeUtf8(bytes) {
|
|
695
|
+
try {
|
|
696
|
+
return UTF8_DECODER.decode(bytes);
|
|
697
|
+
} catch (cause) {
|
|
698
|
+
throw invalidPage("Wiki content is not valid UTF-8.", cause);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
function normalizeLineEndings(value) {
|
|
702
|
+
return value.replaceAll("\r\n", "\n").replaceAll("\r", "\n");
|
|
703
|
+
}
|
|
704
|
+
function parseQuotedString(value, field) {
|
|
705
|
+
if (!value.startsWith("\"") || !value.endsWith("\"")) throw invalidPage(`Frontmatter ${field} must be a double-quoted string.`);
|
|
706
|
+
let parsed;
|
|
707
|
+
try {
|
|
708
|
+
parsed = JSON.parse(value);
|
|
709
|
+
} catch (cause) {
|
|
710
|
+
throw invalidPage(`Frontmatter ${field} contains an invalid quoted string.`, cause);
|
|
711
|
+
}
|
|
712
|
+
if (typeof parsed !== "string" || parsed.includes("\n") || parsed.includes("\r") || parsed.includes("\u2028") || parsed.includes("\u2029")) throw invalidPage(`Frontmatter ${field} must be a single-line string.`);
|
|
713
|
+
if (parsed.trim().length === 0) throw invalidPage(`Frontmatter ${field} must not be empty.`);
|
|
714
|
+
return parsed;
|
|
715
|
+
}
|
|
716
|
+
function parseSourceItem(value) {
|
|
717
|
+
if (!value.startsWith("\"") || !value.endsWith("\"")) throw invalidPage("Frontmatter source IDs must be double-quoted strings.");
|
|
718
|
+
let parsed;
|
|
719
|
+
try {
|
|
720
|
+
parsed = JSON.parse(value);
|
|
721
|
+
} catch (cause) {
|
|
722
|
+
throw invalidPage("Frontmatter contains an invalid quoted source ID.", cause);
|
|
723
|
+
}
|
|
724
|
+
if (typeof parsed !== "string") throw invalidPage("Frontmatter source IDs must be strings.");
|
|
725
|
+
try {
|
|
726
|
+
return sourceId(parsed);
|
|
727
|
+
} catch (cause) {
|
|
728
|
+
throw invalidPage("Frontmatter contains an invalid source ID.", cause);
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
function parseFrontmatter(lines) {
|
|
732
|
+
const values = {};
|
|
733
|
+
const seen = /* @__PURE__ */ new Set();
|
|
734
|
+
for (let index = 0; index < lines.length;) {
|
|
735
|
+
const line = lines[index];
|
|
736
|
+
if (line === void 0 || line.trim().length === 0) throw invalidPage("Frontmatter must not contain blank lines.");
|
|
737
|
+
const match = KEY_LINE.exec(line);
|
|
738
|
+
if (match === null) throw invalidPage("Frontmatter contains malformed YAML.");
|
|
739
|
+
const key = match[1];
|
|
740
|
+
const rawValue = match[2] ?? "";
|
|
741
|
+
if (key !== "title" && key !== "summary" && key !== "sources") throw invalidPage(`Frontmatter contains unknown key ${JSON.stringify(key)}.`);
|
|
742
|
+
if (seen.has(key)) throw invalidPage(`Frontmatter contains duplicate key ${JSON.stringify(key)}.`);
|
|
743
|
+
seen.add(key);
|
|
744
|
+
if (key === "sources") {
|
|
745
|
+
if (rawValue.length !== 0) throw invalidPage("Frontmatter sources must use a block list.");
|
|
746
|
+
const sources = [];
|
|
747
|
+
index += 1;
|
|
748
|
+
while (index < lines.length) {
|
|
749
|
+
const sourceLine = lines[index];
|
|
750
|
+
if (sourceLine === void 0) break;
|
|
751
|
+
const item = SOURCE_ITEM_LINE.exec(sourceLine);
|
|
752
|
+
if (item === null) break;
|
|
753
|
+
sources.push(parseSourceItem(item[1] ?? ""));
|
|
754
|
+
index += 1;
|
|
755
|
+
}
|
|
756
|
+
if (sources.length === 0) throw invalidPage("Frontmatter sources must not be empty.");
|
|
757
|
+
for (let sourceIndex = 1; sourceIndex < sources.length; sourceIndex += 1) {
|
|
758
|
+
const previous = sources[sourceIndex - 1];
|
|
759
|
+
const current = sources[sourceIndex];
|
|
760
|
+
if (previous === void 0 || current === void 0) throw invalidPage("Frontmatter source list is malformed.");
|
|
761
|
+
if (previous === current) throw invalidPage("Frontmatter source IDs must be unique.");
|
|
762
|
+
if (previous > current) throw invalidPage("Frontmatter source IDs must be sorted.");
|
|
763
|
+
}
|
|
764
|
+
values.sources = sources;
|
|
765
|
+
continue;
|
|
766
|
+
}
|
|
767
|
+
if (!rawValue.startsWith(" ")) throw invalidPage(`Frontmatter ${key} must have one value separator.`);
|
|
768
|
+
values[key] = parseQuotedString(rawValue.slice(1), key);
|
|
769
|
+
index += 1;
|
|
770
|
+
}
|
|
771
|
+
if (values.title === void 0 || values.summary === void 0 || values.sources === void 0) throw invalidPage("Frontmatter must contain title, summary, and sources.");
|
|
772
|
+
return {
|
|
773
|
+
title: values.title,
|
|
774
|
+
summary: values.summary,
|
|
775
|
+
sources: values.sources
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
function parsePageMarkdown(markdown) {
|
|
779
|
+
const lines = normalizeLineEndings(markdown).split("\n");
|
|
780
|
+
if (lines[0] !== FRONTMATTER_DELIMITER) throw invalidPage("Page must begin with YAML frontmatter.");
|
|
781
|
+
const closingIndex = lines.indexOf(FRONTMATTER_DELIMITER, 1);
|
|
782
|
+
if (closingIndex < 0) throw invalidPage("Page frontmatter is not closed.");
|
|
783
|
+
if (lines[closingIndex + 1] !== "") throw invalidPage("Page must contain one blank line after frontmatter.");
|
|
784
|
+
const metadata = parseFrontmatter(lines.slice(1, closingIndex));
|
|
785
|
+
const body = lines.slice(closingIndex + 2).join("\n").replace(/\n+$/u, "");
|
|
786
|
+
if (body.trim().length === 0) throw invalidPage("Page Markdown body must not be empty.");
|
|
787
|
+
return {
|
|
788
|
+
metadata,
|
|
789
|
+
body: `${body}\n`,
|
|
790
|
+
bodyStartLine: closingIndex + 3
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
function renderPageMarkdown(metadata, body) {
|
|
794
|
+
const title = parseQuotedString(JSON.stringify(metadata.title), "title");
|
|
795
|
+
const summary = parseQuotedString(JSON.stringify(metadata.summary), "summary");
|
|
796
|
+
if (metadata.sources.length === 0) throw invalidPage("Frontmatter sources must not be empty.");
|
|
797
|
+
const sources = metadata.sources.map((value) => {
|
|
798
|
+
try {
|
|
799
|
+
return sourceId(value);
|
|
800
|
+
} catch (cause) {
|
|
801
|
+
throw invalidPage("Frontmatter contains an invalid source ID.", cause);
|
|
802
|
+
}
|
|
803
|
+
}).sort();
|
|
804
|
+
if (new Set(sources).size !== sources.length) throw invalidPage("Frontmatter source IDs must be unique.");
|
|
805
|
+
const normalizedBody = normalizeLineEndings(body).replace(/\n+$/u, "");
|
|
806
|
+
if (normalizedBody.trim().length === 0) throw invalidPage("Page Markdown body must not be empty.");
|
|
807
|
+
const sourceLines = sources.map((value) => ` - ${JSON.stringify(value)}`).join("\n");
|
|
808
|
+
return `---\ntitle: ${JSON.stringify(title)}\nsummary: ${JSON.stringify(summary)}\nsources:\n${sourceLines}\n---\n\n${normalizedBody}\n`;
|
|
809
|
+
}
|
|
810
|
+
function splitMarkdownSections(body, firstLine = 1) {
|
|
811
|
+
if (!Number.isSafeInteger(firstLine) || firstLine < 1) throw invalidPage("Section start line must be a positive integer.");
|
|
812
|
+
const lines = normalizeLineEndings(body).replace(/\n+$/u, "").split("\n");
|
|
813
|
+
const sections = [];
|
|
814
|
+
const headingTrail = [];
|
|
815
|
+
let sectionStart = firstLine;
|
|
816
|
+
let sectionLines = [];
|
|
817
|
+
let fenceCharacter;
|
|
818
|
+
let fenceLength = 0;
|
|
819
|
+
const flush = () => {
|
|
820
|
+
const text = sectionLines.join("\n").trimEnd();
|
|
821
|
+
if (text.trim().length > 0 || headingTrail.length > 0) sections.push({
|
|
822
|
+
headingTrail: headingTrail.map(({ title }) => title),
|
|
823
|
+
startLine: sectionStart,
|
|
824
|
+
text
|
|
825
|
+
});
|
|
826
|
+
sectionLines = [];
|
|
827
|
+
};
|
|
828
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
829
|
+
const line = lines[index] ?? "";
|
|
830
|
+
if (fenceCharacter !== void 0) {
|
|
831
|
+
sectionLines.push(line);
|
|
832
|
+
if (new RegExp(`^[ \\t]{0,3}${fenceCharacter === "`" ? "`" : "~"}{${fenceLength},}[ \\t]*$`, "u").test(line)) fenceCharacter = void 0;
|
|
833
|
+
continue;
|
|
834
|
+
}
|
|
835
|
+
const fence = FENCE_OPEN$1.exec(line);
|
|
836
|
+
if (fence !== null) {
|
|
837
|
+
const marker = fence[1] ?? "";
|
|
838
|
+
fenceCharacter = marker[0];
|
|
839
|
+
fenceLength = marker.length;
|
|
840
|
+
sectionLines.push(line);
|
|
841
|
+
continue;
|
|
842
|
+
}
|
|
843
|
+
const heading = ATX_HEADING.exec(line);
|
|
844
|
+
if (heading === null) {
|
|
845
|
+
sectionLines.push(line);
|
|
846
|
+
continue;
|
|
847
|
+
}
|
|
848
|
+
flush();
|
|
849
|
+
const level = (heading[1] ?? "").length;
|
|
850
|
+
const rawHeading = (heading[2] ?? "").replace(/[ \t]+#+[ \t]*$/u, "").trim();
|
|
851
|
+
for (let ancestor = headingTrail.at(-1); ancestor !== void 0 && ancestor.level >= level; ancestor = headingTrail.at(-1)) headingTrail.pop();
|
|
852
|
+
headingTrail.push({
|
|
853
|
+
level,
|
|
854
|
+
title: rawHeading
|
|
855
|
+
});
|
|
856
|
+
sectionStart = firstLine + index;
|
|
857
|
+
}
|
|
858
|
+
flush();
|
|
859
|
+
return sections;
|
|
860
|
+
}
|
|
861
|
+
//#endregion
|
|
862
|
+
//#region src/tokenizer.ts
|
|
863
|
+
const TOKEN_RUN = /[\p{L}\p{N}]+/gu;
|
|
864
|
+
const CJK_CHARACTER = /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]$/u;
|
|
865
|
+
/**
|
|
866
|
+
* Tokenizes normalized text deterministically. CJK-only runs are retained as a
|
|
867
|
+
* token and, when longer than one code point, additionally produce overlapping
|
|
868
|
+
* two-code-point grams. Mixed/non-CJK letter-number runs are emitted whole.
|
|
869
|
+
*/
|
|
870
|
+
function tokenize(value) {
|
|
871
|
+
const tokens = [];
|
|
872
|
+
visitNormalizedTokenSpans(value.normalize("NFKC").toLowerCase(), (term) => {
|
|
873
|
+
tokens.push(term);
|
|
874
|
+
});
|
|
875
|
+
return tokens;
|
|
876
|
+
}
|
|
877
|
+
/** Visits existing tokens in normalized text; offsets are UTF-16 slice boundaries. */
|
|
878
|
+
function visitNormalizedTokenSpans(text, visit) {
|
|
879
|
+
for (const match of text.matchAll(TOKEN_RUN)) {
|
|
880
|
+
const run = match[0];
|
|
881
|
+
const start = match.index;
|
|
882
|
+
visit(run, start, start + run.length);
|
|
883
|
+
const characters = Array.from(run);
|
|
884
|
+
if (characters.length > 1 && characters.every((character) => CJK_CHARACTER.test(character))) {
|
|
885
|
+
let offset = start;
|
|
886
|
+
for (let index = 0; index + 1 < characters.length; index += 1) {
|
|
887
|
+
const gram = `${characters[index]}${characters[index + 1]}`;
|
|
888
|
+
visit(gram, offset, offset + gram.length);
|
|
889
|
+
offset += characters[index].length;
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
const HASH_PATTERN = /^[0-9a-f]{64}$/u;
|
|
895
|
+
const K1 = 1.2;
|
|
896
|
+
const B = .75;
|
|
897
|
+
const BOOST_TITLE = 2;
|
|
898
|
+
const BOOST_HEADING = 1.5;
|
|
899
|
+
const BOOST_BODY = 1;
|
|
900
|
+
const codeUnitCompare = (left, right) => left < right ? -1 : left > right ? 1 : 0;
|
|
901
|
+
function sha256$1(bytes) {
|
|
902
|
+
const digest = createHash("sha256");
|
|
903
|
+
digest.update(bytes);
|
|
904
|
+
return digest.digest("hex");
|
|
905
|
+
}
|
|
906
|
+
const canonicalBytes = (value) => encodeUtf8(`${JSON.stringify(value, null, 2)}\n`);
|
|
907
|
+
function corrupt(message, cause) {
|
|
908
|
+
return new LlmWikiError("INDEX_CORRUPT", message, cause === void 0 ? void 0 : { cause });
|
|
909
|
+
}
|
|
910
|
+
function exactObject(value, keys, name) {
|
|
911
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw corrupt(`${name} must be an object.`);
|
|
912
|
+
const object = value;
|
|
913
|
+
const actual = Object.keys(object);
|
|
914
|
+
if (actual.length !== keys.length || actual.some((key) => !keys.includes(key))) throw corrupt(`${name} contains missing or unknown fields.`);
|
|
915
|
+
return object;
|
|
916
|
+
}
|
|
917
|
+
function safeInteger(value, name, positive = false) {
|
|
918
|
+
if (!Number.isSafeInteger(value) || value < (positive ? 1 : 0)) throw corrupt(`${name} must be a ${positive ? "positive" : "non-negative"} safe integer.`);
|
|
919
|
+
return value;
|
|
920
|
+
}
|
|
921
|
+
function finiteAverage(value) {
|
|
922
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw corrupt("averageSectionLength must be finite and non-negative.");
|
|
923
|
+
return value;
|
|
924
|
+
}
|
|
925
|
+
function stringValue(value, name) {
|
|
926
|
+
if (typeof value !== "string") throw corrupt(`${name} must be a string.`);
|
|
927
|
+
return value;
|
|
928
|
+
}
|
|
929
|
+
function parseFingerprints(value, name) {
|
|
930
|
+
if (!Array.isArray(value)) throw corrupt(`${name} must be an array.`);
|
|
931
|
+
let previous;
|
|
932
|
+
return value.map((entry, index) => {
|
|
933
|
+
const object = exactObject(entry, ["pageId", "sha256"], `${name}[${index}]`);
|
|
934
|
+
const id = stringValue(object.pageId, `${name}[${index}].pageId`);
|
|
935
|
+
pageId(id);
|
|
936
|
+
const hash = stringValue(object.sha256, `${name}[${index}].sha256`);
|
|
937
|
+
if (!HASH_PATTERN.test(hash)) throw corrupt(`${name}[${index}].sha256 is invalid.`);
|
|
938
|
+
if (previous !== void 0 && codeUnitCompare(previous, id) >= 0) throw corrupt(`${name} must be uniquely sorted by pageId.`);
|
|
939
|
+
previous = id;
|
|
940
|
+
return {
|
|
941
|
+
pageId: id,
|
|
942
|
+
sha256: hash
|
|
943
|
+
};
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
function parseTermCounts(value, name) {
|
|
947
|
+
if (!Array.isArray(value)) throw corrupt(`${name} must be an array.`);
|
|
948
|
+
let previous;
|
|
949
|
+
return value.map((entry, index) => {
|
|
950
|
+
const object = exactObject(entry, ["term", "count"], `${name}[${index}]`);
|
|
951
|
+
const term = stringValue(object.term, `${name}[${index}].term`);
|
|
952
|
+
if (term.length === 0 || tokenize(term).length === 0) throw corrupt(`${name}[${index}].term is invalid.`);
|
|
953
|
+
if (previous !== void 0 && codeUnitCompare(previous, term) >= 0) throw corrupt(`${name} must be uniquely sorted by term.`);
|
|
954
|
+
previous = term;
|
|
955
|
+
return {
|
|
956
|
+
term,
|
|
957
|
+
count: safeInteger(object.count, `${name}[${index}].count`, true)
|
|
958
|
+
};
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
function stringArray(value, name) {
|
|
962
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) throw corrupt(`${name} must be a string array.`);
|
|
963
|
+
return value;
|
|
964
|
+
}
|
|
965
|
+
function parseSearchIndex(bytes) {
|
|
966
|
+
let value;
|
|
967
|
+
try {
|
|
968
|
+
value = JSON.parse(decodeUtf8(bytes));
|
|
969
|
+
} catch (cause) {
|
|
970
|
+
throw corrupt("search.json is malformed.", cause);
|
|
971
|
+
}
|
|
972
|
+
const root = exactObject(value, [
|
|
973
|
+
"formatVersion",
|
|
974
|
+
"pageFingerprints",
|
|
975
|
+
"documentCount",
|
|
976
|
+
"averageSectionLength",
|
|
977
|
+
"documentFrequencies",
|
|
978
|
+
"sections"
|
|
979
|
+
], "search.json");
|
|
980
|
+
if (root.formatVersion !== 1) throw corrupt("search.json has an incompatible format version.");
|
|
981
|
+
const pageFingerprints = parseFingerprints(root.pageFingerprints, "pageFingerprints");
|
|
982
|
+
const documentCount = safeInteger(root.documentCount, "documentCount");
|
|
983
|
+
const averageSectionLength = finiteAverage(root.averageSectionLength);
|
|
984
|
+
const documentFrequencies = parseTermCounts(root.documentFrequencies, "documentFrequencies");
|
|
985
|
+
if (!Array.isArray(root.sections)) throw corrupt("sections must be an array.");
|
|
986
|
+
let previousPage = "";
|
|
987
|
+
let previousLine = 0;
|
|
988
|
+
const sections = root.sections.map((entry, index) => {
|
|
989
|
+
const object = exactObject(entry, [
|
|
990
|
+
"pageId",
|
|
991
|
+
"title",
|
|
992
|
+
"headingTrail",
|
|
993
|
+
"startLine",
|
|
994
|
+
"sourceIds",
|
|
995
|
+
"normalizedText",
|
|
996
|
+
"length",
|
|
997
|
+
"titleTermFrequencies",
|
|
998
|
+
"headingTermFrequencies",
|
|
999
|
+
"bodyTermFrequencies"
|
|
1000
|
+
], `sections[${index}]`);
|
|
1001
|
+
const id = stringValue(object.pageId, `sections[${index}].pageId`);
|
|
1002
|
+
pageId(id);
|
|
1003
|
+
const startLine = safeInteger(object.startLine, `sections[${index}].startLine`, true);
|
|
1004
|
+
if (codeUnitCompare(previousPage, id) > 0 || previousPage === id && previousLine >= startLine) throw corrupt("sections must be uniquely sorted by pageId and startLine.");
|
|
1005
|
+
previousPage = id;
|
|
1006
|
+
previousLine = startLine;
|
|
1007
|
+
const sourceIds = stringArray(object.sourceIds, `sections[${index}].sourceIds`);
|
|
1008
|
+
let priorSource;
|
|
1009
|
+
for (const idValue of sourceIds) {
|
|
1010
|
+
sourceId(idValue);
|
|
1011
|
+
if (priorSource !== void 0 && codeUnitCompare(priorSource, idValue) >= 0) throw corrupt("section sourceIds must be uniquely sorted.");
|
|
1012
|
+
priorSource = idValue;
|
|
1013
|
+
}
|
|
1014
|
+
return {
|
|
1015
|
+
pageId: id,
|
|
1016
|
+
title: stringValue(object.title, `sections[${index}].title`),
|
|
1017
|
+
headingTrail: stringArray(object.headingTrail, `sections[${index}].headingTrail`),
|
|
1018
|
+
startLine,
|
|
1019
|
+
sourceIds,
|
|
1020
|
+
normalizedText: stringValue(object.normalizedText, `sections[${index}].normalizedText`),
|
|
1021
|
+
length: safeInteger(object.length, `sections[${index}].length`),
|
|
1022
|
+
titleTermFrequencies: parseTermCounts(object.titleTermFrequencies, `sections[${index}].titleTermFrequencies`),
|
|
1023
|
+
headingTermFrequencies: parseTermCounts(object.headingTermFrequencies, `sections[${index}].headingTermFrequencies`),
|
|
1024
|
+
bodyTermFrequencies: parseTermCounts(object.bodyTermFrequencies, `sections[${index}].bodyTermFrequencies`)
|
|
1025
|
+
};
|
|
1026
|
+
});
|
|
1027
|
+
if (documentCount !== sections.length) throw corrupt("documentCount does not match sections.");
|
|
1028
|
+
return {
|
|
1029
|
+
formatVersion: 1,
|
|
1030
|
+
pageFingerprints,
|
|
1031
|
+
documentCount,
|
|
1032
|
+
averageSectionLength,
|
|
1033
|
+
documentFrequencies,
|
|
1034
|
+
sections
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
function parseIndexState(bytes) {
|
|
1038
|
+
let value;
|
|
1039
|
+
try {
|
|
1040
|
+
value = JSON.parse(decodeUtf8(bytes));
|
|
1041
|
+
} catch (cause) {
|
|
1042
|
+
throw corrupt("state.json is malformed.", cause);
|
|
1043
|
+
}
|
|
1044
|
+
const root = exactObject(value, [
|
|
1045
|
+
"formatVersion",
|
|
1046
|
+
"pages",
|
|
1047
|
+
"searchSha256"
|
|
1048
|
+
], "state.json");
|
|
1049
|
+
if (root.formatVersion !== 1) throw corrupt("state.json has an incompatible format version.");
|
|
1050
|
+
const searchHash = stringValue(root.searchSha256, "searchSha256");
|
|
1051
|
+
if (!HASH_PATTERN.test(searchHash)) throw corrupt("searchSha256 is invalid.");
|
|
1052
|
+
return {
|
|
1053
|
+
formatVersion: 1,
|
|
1054
|
+
pages: parseFingerprints(root.pages, "pages"),
|
|
1055
|
+
searchSha256: searchHash
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1058
|
+
async function pinned(paths, signal, root, callback) {
|
|
1059
|
+
if (root !== void 0) return callback(root);
|
|
1060
|
+
const result = await withWikiRoot(paths.authority, { signal }, callback);
|
|
1061
|
+
if (result === null) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Wiki root is missing.");
|
|
1062
|
+
return result;
|
|
1063
|
+
}
|
|
1064
|
+
async function discoverDirectory(directory, pagesRoot, paths, root, signal) {
|
|
1065
|
+
const result = await root.directory(wikiRelativeSegments(paths, directory), { signal }, async (held) => {
|
|
1066
|
+
const pages = [];
|
|
1067
|
+
for (const entry of await held.list(signal)) {
|
|
1068
|
+
throwIfAborted(signal);
|
|
1069
|
+
const path = join(directory, entry.name);
|
|
1070
|
+
if (entry.stat.isSymbolicLink()) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Symbolic links are not allowed in the pages tree.");
|
|
1071
|
+
if (entry.stat.isDirectory()) pages.push(...await discoverDirectory(path, pagesRoot, paths, root, signal));
|
|
1072
|
+
else if (entry.stat.isFile() && entry.name.endsWith(".md")) pages.push({
|
|
1073
|
+
...entry,
|
|
1074
|
+
path,
|
|
1075
|
+
key: relative(pagesRoot, path).split(sep).join("/")
|
|
1076
|
+
});
|
|
1077
|
+
if (!await root.validate(entry, signal)) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Wiki page identity changed during discovery.");
|
|
1078
|
+
}
|
|
1079
|
+
return pages.sort((a, b) => codeUnitCompare(a.key, b.key));
|
|
1080
|
+
});
|
|
1081
|
+
if (result === null) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Wiki pages directory is missing.");
|
|
1082
|
+
return result;
|
|
1083
|
+
}
|
|
1084
|
+
function sameStableFileSnapshot(before, after) {
|
|
1085
|
+
return before.dev === after.dev && before.ino === after.ino && before.size === after.size && before.mtimeNs === after.mtimeNs && before.ctimeNs === after.ctimeNs;
|
|
1086
|
+
}
|
|
1087
|
+
const corpusSnapshots = /* @__PURE__ */ new WeakMap();
|
|
1088
|
+
function samePathIdentity(snapshot, current) {
|
|
1089
|
+
return current.isFile() && !current.isSymbolicLink() && sameStableFileSnapshot(snapshot, current);
|
|
1090
|
+
}
|
|
1091
|
+
async function validatePageSnapshot(page, root, signal) {
|
|
1092
|
+
if (!await root.validate(page, signal)) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Wiki page changed while the corpus was being read.");
|
|
1093
|
+
}
|
|
1094
|
+
async function validateCorpusSnapshot(snapshot, paths, root, signal) {
|
|
1095
|
+
try {
|
|
1096
|
+
throwIfAborted(signal);
|
|
1097
|
+
const currentPages = await discoverDirectory(paths.pages, paths.pages, paths, root, signal);
|
|
1098
|
+
if (currentPages.length !== snapshot.pages.length) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Wiki page membership changed while the corpus was being read.");
|
|
1099
|
+
for (let index = 0; index < snapshot.pages.length; index += 1) {
|
|
1100
|
+
const expected = snapshot.pages[index];
|
|
1101
|
+
const current = currentPages[index];
|
|
1102
|
+
if (current.key !== expected.key || current.path !== expected.path || !samePathIdentity(expected.stat, current.stat)) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Wiki page changed while the corpus was being read.");
|
|
1103
|
+
}
|
|
1104
|
+
} catch (cause) {
|
|
1105
|
+
throwIfAborted(signal);
|
|
1106
|
+
if (cause instanceof LlmWikiError) throw cause;
|
|
1107
|
+
throw new LlmWikiError("UNSAFE_FILESYSTEM", "Unable to validate a stable wiki corpus.", { cause });
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
async function readSafePage(path, paths, root, signal, onExamined) {
|
|
1111
|
+
try {
|
|
1112
|
+
const segments = wikiRelativeSegments(paths, path);
|
|
1113
|
+
const read = await root.directory(segments.slice(0, -1), { signal }, (parent) => parent.read(segments.at(-1), signal));
|
|
1114
|
+
if (read === null) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Wiki page disappeared while being read.");
|
|
1115
|
+
onExamined?.(path);
|
|
1116
|
+
throwIfAborted(signal);
|
|
1117
|
+
const snapshot = {
|
|
1118
|
+
...read.snapshot,
|
|
1119
|
+
path,
|
|
1120
|
+
key: relative(paths.pages, path).split(sep).join("/")
|
|
1121
|
+
};
|
|
1122
|
+
await validatePageSnapshot(snapshot, root, signal);
|
|
1123
|
+
return {
|
|
1124
|
+
bytes: read.bytes,
|
|
1125
|
+
snapshot
|
|
1126
|
+
};
|
|
1127
|
+
} catch (cause) {
|
|
1128
|
+
throwIfAborted(signal);
|
|
1129
|
+
if (cause instanceof LlmWikiError) throw cause;
|
|
1130
|
+
throw new LlmWikiError("UNSAFE_FILESYSTEM", "Unable to read a wiki page safely.", { cause });
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
function frequencies(tokens) {
|
|
1134
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1135
|
+
for (const token of tokens) counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
1136
|
+
return [...counts].sort(([a], [b]) => codeUnitCompare(a, b)).map(([term, count]) => ({
|
|
1137
|
+
term,
|
|
1138
|
+
count
|
|
1139
|
+
}));
|
|
1140
|
+
}
|
|
1141
|
+
function normalizeText(text) {
|
|
1142
|
+
return text.normalize("NFKC").toLowerCase();
|
|
1143
|
+
}
|
|
1144
|
+
function buildSearchIndexFromPages(pages) {
|
|
1145
|
+
const pageFingerprints = pages.map(({ pageId, bytes }) => ({
|
|
1146
|
+
pageId,
|
|
1147
|
+
sha256: sha256$1(bytes)
|
|
1148
|
+
})).sort((a, b) => codeUnitCompare(a.pageId, b.pageId));
|
|
1149
|
+
const pagesById = new Map(pages.map((page) => [page.pageId, page]));
|
|
1150
|
+
const sections = [];
|
|
1151
|
+
const documentFrequency = /* @__PURE__ */ new Map();
|
|
1152
|
+
for (const fingerprint of pageFingerprints) {
|
|
1153
|
+
const page = pagesById.get(fingerprint.pageId);
|
|
1154
|
+
for (const section of splitMarkdownSections(page.body, page.bodyStartLine)) {
|
|
1155
|
+
const titleTokens = tokenize(page.title);
|
|
1156
|
+
const headingTokens = tokenize(section.headingTrail.join(" "));
|
|
1157
|
+
const bodyTokens = tokenize(section.text);
|
|
1158
|
+
const sectionTokens = [...headingTokens, ...bodyTokens];
|
|
1159
|
+
const indexedTokens = [...titleTokens, ...sectionTokens];
|
|
1160
|
+
for (const term of new Set(indexedTokens)) documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
|
|
1161
|
+
sections.push({
|
|
1162
|
+
pageId: fingerprint.pageId,
|
|
1163
|
+
title: page.title,
|
|
1164
|
+
headingTrail: [...section.headingTrail],
|
|
1165
|
+
startLine: section.startLine,
|
|
1166
|
+
sourceIds: [...page.sourceIds].sort(codeUnitCompare),
|
|
1167
|
+
normalizedText: normalizeText(section.text),
|
|
1168
|
+
length: sectionTokens.length,
|
|
1169
|
+
titleTermFrequencies: frequencies(titleTokens),
|
|
1170
|
+
headingTermFrequencies: frequencies(headingTokens),
|
|
1171
|
+
bodyTermFrequencies: frequencies(bodyTokens)
|
|
1172
|
+
});
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
sections.sort((a, b) => codeUnitCompare(a.pageId, b.pageId) || a.startLine - b.startLine);
|
|
1176
|
+
const documentCount = sections.length;
|
|
1177
|
+
const search = {
|
|
1178
|
+
formatVersion: 1,
|
|
1179
|
+
pageFingerprints,
|
|
1180
|
+
documentCount,
|
|
1181
|
+
averageSectionLength: documentCount === 0 ? 0 : sections.reduce((sum, section) => sum + section.length, 0) / documentCount,
|
|
1182
|
+
documentFrequencies: [...documentFrequency].sort(([a], [b]) => codeUnitCompare(a, b)).map(([term, count]) => ({
|
|
1183
|
+
term,
|
|
1184
|
+
count
|
|
1185
|
+
})),
|
|
1186
|
+
sections
|
|
1187
|
+
};
|
|
1188
|
+
const searchBytes = canonicalBytes(search);
|
|
1189
|
+
const state = {
|
|
1190
|
+
formatVersion: 1,
|
|
1191
|
+
pages: pageFingerprints,
|
|
1192
|
+
searchSha256: sha256$1(searchBytes)
|
|
1193
|
+
};
|
|
1194
|
+
return {
|
|
1195
|
+
state,
|
|
1196
|
+
search,
|
|
1197
|
+
searchBytes,
|
|
1198
|
+
stateBytes: canonicalBytes(state)
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
async function buildSearchIndex(paths, signal, onPageExamined, root) {
|
|
1202
|
+
return pinned(paths, signal, root, async (held) => {
|
|
1203
|
+
const discovered = await discoverDirectory(paths.pages, paths.pages, paths, held, signal);
|
|
1204
|
+
const pages = [];
|
|
1205
|
+
const pageSnapshots = [];
|
|
1206
|
+
for (const page of discovered) {
|
|
1207
|
+
const { bytes, snapshot } = await readSafePage(page.path, paths, held, signal, onPageExamined);
|
|
1208
|
+
pageSnapshots.push(snapshot);
|
|
1209
|
+
const id = pageId(page.key.replace(/\.md$/u, ""));
|
|
1210
|
+
const parsed = parsePageMarkdown(decodeUtf8(bytes));
|
|
1211
|
+
pages.push({
|
|
1212
|
+
pageId: id,
|
|
1213
|
+
bytes,
|
|
1214
|
+
title: parsed.metadata.title,
|
|
1215
|
+
sourceIds: parsed.metadata.sources,
|
|
1216
|
+
body: parsed.body,
|
|
1217
|
+
bodyStartLine: parsed.bodyStartLine
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
const corpusSnapshot = { pages: pageSnapshots };
|
|
1221
|
+
const built = buildSearchIndexFromPages(pages);
|
|
1222
|
+
await validateCorpusSnapshot(corpusSnapshot, paths, held, signal);
|
|
1223
|
+
corpusSnapshots.set(built, corpusSnapshot);
|
|
1224
|
+
return built;
|
|
1225
|
+
});
|
|
1226
|
+
}
|
|
1227
|
+
function trustedSearchIndex(searchBytes, stateBytes, expected) {
|
|
1228
|
+
const search = parseSearchIndex(searchBytes);
|
|
1229
|
+
parseIndexState(stateBytes);
|
|
1230
|
+
const searchMatches = searchBytes.byteLength === expected.searchBytes.byteLength && searchBytes.every((byte, index) => byte === expected.searchBytes[index]);
|
|
1231
|
+
const stateMatches = stateBytes.byteLength === expected.stateBytes.byteLength && stateBytes.every((byte, index) => byte === expected.stateBytes[index]);
|
|
1232
|
+
return searchMatches && stateMatches ? search : null;
|
|
1233
|
+
}
|
|
1234
|
+
async function validateBuiltIndexSnapshot(paths, built, signal, root) {
|
|
1235
|
+
const snapshot = corpusSnapshots.get(built);
|
|
1236
|
+
if (snapshot !== void 0) await pinned(paths, signal, root, (held) => validateCorpusSnapshot(snapshot, paths, held, signal));
|
|
1237
|
+
}
|
|
1238
|
+
async function writeIndex(paths, built, signal, root) {
|
|
1239
|
+
await pinned(paths, signal, root, async (held) => {
|
|
1240
|
+
await validateBuiltIndexSnapshot(paths, built, signal, held);
|
|
1241
|
+
await held.directory(wikiRelativeSegments(paths, paths.index), {
|
|
1242
|
+
create: true,
|
|
1243
|
+
signal
|
|
1244
|
+
}, async (index) => {
|
|
1245
|
+
await validateBuiltIndexSnapshot(paths, built, signal, held);
|
|
1246
|
+
await atomicWriteFile(index, "search.json", built.searchBytes, { signal });
|
|
1247
|
+
throwIfAborted(signal);
|
|
1248
|
+
await validateBuiltIndexSnapshot(paths, built, signal, held);
|
|
1249
|
+
await atomicWriteFile(index, "state.json", built.stateBytes, { signal });
|
|
1250
|
+
await validateBuiltIndexSnapshot(paths, built, signal, held);
|
|
1251
|
+
});
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
function countFor(items, term) {
|
|
1255
|
+
return items.find((item) => item.term === term)?.count ?? 0;
|
|
1256
|
+
}
|
|
1257
|
+
function utf8Snippet(text, maxBytes) {
|
|
1258
|
+
const lines = text.split("\n");
|
|
1259
|
+
let result = "";
|
|
1260
|
+
for (const line of lines) {
|
|
1261
|
+
const candidate = result.length === 0 ? line : `${result}\n${line}`;
|
|
1262
|
+
if (encodeUtf8(candidate).byteLength <= maxBytes) {
|
|
1263
|
+
result = candidate;
|
|
1264
|
+
continue;
|
|
1265
|
+
}
|
|
1266
|
+
if (result.length > 0) break;
|
|
1267
|
+
for (const character of line) {
|
|
1268
|
+
if (encodeUtf8(result + character).byteLength > maxBytes) break;
|
|
1269
|
+
result += character;
|
|
1270
|
+
}
|
|
1271
|
+
break;
|
|
1272
|
+
}
|
|
1273
|
+
return result;
|
|
1274
|
+
}
|
|
1275
|
+
function querySnippet(section, queryTerms, maxBytes) {
|
|
1276
|
+
const text = section.normalizedText;
|
|
1277
|
+
const terms = new Set(queryTerms.filter((term) => countFor(section.bodyTermFrequencies, term) > 0 && Buffer.byteLength(term, "utf8") <= maxBytes));
|
|
1278
|
+
let selected;
|
|
1279
|
+
if (terms.size > 0) visitNormalizedTokenSpans(text, (term, start, end) => {
|
|
1280
|
+
if (terms.has(term) && (!selected || start < selected.start || start === selected.start && codeUnitCompare(term, selected.term) < 0)) selected = {
|
|
1281
|
+
term,
|
|
1282
|
+
start,
|
|
1283
|
+
end
|
|
1284
|
+
};
|
|
1285
|
+
});
|
|
1286
|
+
if (!selected) return utf8Snippet(text, maxBytes);
|
|
1287
|
+
let start = selected.start;
|
|
1288
|
+
let end = selected.end;
|
|
1289
|
+
let remaining = maxBytes - Buffer.byteLength(selected.term, "utf8");
|
|
1290
|
+
const extendLeft = (budget) => {
|
|
1291
|
+
let used = 0;
|
|
1292
|
+
while (start > 0) {
|
|
1293
|
+
const last = text.charCodeAt(start - 1);
|
|
1294
|
+
const previous = start - (last >= 56320 && last <= 57343 ? 2 : 1);
|
|
1295
|
+
const bytes = Buffer.byteLength(text.slice(previous, start), "utf8");
|
|
1296
|
+
if (used + bytes > budget) break;
|
|
1297
|
+
start = previous;
|
|
1298
|
+
used += bytes;
|
|
1299
|
+
}
|
|
1300
|
+
return used;
|
|
1301
|
+
};
|
|
1302
|
+
remaining -= extendLeft(Math.floor(remaining / 2));
|
|
1303
|
+
while (end < text.length) {
|
|
1304
|
+
const next = end + (text.codePointAt(end) > 65535 ? 2 : 1);
|
|
1305
|
+
const bytes = Buffer.byteLength(text.slice(end, next), "utf8");
|
|
1306
|
+
if (bytes > remaining) break;
|
|
1307
|
+
end = next;
|
|
1308
|
+
remaining -= bytes;
|
|
1309
|
+
}
|
|
1310
|
+
extendLeft(remaining);
|
|
1311
|
+
return text.slice(start, end);
|
|
1312
|
+
}
|
|
1313
|
+
function searchBuiltIndex(index, query, options) {
|
|
1314
|
+
throwIfAborted(options.signal);
|
|
1315
|
+
if (!Number.isSafeInteger(options.limit) || options.limit < 1 || !Number.isSafeInteger(options.maxResults) || options.maxResults < 1 || !Number.isSafeInteger(options.maxSnippetBytes) || options.maxSnippetBytes < 1) throw new LlmWikiError("LIMIT_EXCEEDED", "Search limits must be positive safe integers.");
|
|
1316
|
+
const queryTerms = [...new Set(tokenize(query))];
|
|
1317
|
+
if (queryTerms.length === 0) throw new LlmWikiError("INVALID_PAGE", "Search query must contain at least one Unicode letter or number.");
|
|
1318
|
+
const limit = Math.min(options.limit, options.maxResults);
|
|
1319
|
+
const hits = [];
|
|
1320
|
+
for (const section of index.sections) {
|
|
1321
|
+
throwIfAborted(options.signal);
|
|
1322
|
+
let score = 0;
|
|
1323
|
+
for (const term of queryTerms) {
|
|
1324
|
+
const df = countFor(index.documentFrequencies, term);
|
|
1325
|
+
if (df === 0) continue;
|
|
1326
|
+
const weightedFrequency = BOOST_TITLE * countFor(section.titleTermFrequencies, term) + BOOST_HEADING * countFor(section.headingTermFrequencies, term) + BOOST_BODY * countFor(section.bodyTermFrequencies, term);
|
|
1327
|
+
if (weightedFrequency === 0) continue;
|
|
1328
|
+
const idf = Math.log(1 + (index.documentCount - df + .5) / (df + .5));
|
|
1329
|
+
const normalization = index.averageSectionLength === 0 ? 1 : 1 - B + B * section.length / index.averageSectionLength;
|
|
1330
|
+
score += idf * (weightedFrequency * 2.2) / (weightedFrequency + K1 * normalization);
|
|
1331
|
+
}
|
|
1332
|
+
if (!Number.isFinite(score)) throw corrupt("Search produced a non-finite score.");
|
|
1333
|
+
if (score > 0) hits.push({
|
|
1334
|
+
section,
|
|
1335
|
+
score
|
|
1336
|
+
});
|
|
1337
|
+
}
|
|
1338
|
+
hits.sort((a, b) => b.score - a.score || codeUnitCompare(a.section.pageId, b.section.pageId) || a.section.startLine - b.section.startLine);
|
|
1339
|
+
return hits.slice(0, limit).map(({ section, score }) => ({
|
|
1340
|
+
pageId: pageId(section.pageId),
|
|
1341
|
+
title: section.title,
|
|
1342
|
+
headingTrail: section.headingTrail,
|
|
1343
|
+
startLine: section.startLine,
|
|
1344
|
+
score,
|
|
1345
|
+
snippet: querySnippet(section, queryTerms, options.maxSnippetBytes),
|
|
1346
|
+
sourceIds: section.sourceIds.map(sourceId)
|
|
1347
|
+
}));
|
|
1348
|
+
}
|
|
1349
|
+
//#endregion
|
|
1350
|
+
//#region src/lint.ts
|
|
1351
|
+
const HASH$1 = /^[0-9a-f]{64}$/u;
|
|
1352
|
+
const TEMP_FILE = /^\..+\.tmp-\d+-[0-9a-f]+$/u;
|
|
1353
|
+
const EXTERNAL_SCHEME = /^[A-Za-z][A-Za-z0-9+.-]*:/u;
|
|
1354
|
+
const FENCE_OPEN = /^[ \t]{0,3}(`{3,}|~{3,})/u;
|
|
1355
|
+
const MARKDOWN_LINK = /(?<!!)\[[^\]]*\]\(([^\s)]+)(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\)/gu;
|
|
1356
|
+
const WIKILINK = /\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|[^\]]*)?\]\]/gu;
|
|
1357
|
+
function compareCodeUnits$1(left, right) {
|
|
1358
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
1359
|
+
}
|
|
1360
|
+
function relativePath(paths, path) {
|
|
1361
|
+
const value = relative(paths.root, path).split(sep).join("/");
|
|
1362
|
+
return value.length === 0 ? "." : value;
|
|
1363
|
+
}
|
|
1364
|
+
function diagnostic(context, code, severity, path, message, line) {
|
|
1365
|
+
context.diagnostics.push(line === void 0 ? {
|
|
1366
|
+
code,
|
|
1367
|
+
severity,
|
|
1368
|
+
path: relativePath(context.paths, path),
|
|
1369
|
+
message
|
|
1370
|
+
} : {
|
|
1371
|
+
code,
|
|
1372
|
+
severity,
|
|
1373
|
+
path: relativePath(context.paths, path),
|
|
1374
|
+
line,
|
|
1375
|
+
message
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1378
|
+
function sha256(bytes) {
|
|
1379
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
1380
|
+
}
|
|
1381
|
+
async function stat(path, context) {
|
|
1382
|
+
const segments = wikiRelativeSegments(context.paths, path);
|
|
1383
|
+
if (segments.length === 0) return {
|
|
1384
|
+
isDirectory: () => true,
|
|
1385
|
+
isSymbolicLink: () => false,
|
|
1386
|
+
isFile: () => false
|
|
1387
|
+
};
|
|
1388
|
+
return context.root.directory(segments.slice(0, -1), { signal: context.signal }, (parent) => parent.inspect(segments.at(-1), context.signal));
|
|
1389
|
+
}
|
|
1390
|
+
async function readBytes$1(path, context) {
|
|
1391
|
+
const segments = wikiRelativeSegments(context.paths, path);
|
|
1392
|
+
const read = await context.root.directory(segments.slice(0, -1), { signal: context.signal }, (parent) => parent.read(segments.at(-1), context.signal));
|
|
1393
|
+
if (read === null) return null;
|
|
1394
|
+
context.examinedPaths.add(path);
|
|
1395
|
+
throwIfAborted(context.signal);
|
|
1396
|
+
return read.bytes;
|
|
1397
|
+
}
|
|
1398
|
+
async function entries(path, context) {
|
|
1399
|
+
const listed = await context.root.directory(wikiRelativeSegments(context.paths, path), { signal: context.signal }, (parent) => parent.list(context.signal));
|
|
1400
|
+
if (listed === null) return [];
|
|
1401
|
+
return listed.map(({ name, stat }) => ({
|
|
1402
|
+
name,
|
|
1403
|
+
isSymbolicLink: () => stat.isSymbolicLink(),
|
|
1404
|
+
isDirectory: () => stat.isDirectory(),
|
|
1405
|
+
isFile: () => stat.isFile()
|
|
1406
|
+
}));
|
|
1407
|
+
}
|
|
1408
|
+
async function inspectRequiredDirectory(path, context) {
|
|
1409
|
+
const info = await stat(path, context);
|
|
1410
|
+
if (info === null) {
|
|
1411
|
+
diagnostic(context, "REQUIRED_DIRECTORY_MISSING", "error", path, "Required wiki directory is missing.");
|
|
1412
|
+
return false;
|
|
1413
|
+
}
|
|
1414
|
+
if (info.isSymbolicLink()) return false;
|
|
1415
|
+
if (!info.isDirectory()) {
|
|
1416
|
+
diagnostic(context, "REQUIRED_PATH_NOT_DIRECTORY", "error", path, "Required wiki path is not a directory.");
|
|
1417
|
+
return false;
|
|
1418
|
+
}
|
|
1419
|
+
return true;
|
|
1420
|
+
}
|
|
1421
|
+
async function scanTempsAndSymlinks(path, context) {
|
|
1422
|
+
const info = await stat(path, context);
|
|
1423
|
+
if (info === null || !info.isDirectory() || info.isSymbolicLink()) return;
|
|
1424
|
+
for (const entry of await entries(path, context)) {
|
|
1425
|
+
const child = join(path, entry.name);
|
|
1426
|
+
if (entry.isSymbolicLink()) {
|
|
1427
|
+
diagnostic(context, "UNSAFE_SYMLINK", "error", child, "Symbolic links are not allowed in the wiki.");
|
|
1428
|
+
continue;
|
|
1429
|
+
}
|
|
1430
|
+
if (entry.isDirectory()) {
|
|
1431
|
+
await scanTempsAndSymlinks(child, context);
|
|
1432
|
+
continue;
|
|
1433
|
+
}
|
|
1434
|
+
if (TEMP_FILE.test(entry.name)) diagnostic(context, "TEMP_FILE_ABANDONED", "warning", child, "Abandoned atomic-write temporary file.");
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
function validSourceMetadata(value) {
|
|
1438
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1439
|
+
const object = value;
|
|
1440
|
+
return typeof object.id === "string" && isSourceId(object.id) && typeof object.name === "string" && object.name.trim().length > 0 && typeof object.mediaType === "string" && object.mediaType.trim().length > 0 && Number.isSafeInteger(object.byteCount) && object.byteCount >= 0 && typeof object.capturedAt === "string" && !Number.isNaN(Date.parse(object.capturedAt)) && new Date(object.capturedAt).toISOString() === object.capturedAt && (object.origin === void 0 || typeof object.origin === "string" && object.origin.trim().length > 0);
|
|
1441
|
+
}
|
|
1442
|
+
async function inspectSources(context) {
|
|
1443
|
+
const validSources = /* @__PURE__ */ new Set();
|
|
1444
|
+
if (!await inspectRequiredDirectory(context.paths.sources, context)) return validSources;
|
|
1445
|
+
for (const entry of await entries(context.paths.sources, context)) {
|
|
1446
|
+
const directory = join(context.paths.sources, entry.name);
|
|
1447
|
+
if (entry.isSymbolicLink()) continue;
|
|
1448
|
+
if (!entry.isDirectory() || !isSourceId(entry.name)) {
|
|
1449
|
+
diagnostic(context, "SOURCE_INVALID_ID", "error", directory, "Source directory name must be a lowercase SHA-256 ID.");
|
|
1450
|
+
continue;
|
|
1451
|
+
}
|
|
1452
|
+
const contentPath = join(directory, "content");
|
|
1453
|
+
const contentStat = await stat(contentPath, context);
|
|
1454
|
+
let content = null;
|
|
1455
|
+
if (contentStat === null) diagnostic(context, "SOURCE_CONTENT_MISSING", "error", contentPath, "Source content is missing.");
|
|
1456
|
+
else if (contentStat.isSymbolicLink()) {} else if (!contentStat.isFile()) diagnostic(context, "SOURCE_CONTENT_NOT_FILE", "error", contentPath, "Source content must be a regular file.");
|
|
1457
|
+
else content = await readBytes$1(contentPath, context);
|
|
1458
|
+
if (content !== null && sha256(content) !== entry.name) diagnostic(context, "SOURCE_HASH_MISMATCH", "error", contentPath, "Source ID does not match the SHA-256 hash of content.");
|
|
1459
|
+
const metadataPath = join(directory, "metadata.json");
|
|
1460
|
+
const metadataStat = await stat(metadataPath, context);
|
|
1461
|
+
let metadata = null;
|
|
1462
|
+
if (metadataStat === null) diagnostic(context, "SOURCE_METADATA_MISSING", "error", metadataPath, "Source metadata is missing.");
|
|
1463
|
+
else if (metadataStat.isSymbolicLink()) {} else if (!metadataStat.isFile()) diagnostic(context, "SOURCE_METADATA_NOT_FILE", "error", metadataPath, "Source metadata must be a regular file.");
|
|
1464
|
+
else {
|
|
1465
|
+
const bytes = await readBytes$1(metadataPath, context);
|
|
1466
|
+
if (bytes !== null) try {
|
|
1467
|
+
const parsed = JSON.parse(decodeUtf8(bytes));
|
|
1468
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
1469
|
+
const allowed = {
|
|
1470
|
+
id: true,
|
|
1471
|
+
name: true,
|
|
1472
|
+
mediaType: true,
|
|
1473
|
+
byteCount: true,
|
|
1474
|
+
capturedAt: true,
|
|
1475
|
+
origin: true
|
|
1476
|
+
};
|
|
1477
|
+
for (const key of Object.keys(parsed).sort(compareCodeUnits$1)) if (allowed[key] !== true) diagnostic(context, "SOURCE_METADATA_UNKNOWN_KEY", "error", metadataPath, `Source metadata contains unknown key ${JSON.stringify(key)}.`);
|
|
1478
|
+
}
|
|
1479
|
+
if (!validSourceMetadata(parsed)) diagnostic(context, "SOURCE_METADATA_INVALID", "error", metadataPath, "Source metadata does not match the required schema.");
|
|
1480
|
+
else metadata = parsed;
|
|
1481
|
+
} catch {
|
|
1482
|
+
diagnostic(context, "SOURCE_METADATA_MALFORMED", "error", metadataPath, "Source metadata is not valid UTF-8 JSON.");
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
if (metadata !== null && metadata.id !== entry.name) diagnostic(context, "SOURCE_METADATA_ID_MISMATCH", "error", metadataPath, "Source metadata ID does not match its directory name.");
|
|
1486
|
+
if (metadata !== null && content !== null && metadata.byteCount !== content.byteLength) diagnostic(context, "SOURCE_METADATA_BYTE_COUNT_MISMATCH", "error", metadataPath, "Source metadata byteCount does not match content bytes.");
|
|
1487
|
+
if (content !== null && sha256(content) === entry.name && metadata !== null && metadata.id === entry.name && metadata.byteCount === content.byteLength) validSources.add(entry.name);
|
|
1488
|
+
}
|
|
1489
|
+
return validSources;
|
|
1490
|
+
}
|
|
1491
|
+
async function collectPageFiles(directory, context, output) {
|
|
1492
|
+
for (const entry of await entries(directory, context)) {
|
|
1493
|
+
const path = join(directory, entry.name);
|
|
1494
|
+
if (entry.isSymbolicLink()) continue;
|
|
1495
|
+
if (entry.isDirectory()) await collectPageFiles(path, context, output);
|
|
1496
|
+
else output.push(path);
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
function linkTargets(body, firstLine) {
|
|
1500
|
+
const result = [];
|
|
1501
|
+
const lines = body.replaceAll("\r\n", "\n").replaceAll("\r", "\n").split("\n");
|
|
1502
|
+
let fence = null;
|
|
1503
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
1504
|
+
const line = lines[index] ?? "";
|
|
1505
|
+
if (fence !== null) {
|
|
1506
|
+
if (new RegExp(`^[ \\t]{0,3}${fence.character === "`" ? "`" : "~"}{${fence.length},}[ \\t]*$`, "u").test(line)) fence = null;
|
|
1507
|
+
continue;
|
|
1508
|
+
}
|
|
1509
|
+
const opening = FENCE_OPEN.exec(line);
|
|
1510
|
+
if (opening !== null) {
|
|
1511
|
+
const marker = opening[1] ?? "";
|
|
1512
|
+
fence = {
|
|
1513
|
+
character: marker[0] ?? "`",
|
|
1514
|
+
length: marker.length
|
|
1515
|
+
};
|
|
1516
|
+
continue;
|
|
1517
|
+
}
|
|
1518
|
+
for (const match of line.matchAll(MARKDOWN_LINK)) result.push({
|
|
1519
|
+
target: match[1] ?? "",
|
|
1520
|
+
line: firstLine + index,
|
|
1521
|
+
wiki: false
|
|
1522
|
+
});
|
|
1523
|
+
for (const match of line.matchAll(WIKILINK)) result.push({
|
|
1524
|
+
target: match[1]?.trim() ?? "",
|
|
1525
|
+
line: firstLine + index,
|
|
1526
|
+
wiki: true
|
|
1527
|
+
});
|
|
1528
|
+
}
|
|
1529
|
+
return result;
|
|
1530
|
+
}
|
|
1531
|
+
function validateLinks(page, pageIds, context) {
|
|
1532
|
+
const linkedPages = /* @__PURE__ */ new Set();
|
|
1533
|
+
const pageDirectory = posix.dirname(page.id);
|
|
1534
|
+
for (const link of linkTargets(page.body, page.bodyStartLine)) {
|
|
1535
|
+
let target = link.target;
|
|
1536
|
+
if (target.length === 0 || target.startsWith("#") || target.startsWith("//") || EXTERNAL_SCHEME.test(target)) continue;
|
|
1537
|
+
target = target.split("#", 1)[0] ?? "";
|
|
1538
|
+
target = target.split("?", 1)[0] ?? "";
|
|
1539
|
+
if (target.length === 0) continue;
|
|
1540
|
+
const hadExtension = target.toLowerCase().endsWith(".md");
|
|
1541
|
+
if (hadExtension) target = target.slice(0, -3);
|
|
1542
|
+
const resolved = posix.normalize(posix.join(pageDirectory, target));
|
|
1543
|
+
if (resolved === ".." || resolved.startsWith("../") || target.startsWith("/") || target.includes("\\")) {
|
|
1544
|
+
diagnostic(context, "LINK_ESCAPES_PAGES", "error", page.path, "Relative page link escapes the pages directory.", link.line);
|
|
1545
|
+
continue;
|
|
1546
|
+
}
|
|
1547
|
+
if (!isPageId(resolved) || !hadExtension && !link.wiki) continue;
|
|
1548
|
+
if (!pageIds.has(resolved)) diagnostic(context, "BROKEN_PAGE_LINK", "error", page.path, `Linked page ${JSON.stringify(resolved)} does not exist.`, link.line);
|
|
1549
|
+
else if (resolved !== page.id) linkedPages.add(resolved);
|
|
1550
|
+
}
|
|
1551
|
+
return linkedPages;
|
|
1552
|
+
}
|
|
1553
|
+
async function inspectPages(validSources, context) {
|
|
1554
|
+
const pages = [];
|
|
1555
|
+
const referencedSources = /* @__PURE__ */ new Set();
|
|
1556
|
+
if (!await inspectRequiredDirectory(context.paths.pages, context)) return referencedSources;
|
|
1557
|
+
const files = [];
|
|
1558
|
+
await collectPageFiles(context.paths.pages, context, files);
|
|
1559
|
+
files.sort(compareCodeUnits$1);
|
|
1560
|
+
for (const path of files) {
|
|
1561
|
+
const logical = relative(context.paths.pages, path).split(sep).join("/");
|
|
1562
|
+
if (extname(logical) !== ".md" || !isPageId(logical.slice(0, -3))) {
|
|
1563
|
+
diagnostic(context, "PAGE_INVALID_PATH", "error", path, "Page path must be a normalized relative path ending in lowercase .md.");
|
|
1564
|
+
continue;
|
|
1565
|
+
}
|
|
1566
|
+
const bytes = await readBytes$1(path, context);
|
|
1567
|
+
if (bytes === null) continue;
|
|
1568
|
+
try {
|
|
1569
|
+
const parsed = parsePageMarkdown(decodeUtf8(bytes));
|
|
1570
|
+
const canonical = Buffer.from(encodeUtf8(renderPageMarkdown(parsed.metadata, parsed.body))).equals(Buffer.from(bytes));
|
|
1571
|
+
if (!canonical) diagnostic(context, "PAGE_INVALID_MARKDOWN", "error", path, "Page is not valid canonical wiki Markdown.");
|
|
1572
|
+
for (const source of parsed.metadata.sources) if (!validSources.has(source)) diagnostic(context, "PAGE_MISSING_SOURCE", "error", path, `Page references missing or invalid source ${JSON.stringify(source)}.`);
|
|
1573
|
+
else if (canonical) referencedSources.add(source);
|
|
1574
|
+
pages.push({
|
|
1575
|
+
id: logical.slice(0, -3),
|
|
1576
|
+
path,
|
|
1577
|
+
bytes,
|
|
1578
|
+
title: parsed.metadata.title,
|
|
1579
|
+
sourceIds: parsed.metadata.sources,
|
|
1580
|
+
body: parsed.body,
|
|
1581
|
+
bodyStartLine: parsed.bodyStartLine
|
|
1582
|
+
});
|
|
1583
|
+
} catch {
|
|
1584
|
+
diagnostic(context, "PAGE_INVALID_MARKDOWN", "error", path, "Page is not valid canonical wiki Markdown.");
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
const titles = /* @__PURE__ */ new Map();
|
|
1588
|
+
for (const page of pages) {
|
|
1589
|
+
const normalized = page.title.normalize("NFKC").toLowerCase();
|
|
1590
|
+
const group = titles.get(normalized) ?? [];
|
|
1591
|
+
group.push(page);
|
|
1592
|
+
titles.set(normalized, group);
|
|
1593
|
+
}
|
|
1594
|
+
for (const group of titles.values()) {
|
|
1595
|
+
if (group.length < 2) continue;
|
|
1596
|
+
for (const page of group) diagnostic(context, "DUPLICATE_TITLE", "warning", page.path, "Page title duplicates another title after Unicode normalization.");
|
|
1597
|
+
}
|
|
1598
|
+
const pageIds = new Set(pages.map(({ id }) => id));
|
|
1599
|
+
const inbound = /* @__PURE__ */ new Set();
|
|
1600
|
+
for (const page of pages) for (const target of validateLinks(page, pageIds, context)) inbound.add(target);
|
|
1601
|
+
if (pages.length > 1) {
|
|
1602
|
+
for (const page of pages) if (!inbound.has(page.id)) diagnostic(context, "ORPHAN_PAGE", "warning", page.path, "Page has no incoming links from another page.");
|
|
1603
|
+
}
|
|
1604
|
+
return referencedSources;
|
|
1605
|
+
}
|
|
1606
|
+
function exactKeys(value, keys) {
|
|
1607
|
+
return Object.keys(value).length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
|
1608
|
+
}
|
|
1609
|
+
function record(value) {
|
|
1610
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1611
|
+
}
|
|
1612
|
+
function safeNonnegativeInteger(value) {
|
|
1613
|
+
return Number.isSafeInteger(value) && value >= 0;
|
|
1614
|
+
}
|
|
1615
|
+
function positiveInteger(value) {
|
|
1616
|
+
return Number.isSafeInteger(value) && value > 0;
|
|
1617
|
+
}
|
|
1618
|
+
function sortedUniqueStrings(value) {
|
|
1619
|
+
if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) return false;
|
|
1620
|
+
let previous;
|
|
1621
|
+
for (const item of value) {
|
|
1622
|
+
if (previous !== void 0 && previous >= item) return false;
|
|
1623
|
+
previous = item;
|
|
1624
|
+
}
|
|
1625
|
+
return true;
|
|
1626
|
+
}
|
|
1627
|
+
function validFrequencyArray(value) {
|
|
1628
|
+
if (!Array.isArray(value)) return false;
|
|
1629
|
+
let previousTerm;
|
|
1630
|
+
for (const item of value) {
|
|
1631
|
+
if (!record(item) || !exactKeys(item, ["term", "count"]) || typeof item.term !== "string" || item.term.length === 0 || !positiveInteger(item.count)) return false;
|
|
1632
|
+
if (previousTerm !== void 0 && previousTerm >= item.term) return false;
|
|
1633
|
+
previousTerm = item.term;
|
|
1634
|
+
}
|
|
1635
|
+
return true;
|
|
1636
|
+
}
|
|
1637
|
+
function validFingerprints(value) {
|
|
1638
|
+
if (!Array.isArray(value)) return false;
|
|
1639
|
+
let previousPageId;
|
|
1640
|
+
for (const item of value) {
|
|
1641
|
+
if (!record(item) || !exactKeys(item, ["pageId", "sha256"]) || typeof item.pageId !== "string" || !isPageId(item.pageId) || typeof item.sha256 !== "string" || !HASH$1.test(item.sha256)) return false;
|
|
1642
|
+
if (previousPageId !== void 0 && previousPageId >= item.pageId) return false;
|
|
1643
|
+
previousPageId = item.pageId;
|
|
1644
|
+
}
|
|
1645
|
+
return true;
|
|
1646
|
+
}
|
|
1647
|
+
function validState(value) {
|
|
1648
|
+
return record(value) && exactKeys(value, [
|
|
1649
|
+
"formatVersion",
|
|
1650
|
+
"pages",
|
|
1651
|
+
"searchSha256"
|
|
1652
|
+
]) && value.formatVersion === 1 && validFingerprints(value.pages) && typeof value.searchSha256 === "string" && HASH$1.test(value.searchSha256);
|
|
1653
|
+
}
|
|
1654
|
+
function validSearch(value) {
|
|
1655
|
+
if (!record(value) || !exactKeys(value, [
|
|
1656
|
+
"formatVersion",
|
|
1657
|
+
"pageFingerprints",
|
|
1658
|
+
"documentCount",
|
|
1659
|
+
"averageSectionLength",
|
|
1660
|
+
"documentFrequencies",
|
|
1661
|
+
"sections"
|
|
1662
|
+
]) || value.formatVersion !== 1 || !validFingerprints(value.pageFingerprints) || !safeNonnegativeInteger(value.documentCount) || typeof value.averageSectionLength !== "number" || !Number.isFinite(value.averageSectionLength) || value.averageSectionLength < 0 || !validFrequencyArray(value.documentFrequencies) || !Array.isArray(value.sections)) return false;
|
|
1663
|
+
let previous;
|
|
1664
|
+
for (const item of value.sections) {
|
|
1665
|
+
if (!record(item) || !exactKeys(item, [
|
|
1666
|
+
"pageId",
|
|
1667
|
+
"title",
|
|
1668
|
+
"headingTrail",
|
|
1669
|
+
"startLine",
|
|
1670
|
+
"sourceIds",
|
|
1671
|
+
"normalizedText",
|
|
1672
|
+
"length",
|
|
1673
|
+
"titleTermFrequencies",
|
|
1674
|
+
"headingTermFrequencies",
|
|
1675
|
+
"bodyTermFrequencies"
|
|
1676
|
+
]) || typeof item.pageId !== "string" || !isPageId(item.pageId) || typeof item.title !== "string" || !Array.isArray(item.headingTrail) || !item.headingTrail.every((part) => typeof part === "string") || !positiveInteger(item.startLine) || !sortedUniqueStrings(item.sourceIds) || !item.sourceIds.every(isSourceId) || typeof item.normalizedText !== "string" || !safeNonnegativeInteger(item.length) || !validFrequencyArray(item.titleTermFrequencies) || !validFrequencyArray(item.headingTermFrequencies) || !validFrequencyArray(item.bodyTermFrequencies)) return false;
|
|
1677
|
+
if (previous !== void 0 && (previous.pageId > item.pageId || previous.pageId === item.pageId && previous.startLine >= item.startLine)) return false;
|
|
1678
|
+
previous = {
|
|
1679
|
+
pageId: item.pageId,
|
|
1680
|
+
startLine: item.startLine
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
return value.documentCount === value.sections.length;
|
|
1684
|
+
}
|
|
1685
|
+
async function inspectIndex(context) {
|
|
1686
|
+
const indexStat = await stat(context.paths.index, context);
|
|
1687
|
+
if (indexStat === null) {
|
|
1688
|
+
diagnostic(context, "INDEX_MISSING", "warning", context.paths.index, "Derived search index is missing.");
|
|
1689
|
+
return;
|
|
1690
|
+
}
|
|
1691
|
+
if (indexStat.isSymbolicLink()) return;
|
|
1692
|
+
if (!indexStat.isDirectory()) {
|
|
1693
|
+
diagnostic(context, "REQUIRED_PATH_NOT_DIRECTORY", "error", context.paths.index, "Required wiki path is not a directory.");
|
|
1694
|
+
return;
|
|
1695
|
+
}
|
|
1696
|
+
const statePath = context.paths.indexFile("state.json");
|
|
1697
|
+
const searchPath = context.paths.indexFile("search.json");
|
|
1698
|
+
const [stateStat, searchStat] = await allSettledOnFailure([stat(statePath, context), stat(searchPath, context)]);
|
|
1699
|
+
if (stateStat?.isSymbolicLink() === true || searchStat?.isSymbolicLink() === true) return;
|
|
1700
|
+
if (stateStat === null || searchStat === null) {
|
|
1701
|
+
diagnostic(context, "INDEX_MISSING", "warning", context.paths.index, "Derived search index is missing.");
|
|
1702
|
+
return;
|
|
1703
|
+
}
|
|
1704
|
+
const [stateBytes, searchBytes] = await allSettledOnFailure([readBytes$1(statePath, context), readBytes$1(searchPath, context)]);
|
|
1705
|
+
if (stateBytes === null || searchBytes === null) {
|
|
1706
|
+
diagnostic(context, "INDEX_MISSING", "warning", context.paths.index, "Derived search index is missing.");
|
|
1707
|
+
return;
|
|
1708
|
+
}
|
|
1709
|
+
let stateUnknown;
|
|
1710
|
+
let searchUnknown;
|
|
1711
|
+
try {
|
|
1712
|
+
stateUnknown = JSON.parse(decodeUtf8(stateBytes));
|
|
1713
|
+
} catch {
|
|
1714
|
+
diagnostic(context, "INDEX_MALFORMED", "error", statePath, "Index state is not valid UTF-8 JSON.");
|
|
1715
|
+
return;
|
|
1716
|
+
}
|
|
1717
|
+
try {
|
|
1718
|
+
searchUnknown = JSON.parse(decodeUtf8(searchBytes));
|
|
1719
|
+
} catch {
|
|
1720
|
+
diagnostic(context, "INDEX_MALFORMED", "error", searchPath, "Search index is not valid UTF-8 JSON.");
|
|
1721
|
+
return;
|
|
1722
|
+
}
|
|
1723
|
+
if (record(stateUnknown) && Object.hasOwn(stateUnknown, "formatVersion") && stateUnknown.formatVersion !== 1) {
|
|
1724
|
+
diagnostic(context, "INDEX_INCOMPATIBLE", "error", statePath, "Index state uses an unsupported format version.");
|
|
1725
|
+
return;
|
|
1726
|
+
}
|
|
1727
|
+
if (record(searchUnknown) && Object.hasOwn(searchUnknown, "formatVersion") && searchUnknown.formatVersion !== 1) {
|
|
1728
|
+
diagnostic(context, "INDEX_INCOMPATIBLE", "error", searchPath, "Search index uses an unsupported format version.");
|
|
1729
|
+
return;
|
|
1730
|
+
}
|
|
1731
|
+
if (!validState(stateUnknown)) {
|
|
1732
|
+
diagnostic(context, "INDEX_MALFORMED", "error", statePath, "Index state does not match formatVersion 1.");
|
|
1733
|
+
return;
|
|
1734
|
+
}
|
|
1735
|
+
if (!validSearch(searchUnknown)) {
|
|
1736
|
+
diagnostic(context, "INDEX_MALFORMED", "error", searchPath, "Search index does not match formatVersion 1.");
|
|
1737
|
+
return;
|
|
1738
|
+
}
|
|
1739
|
+
if (`${JSON.stringify(stateUnknown, null, 2)}\n` !== decodeUtf8(stateBytes)) {
|
|
1740
|
+
diagnostic(context, "INDEX_MALFORMED", "error", statePath, "Index state is not canonically serialized.");
|
|
1741
|
+
return;
|
|
1742
|
+
}
|
|
1743
|
+
if (`${JSON.stringify(searchUnknown, null, 2)}\n` !== decodeUtf8(searchBytes)) {
|
|
1744
|
+
diagnostic(context, "INDEX_MALFORMED", "error", searchPath, "Search index is not canonically serialized.");
|
|
1745
|
+
return;
|
|
1746
|
+
}
|
|
1747
|
+
try {
|
|
1748
|
+
const expected = await buildSearchIndex(context.paths, context.signal, (path) => context.examinedPaths.add(path), context.root);
|
|
1749
|
+
const matches = trustedSearchIndex(searchBytes, stateBytes, expected) !== null;
|
|
1750
|
+
await validateBuiltIndexSnapshot(context.paths, expected, context.signal, context.root);
|
|
1751
|
+
if (!matches) diagnostic(context, "INDEX_STALE", "warning", context.paths.index, "Derived search index is stale or has a hash mismatch.");
|
|
1752
|
+
} catch (cause) {
|
|
1753
|
+
if (cause instanceof LlmWikiError && cause.code === "ABORTED") throw cause;
|
|
1754
|
+
throwIfAborted(context.signal);
|
|
1755
|
+
const nestedCode = cause instanceof Error && cause.cause !== void 0 ? cause.cause.code : void 0;
|
|
1756
|
+
if (!(cause instanceof LlmWikiError && (cause.code === "INVALID_PAGE" || cause.code === "INVALID_PATH" || cause.code === "UNSAFE_FILESYSTEM" && (cause.cause === void 0 || nestedCode === "ENOENT" || nestedCode === "ENOTDIR")))) throw cause;
|
|
1757
|
+
diagnostic(context, "INDEX_STALE", "warning", context.paths.index, "Derived search index is stale or has a hash mismatch.");
|
|
1758
|
+
}
|
|
1759
|
+
}
|
|
1760
|
+
function lintRootFailure(paths, cause) {
|
|
1761
|
+
const components = paths.authority.root.split("/").filter(Boolean);
|
|
1762
|
+
if (!(cause instanceof WikiTraversalError) || cause.segments.length !== components.length || !cause.segments.every((segment, index) => segment === components[index]) || cause.entryKind === "unknown") throw cause;
|
|
1763
|
+
const symlink = cause.entryKind === "symlink";
|
|
1764
|
+
return {
|
|
1765
|
+
diagnostics: [{
|
|
1766
|
+
code: symlink ? "UNSAFE_SYMLINK" : "ROOT_NOT_DIRECTORY",
|
|
1767
|
+
severity: "error",
|
|
1768
|
+
path: ".",
|
|
1769
|
+
message: symlink ? "Wiki root must not be a symbolic link." : "Wiki root is not a directory."
|
|
1770
|
+
}],
|
|
1771
|
+
errorCount: 1,
|
|
1772
|
+
warningCount: 0,
|
|
1773
|
+
filesExamined: 0
|
|
1774
|
+
};
|
|
1775
|
+
}
|
|
1776
|
+
async function lintWiki(paths, signal, root) {
|
|
1777
|
+
throwIfAborted(signal);
|
|
1778
|
+
if (root === void 0) try {
|
|
1779
|
+
return await withWikiRoot(paths.authority, { signal }, (held) => lintWiki(paths, signal, held)) ?? lintWiki(paths, signal, null);
|
|
1780
|
+
} catch (cause) {
|
|
1781
|
+
throwIfAborted(signal);
|
|
1782
|
+
return lintRootFailure(paths, cause);
|
|
1783
|
+
}
|
|
1784
|
+
if (root === null) return {
|
|
1785
|
+
diagnostics: [{
|
|
1786
|
+
code: "ROOT_MISSING",
|
|
1787
|
+
severity: "error",
|
|
1788
|
+
path: ".",
|
|
1789
|
+
message: "Wiki root is missing."
|
|
1790
|
+
}],
|
|
1791
|
+
errorCount: 1,
|
|
1792
|
+
warningCount: 0,
|
|
1793
|
+
filesExamined: 0
|
|
1794
|
+
};
|
|
1795
|
+
const context = {
|
|
1796
|
+
paths,
|
|
1797
|
+
root,
|
|
1798
|
+
signal,
|
|
1799
|
+
diagnostics: [],
|
|
1800
|
+
examinedPaths: /* @__PURE__ */ new Set()
|
|
1801
|
+
};
|
|
1802
|
+
{
|
|
1803
|
+
await scanTempsAndSymlinks(paths.root, context);
|
|
1804
|
+
const schemaStat = await stat(paths.schema, context);
|
|
1805
|
+
if (schemaStat === null) diagnostic(context, "SCHEMA_MISSING", "error", paths.schema, "Wiki schema is missing.");
|
|
1806
|
+
else if (schemaStat.isSymbolicLink()) {} else if (!schemaStat.isFile()) diagnostic(context, "REQUIRED_PATH_NOT_DIRECTORY", "error", paths.schema, "Wiki schema must be a regular file.");
|
|
1807
|
+
else {
|
|
1808
|
+
const bytes = await readBytes$1(paths.schema, context);
|
|
1809
|
+
if (bytes !== null) try {
|
|
1810
|
+
decodeUtf8(bytes);
|
|
1811
|
+
} catch {
|
|
1812
|
+
diagnostic(context, "INVALID_UTF8", "error", paths.schema, "Wiki schema is not valid UTF-8.");
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
const sources = await inspectSources(context);
|
|
1816
|
+
const referencedSources = await inspectPages(sources, context);
|
|
1817
|
+
for (const id of sources) if (!referencedSources.has(id)) diagnostic(context, "SOURCE_UNREFERENCED", "warning", paths.sourceMetadata(sourceId(id)), "Source is not referenced by any valid page.");
|
|
1818
|
+
await inspectIndex(context);
|
|
1819
|
+
}
|
|
1820
|
+
throwIfAborted(signal);
|
|
1821
|
+
context.diagnostics.sort((left, right) => compareCodeUnits$1(left.path, right.path) || (left.line ?? Number.POSITIVE_INFINITY) - (right.line ?? Number.POSITIVE_INFINITY) || compareCodeUnits$1(left.code, right.code) || compareCodeUnits$1(left.message, right.message));
|
|
1822
|
+
return {
|
|
1823
|
+
diagnostics: context.diagnostics,
|
|
1824
|
+
errorCount: context.diagnostics.filter(({ severity }) => severity === "error").length,
|
|
1825
|
+
warningCount: context.diagnostics.filter(({ severity }) => severity === "warning").length,
|
|
1826
|
+
filesExamined: context.examinedPaths.size
|
|
1827
|
+
};
|
|
1828
|
+
}
|
|
1829
|
+
//#endregion
|
|
1830
|
+
//#region src/service.ts
|
|
1831
|
+
const DEFAULT_SCHEMA = `# LLM Wiki Schema
|
|
1832
|
+
|
|
1833
|
+
This schema is human-owned organization and workflow guidance. The plugin creates it only when absent, exposes it through status, and never rewrites it; system and user instructions take precedence. There is no schema mutation API; schema evolution remains intentionally unresolved pending authorization/confirmation, visible audit evidence, and optimistic-concurrency/lost-update product decisions.
|
|
1834
|
+
|
|
1835
|
+
Pages are durable source-linked Markdown notes. Keep titles and summaries concise, organize related claims under headings, maintain useful page links, preserve material disagreements and dated supersessions, and cite every relevant existing immutable source ID in frontmatter. Source citation proves record existence, not claim-level support.
|
|
1836
|
+
|
|
1837
|
+
Evidence maintenance: call llmwiki_status first and read schemaText when non-null. On a fresh root, status returns schemaText null without creating storage; supplied material alone is not authorization to preserve it. Only with explicit authorization to preserve the source, call llmwiki_add_source to initialize storage, then call status again and read the schema before classification or page maintenance. List sources and pages, then search and read relevant records before writing. Only with explicit authorization to preserve candidate material, add it if the fresh-root branch did not, then classify its effect as new, update, contradiction, or no material change. Separately, only when the user request authorizes maintenance, update every materially affected page, preserve disagreements and links, and cite existing source IDs. Run llmwiki_lint unconditionally before any semantic-review pass, including read-only, no-write, and no-material-change cases; it is structural only and never repairs artifacts or makes semantic judgments. After any authorized durable updates, rerun structural lint.
|
|
1838
|
+
|
|
1839
|
+
Page writes require expectedSha256: null for a new page, or the exact current raw-byte sha256 from llmwiki_read_page or llmwiki_list_pages for an update. On PAGE_CONFLICT, reread the winning page and reconcile before any further authorized write; never automatically retry or overwrite blindly. This concurrency protection is limited to one service activation, not cross-process writers.
|
|
1840
|
+
|
|
1841
|
+
Semantic review (separate from structural lint): only after the unconditional structural lint, list pages and sources. Select and state the review scope, compare dated and qualified claims across every scoped page, every source it cites, and every new candidate source relevant to that scope, and report classified contradiction, superseded, unsupported, and missing-link findings with visible page and source IDs. These semantic findings are agent judgments, never llmwiki_lint diagnostics. Only when the user request authorizes maintenance, update affected pages while preserving disagreements or dated supersessions and maintain links; after any such durable updates, rerun structural lint.
|
|
1842
|
+
`;
|
|
1843
|
+
const HASH = /^[0-9a-f]{64}$/u;
|
|
1844
|
+
const NON_WHITESPACE = /\S/u;
|
|
1845
|
+
const INCOMPLETE_UTF8_RANGE = "Source byte range contains no complete UTF-8 code point; increase the limit.";
|
|
1846
|
+
const CURSOR_TEXT = /^[A-Za-z0-9_-]+$/u;
|
|
1847
|
+
function compareCodeUnits(left, right) {
|
|
1848
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
1849
|
+
}
|
|
1850
|
+
function catalogCorrupt(message) {
|
|
1851
|
+
return new LlmWikiError("CATALOG_CORRUPT", message);
|
|
1852
|
+
}
|
|
1853
|
+
function invalidCursor() {
|
|
1854
|
+
throw new LlmWikiError("INVALID_CURSOR", "Catalog cursor is invalid.");
|
|
1855
|
+
}
|
|
1856
|
+
function encodeCursor(kind, after) {
|
|
1857
|
+
return Buffer.from(JSON.stringify({
|
|
1858
|
+
v: 1,
|
|
1859
|
+
kind,
|
|
1860
|
+
after
|
|
1861
|
+
}), "utf8").toString("base64url");
|
|
1862
|
+
}
|
|
1863
|
+
function decodeCursor(cursor, kind) {
|
|
1864
|
+
if (cursor === void 0) return void 0;
|
|
1865
|
+
if (!CURSOR_TEXT.test(cursor) || cursor.includes("=")) return invalidCursor();
|
|
1866
|
+
let text;
|
|
1867
|
+
try {
|
|
1868
|
+
const bytes = Buffer.from(cursor, "base64url");
|
|
1869
|
+
if (bytes.toString("base64url") !== cursor) return invalidCursor();
|
|
1870
|
+
text = decodeUtf8(bytes);
|
|
1871
|
+
} catch {
|
|
1872
|
+
return invalidCursor();
|
|
1873
|
+
}
|
|
1874
|
+
let value;
|
|
1875
|
+
try {
|
|
1876
|
+
value = JSON.parse(text);
|
|
1877
|
+
} catch {
|
|
1878
|
+
return invalidCursor();
|
|
1879
|
+
}
|
|
1880
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return invalidCursor();
|
|
1881
|
+
const object = value;
|
|
1882
|
+
if (object.v !== 1 || object.kind !== kind || typeof object.after !== "string" || (kind === "sources" ? !isSourceId(object.after) : !isPageId(object.after)) || JSON.stringify({
|
|
1883
|
+
v: 1,
|
|
1884
|
+
kind,
|
|
1885
|
+
after: object.after
|
|
1886
|
+
}) !== text) return invalidCursor();
|
|
1887
|
+
return object.after;
|
|
1888
|
+
}
|
|
1889
|
+
function catalogLimit(request, maximum) {
|
|
1890
|
+
const value = request.limit ?? maximum;
|
|
1891
|
+
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw limit("Catalog limit is outside the configured range.");
|
|
1892
|
+
return value;
|
|
1893
|
+
}
|
|
1894
|
+
function appendCatalogItem(items, id, after, limitValue, create) {
|
|
1895
|
+
if (after !== void 0 && compareCodeUnits(id, after) <= 0) return false;
|
|
1896
|
+
if (items.length === limitValue) return true;
|
|
1897
|
+
items.push(create());
|
|
1898
|
+
return false;
|
|
1899
|
+
}
|
|
1900
|
+
function finishCatalogPage(kind, items, hasMore) {
|
|
1901
|
+
return {
|
|
1902
|
+
items,
|
|
1903
|
+
nextCursor: hasMore ? encodeCursor(kind, items[items.length - 1].id) : null
|
|
1904
|
+
};
|
|
1905
|
+
}
|
|
1906
|
+
function unsafe() {
|
|
1907
|
+
throw new LlmWikiError("UNSAFE_FILESYSTEM", "The wiki filesystem changed or contains an unsafe entry.");
|
|
1908
|
+
}
|
|
1909
|
+
async function inDirectory(paths, path, signal, work) {
|
|
1910
|
+
if (paths.rootDirectory === null) return null;
|
|
1911
|
+
return paths.rootDirectory.directory(wikiRelativeSegments(paths, path), { signal }, work);
|
|
1912
|
+
}
|
|
1913
|
+
async function inspectPath(paths, path, signal) {
|
|
1914
|
+
const segments = wikiRelativeSegments(paths, path);
|
|
1915
|
+
const name = segments.at(-1);
|
|
1916
|
+
return inDirectory(paths, join(paths.root, ...segments.slice(0, -1)), signal, (directory) => directory.inspect(name, signal));
|
|
1917
|
+
}
|
|
1918
|
+
async function readBytes(paths, path, signal) {
|
|
1919
|
+
const segments = wikiRelativeSegments(paths, path);
|
|
1920
|
+
return inDirectory(paths, join(paths.root, ...segments.slice(0, -1)), signal, async (directory) => (await directory.read(segments.at(-1), signal))?.bytes ?? null);
|
|
1921
|
+
}
|
|
1922
|
+
async function collectCatalogPageFiles(directory, paths, output, snapshots, signal) {
|
|
1923
|
+
const pending = [{
|
|
1924
|
+
path: directory,
|
|
1925
|
+
logicalPath: ""
|
|
1926
|
+
}];
|
|
1927
|
+
while (pending.length > 0) {
|
|
1928
|
+
throwIfAborted(signal);
|
|
1929
|
+
const current = pending.pop();
|
|
1930
|
+
const read = await readSafeCatalogDirectory(current.path, paths, signal);
|
|
1931
|
+
snapshots.push(read.snapshot);
|
|
1932
|
+
for (const entry of read.entries) {
|
|
1933
|
+
const logicalPath = current.logicalPath === "" ? entry.name : `${current.logicalPath}/${entry.name}`;
|
|
1934
|
+
const child = join(current.path, entry.name);
|
|
1935
|
+
if (entry.isSymbolicLink()) unsafe();
|
|
1936
|
+
if (entry.isDirectory()) pending.push({
|
|
1937
|
+
path: child,
|
|
1938
|
+
logicalPath
|
|
1939
|
+
});
|
|
1940
|
+
else if (entry.isFile()) {
|
|
1941
|
+
const snapshot = await snapshotSafeCatalogFile(child, paths, signal);
|
|
1942
|
+
snapshots.push(snapshot);
|
|
1943
|
+
output.push({
|
|
1944
|
+
path: child,
|
|
1945
|
+
logicalPath,
|
|
1946
|
+
snapshot
|
|
1947
|
+
});
|
|
1948
|
+
} else unsafe();
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
const EMPTY_INDEX_STATUS = Object.freeze({
|
|
1953
|
+
present: false,
|
|
1954
|
+
fresh: false,
|
|
1955
|
+
formatVersion: null,
|
|
1956
|
+
sectionCount: 0
|
|
1957
|
+
});
|
|
1958
|
+
function hash(bytes) {
|
|
1959
|
+
return createHash("sha256").update(bytes).digest("hex");
|
|
1960
|
+
}
|
|
1961
|
+
function canonicalJson(value) {
|
|
1962
|
+
return encodeUtf8(`${JSON.stringify(value, null, 2)}\n`);
|
|
1963
|
+
}
|
|
1964
|
+
function sameStableSnapshot(before, after) {
|
|
1965
|
+
return before.dev === after.dev && before.ino === after.ino && before.size === after.size && before.mtimeNs === after.mtimeNs && before.ctimeNs === after.ctimeNs;
|
|
1966
|
+
}
|
|
1967
|
+
async function revalidateCatalogSnapshot(snapshot, paths, signal) {
|
|
1968
|
+
try {
|
|
1969
|
+
const current = snapshot.kind === "file" ? await snapshotSafeCatalogFile(snapshot.path, paths, signal) : (await readSafeCatalogDirectory(snapshot.path, paths, signal)).snapshot;
|
|
1970
|
+
if (!sameStableSnapshot(snapshot.stats, current.stats)) unsafe();
|
|
1971
|
+
if (snapshot.children !== void 0 && (current.children?.length !== snapshot.children.length || current.children.some((child, index) => child !== snapshot.children[index]))) unsafe();
|
|
1972
|
+
} catch (cause) {
|
|
1973
|
+
throwIfAborted(signal);
|
|
1974
|
+
if (cause instanceof LlmWikiError) throw cause;
|
|
1975
|
+
throw new LlmWikiError("UNSAFE_FILESYSTEM", "Unable to revalidate the catalog safely.", { cause });
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
async function readSafeCatalogDirectory(path, paths, signal) {
|
|
1979
|
+
const before = await inspectPath(paths, path, signal);
|
|
1980
|
+
if (before === null || !before.isDirectory() || before.isSymbolicLink()) unsafe();
|
|
1981
|
+
const listed = await inDirectory(paths, path, signal, (directory) => directory.list(signal));
|
|
1982
|
+
if (listed === null) unsafe();
|
|
1983
|
+
const final = await inspectPath(paths, path, signal);
|
|
1984
|
+
if (final === null || !sameStableSnapshot(before, final)) unsafe();
|
|
1985
|
+
const entries = listed.map((entry) => ({
|
|
1986
|
+
name: entry.name,
|
|
1987
|
+
isFile: () => entry.stat.isFile(),
|
|
1988
|
+
isDirectory: () => entry.stat.isDirectory(),
|
|
1989
|
+
isSymbolicLink: () => entry.stat.isSymbolicLink()
|
|
1990
|
+
}));
|
|
1991
|
+
return {
|
|
1992
|
+
entries,
|
|
1993
|
+
snapshot: {
|
|
1994
|
+
path,
|
|
1995
|
+
stats: final,
|
|
1996
|
+
kind: "directory",
|
|
1997
|
+
children: entries.map((entry) => `${entry.name}\0${entry.isFile() ? "file" : entry.isDirectory() ? "directory" : entry.isSymbolicLink() ? "symlink" : "other"}`).sort(compareCodeUnits)
|
|
1998
|
+
}
|
|
1999
|
+
};
|
|
2000
|
+
}
|
|
2001
|
+
async function snapshotSafeCatalogFile(path, paths, signal) {
|
|
2002
|
+
const stats = await inspectPath(paths, path, signal);
|
|
2003
|
+
if (stats === null || !stats.isFile() || stats.isSymbolicLink()) unsafe();
|
|
2004
|
+
return {
|
|
2005
|
+
path,
|
|
2006
|
+
stats,
|
|
2007
|
+
kind: "file"
|
|
2008
|
+
};
|
|
2009
|
+
}
|
|
2010
|
+
async function readSafeCatalogFile(path, paths, signal) {
|
|
2011
|
+
const segments = wikiRelativeSegments(paths, path);
|
|
2012
|
+
const read = await inDirectory(paths, join(paths.root, ...segments.slice(0, -1)), signal, (directory) => directory.read(segments.at(-1), signal));
|
|
2013
|
+
if (read === null) throw Object.assign(/* @__PURE__ */ new Error("Wiki file is missing."), { code: "ENOENT" });
|
|
2014
|
+
return {
|
|
2015
|
+
bytes: read.bytes,
|
|
2016
|
+
snapshot: {
|
|
2017
|
+
path,
|
|
2018
|
+
stats: read.snapshot.stat,
|
|
2019
|
+
kind: "file"
|
|
2020
|
+
}
|
|
2021
|
+
};
|
|
2022
|
+
}
|
|
2023
|
+
function missing(code, message) {
|
|
2024
|
+
return new LlmWikiError(code, message);
|
|
2025
|
+
}
|
|
2026
|
+
function limit(message) {
|
|
2027
|
+
return new LlmWikiError("LIMIT_EXCEEDED", message);
|
|
2028
|
+
}
|
|
2029
|
+
function isMissing(cause) {
|
|
2030
|
+
return cause.code === "ENOENT";
|
|
2031
|
+
}
|
|
2032
|
+
function sanitizeFailure(cause) {
|
|
2033
|
+
if (cause instanceof LlmWikiError) throw cause.cause === void 0 ? cause : new LlmWikiError(cause.code, cause.message);
|
|
2034
|
+
if (typeof cause.code === "string") throw new LlmWikiError("UNSAFE_FILESYSTEM", "The wiki filesystem operation failed.");
|
|
2035
|
+
throw cause;
|
|
2036
|
+
}
|
|
2037
|
+
function validateText(value, field) {
|
|
2038
|
+
if (!NON_WHITESPACE.test(value)) throw new LlmWikiError("INVALID_PAGE", `${field} must not be empty.`);
|
|
2039
|
+
}
|
|
2040
|
+
function parseMetadata(bytes, expectedId) {
|
|
2041
|
+
let value;
|
|
2042
|
+
try {
|
|
2043
|
+
value = JSON.parse(decodeUtf8(bytes));
|
|
2044
|
+
} catch (cause) {
|
|
2045
|
+
throw new LlmWikiError("INVALID_PAGE", "Source metadata is malformed.", { cause });
|
|
2046
|
+
}
|
|
2047
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) throw new LlmWikiError("INVALID_PAGE", "Source metadata is malformed.");
|
|
2048
|
+
const object = value;
|
|
2049
|
+
const keys = Object.keys(object);
|
|
2050
|
+
const expectedKeys = object.origin === void 0 ? [
|
|
2051
|
+
"id",
|
|
2052
|
+
"name",
|
|
2053
|
+
"mediaType",
|
|
2054
|
+
"byteCount",
|
|
2055
|
+
"capturedAt"
|
|
2056
|
+
] : [
|
|
2057
|
+
"id",
|
|
2058
|
+
"name",
|
|
2059
|
+
"mediaType",
|
|
2060
|
+
"byteCount",
|
|
2061
|
+
"capturedAt",
|
|
2062
|
+
"origin"
|
|
2063
|
+
];
|
|
2064
|
+
if (keys.length !== expectedKeys.length || keys.some((key, index) => key !== expectedKeys[index]) || object.id !== expectedId || typeof object.name !== "string" || !NON_WHITESPACE.test(object.name) || typeof object.mediaType !== "string" || !NON_WHITESPACE.test(object.mediaType) || typeof object.byteCount !== "number" || !Number.isSafeInteger(object.byteCount) || object.byteCount < 0 || typeof object.capturedAt !== "string" || Number.isNaN(Date.parse(object.capturedAt)) || new Date(object.capturedAt).toISOString() !== object.capturedAt || object.origin !== void 0 && (typeof object.origin !== "string" || !NON_WHITESPACE.test(object.origin)) || !Buffer.from(canonicalJson(object)).equals(Buffer.from(bytes))) throw new LlmWikiError("INVALID_PAGE", "Source metadata does not match its required schema.");
|
|
2065
|
+
return object;
|
|
2066
|
+
}
|
|
2067
|
+
function alignedRange(bytes, offset, limitValue) {
|
|
2068
|
+
let start = offset;
|
|
2069
|
+
while (start < bytes.byteLength && (bytes[start] & 192) === 128) start += 1;
|
|
2070
|
+
let end = Math.max(start, Math.min(bytes.byteLength, offset + limitValue));
|
|
2071
|
+
while (end > start && end < bytes.byteLength && (bytes[end] & 192) === 128) end -= 1;
|
|
2072
|
+
return {
|
|
2073
|
+
start,
|
|
2074
|
+
end
|
|
2075
|
+
};
|
|
2076
|
+
}
|
|
2077
|
+
async function regularFile(path, paths, signal) {
|
|
2078
|
+
const stat = await inspectPath(paths, path, signal);
|
|
2079
|
+
if (stat === null) return false;
|
|
2080
|
+
if (!stat.isFile() || stat.isSymbolicLink()) unsafe();
|
|
2081
|
+
return true;
|
|
2082
|
+
}
|
|
2083
|
+
async function regularDirectory(path, paths, signal) {
|
|
2084
|
+
const stat = await inspectPath(paths, path, signal);
|
|
2085
|
+
if (stat === null) return false;
|
|
2086
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) unsafe();
|
|
2087
|
+
return true;
|
|
2088
|
+
}
|
|
2089
|
+
function wikiRootPresent(paths, signal) {
|
|
2090
|
+
throwIfAborted(signal);
|
|
2091
|
+
return paths.rootDirectory !== null;
|
|
2092
|
+
}
|
|
2093
|
+
async function countFiles(directory, suffix, paths, signal) {
|
|
2094
|
+
let total = 0;
|
|
2095
|
+
const pending = [directory];
|
|
2096
|
+
while (pending.length > 0) {
|
|
2097
|
+
const current = pending.pop();
|
|
2098
|
+
const entries = await inDirectory(paths, current, signal, (capability) => capability.list(signal));
|
|
2099
|
+
if (entries === null) unsafe();
|
|
2100
|
+
for (const entry of entries) {
|
|
2101
|
+
throwIfAborted(signal);
|
|
2102
|
+
if (entry.stat.isSymbolicLink()) unsafe();
|
|
2103
|
+
if (entry.stat.isDirectory()) {
|
|
2104
|
+
if (suffix !== void 0 && entry.name.endsWith(suffix)) unsafe();
|
|
2105
|
+
pending.push(join(current, entry.name));
|
|
2106
|
+
} else if (!entry.stat.isFile()) unsafe();
|
|
2107
|
+
else if (suffix === void 0 || entry.name.endsWith(suffix)) total += 1;
|
|
2108
|
+
}
|
|
2109
|
+
}
|
|
2110
|
+
return total;
|
|
2111
|
+
}
|
|
2112
|
+
async function countSources(paths, signal) {
|
|
2113
|
+
const entries = await inDirectory(paths, paths.sources, signal, (directory) => directory.list(signal));
|
|
2114
|
+
if (entries === null) unsafe();
|
|
2115
|
+
let total = 0;
|
|
2116
|
+
for (const entry of entries) {
|
|
2117
|
+
if (entry.stat.isSymbolicLink() || !entry.stat.isDirectory()) unsafe();
|
|
2118
|
+
const directory = join(paths.sources, entry.name);
|
|
2119
|
+
await countFiles(directory, void 0, paths, signal);
|
|
2120
|
+
if (HASH.test(entry.name)) {
|
|
2121
|
+
if (!await regularFile(join(directory, "content"), paths, signal) || !await regularFile(join(directory, "metadata.json"), paths, signal)) unsafe();
|
|
2122
|
+
total += 1;
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
2125
|
+
return total;
|
|
2126
|
+
}
|
|
2127
|
+
const configKey = Symbol("llmwiki.config");
|
|
2128
|
+
var LlmWikiService = class extends Service {
|
|
2129
|
+
static Config = Config;
|
|
2130
|
+
[configKey];
|
|
2131
|
+
pathsValue;
|
|
2132
|
+
queue = Promise.resolve();
|
|
2133
|
+
queued = /* @__PURE__ */ new Set();
|
|
2134
|
+
disposed = false;
|
|
2135
|
+
constructor(ctx, config = {}) {
|
|
2136
|
+
super(ctx, "llmwiki");
|
|
2137
|
+
const resolved = resolveConfig(config);
|
|
2138
|
+
this[configKey] = Object.freeze({
|
|
2139
|
+
...resolved,
|
|
2140
|
+
root: resolve(process.cwd(), resolved.root)
|
|
2141
|
+
});
|
|
2142
|
+
ctx.effect(() => () => {
|
|
2143
|
+
this.disposed = true;
|
|
2144
|
+
const error = new LlmWikiError("NOT_INITIALIZED", "The llmwiki service has been disposed.");
|
|
2145
|
+
for (const reject of this.queued) reject(error);
|
|
2146
|
+
this.queued.clear();
|
|
2147
|
+
return this.queue.catch(() => void 0);
|
|
2148
|
+
}, "llmwiki.service");
|
|
2149
|
+
}
|
|
2150
|
+
enqueue(work, signal, initialize = true) {
|
|
2151
|
+
if (this.disposed) return Promise.reject(new LlmWikiError("NOT_INITIALIZED", "The llmwiki service has been disposed."));
|
|
2152
|
+
let started = false;
|
|
2153
|
+
let settled = false;
|
|
2154
|
+
let cancelled = false;
|
|
2155
|
+
let resolveResult;
|
|
2156
|
+
let rejectResult;
|
|
2157
|
+
const result = new Promise((resolvePromise, rejectPromise) => {
|
|
2158
|
+
resolveResult = resolvePromise;
|
|
2159
|
+
rejectResult = rejectPromise;
|
|
2160
|
+
});
|
|
2161
|
+
result.catch(() => void 0);
|
|
2162
|
+
const rejectQueued = (error) => {
|
|
2163
|
+
if (started || settled) return;
|
|
2164
|
+
cancelled = true;
|
|
2165
|
+
settled = true;
|
|
2166
|
+
rejectResult(error);
|
|
2167
|
+
};
|
|
2168
|
+
const onAbort = () => rejectQueued(new LlmWikiError("ABORTED", "The operation was aborted."));
|
|
2169
|
+
this.queued.add(rejectQueued);
|
|
2170
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2171
|
+
if (signal?.aborted) onAbort();
|
|
2172
|
+
const scheduled = this.queue.then(async () => {
|
|
2173
|
+
started = true;
|
|
2174
|
+
this.queued.delete(rejectQueued);
|
|
2175
|
+
if (cancelled) return;
|
|
2176
|
+
try {
|
|
2177
|
+
if (this.disposed) throw new LlmWikiError("NOT_INITIALIZED", "The llmwiki service has been disposed.");
|
|
2178
|
+
const paths = this.pathsValue ?? createWikiPaths(this[configKey].root);
|
|
2179
|
+
let ran = false;
|
|
2180
|
+
const pinned = await withWikiRoot(paths.authority, {
|
|
2181
|
+
create: initialize,
|
|
2182
|
+
signal
|
|
2183
|
+
}, async (rootDirectory) => {
|
|
2184
|
+
ran = true;
|
|
2185
|
+
const operationPaths = {
|
|
2186
|
+
...paths,
|
|
2187
|
+
rootDirectory
|
|
2188
|
+
};
|
|
2189
|
+
if (initialize) await this.initialize(operationPaths, signal);
|
|
2190
|
+
throwIfAborted(signal);
|
|
2191
|
+
return work(operationPaths);
|
|
2192
|
+
});
|
|
2193
|
+
const value = ran ? pinned : await work({
|
|
2194
|
+
...paths,
|
|
2195
|
+
rootDirectory: null
|
|
2196
|
+
});
|
|
2197
|
+
if (!settled) {
|
|
2198
|
+
settled = true;
|
|
2199
|
+
resolveResult(value);
|
|
2200
|
+
}
|
|
2201
|
+
} catch (cause) {
|
|
2202
|
+
if (!settled) {
|
|
2203
|
+
settled = true;
|
|
2204
|
+
try {
|
|
2205
|
+
sanitizeFailure(cause);
|
|
2206
|
+
} catch (error) {
|
|
2207
|
+
rejectResult(error);
|
|
2208
|
+
}
|
|
2209
|
+
}
|
|
2210
|
+
} finally {
|
|
2211
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2212
|
+
}
|
|
2213
|
+
});
|
|
2214
|
+
this.queue = scheduled.then(() => void 0, () => void 0);
|
|
2215
|
+
return result;
|
|
2216
|
+
}
|
|
2217
|
+
async initialize(paths, signal) {
|
|
2218
|
+
if (this.pathsValue !== void 0) return;
|
|
2219
|
+
const root = paths.rootDirectory;
|
|
2220
|
+
for (const name of [
|
|
2221
|
+
"sources",
|
|
2222
|
+
"pages",
|
|
2223
|
+
".index"
|
|
2224
|
+
]) if (await root.directory([name], {
|
|
2225
|
+
create: true,
|
|
2226
|
+
signal
|
|
2227
|
+
}, () => Promise.resolve(true)) === null) unsafe();
|
|
2228
|
+
if (!await regularFile(paths.schema, paths, signal)) try {
|
|
2229
|
+
await root.createFileExclusive("schema.md", encodeUtf8(DEFAULT_SCHEMA), {
|
|
2230
|
+
mode: 384,
|
|
2231
|
+
signal
|
|
2232
|
+
});
|
|
2233
|
+
} catch (cause) {
|
|
2234
|
+
throwIfAborted(signal);
|
|
2235
|
+
if (cause.code !== "EEXIST") throw cause;
|
|
2236
|
+
}
|
|
2237
|
+
if (await root.read("schema.md", signal) === null) unsafe();
|
|
2238
|
+
paths.authority.identity = await root.identity();
|
|
2239
|
+
this.pathsValue = createWikiPaths(paths.root, paths.authority);
|
|
2240
|
+
}
|
|
2241
|
+
async status(signal) {
|
|
2242
|
+
return this.enqueue(async (paths) => {
|
|
2243
|
+
if (!wikiRootPresent(paths, signal)) return {
|
|
2244
|
+
initialized: false,
|
|
2245
|
+
sourceCount: 0,
|
|
2246
|
+
pageCount: 0,
|
|
2247
|
+
schemaText: null,
|
|
2248
|
+
index: EMPTY_INDEX_STATUS
|
|
2249
|
+
};
|
|
2250
|
+
const [schemaPresent, sourcesPresent, pagesPresent, indexPresent] = await allSettledOnFailure([
|
|
2251
|
+
regularFile(paths.schema, paths, signal),
|
|
2252
|
+
regularDirectory(paths.sources, paths, signal),
|
|
2253
|
+
regularDirectory(paths.pages, paths, signal),
|
|
2254
|
+
regularDirectory(paths.index, paths, signal)
|
|
2255
|
+
]);
|
|
2256
|
+
const [schemaBytes, sourceCount, pageCount, index] = await allSettledOnFailure([
|
|
2257
|
+
schemaPresent ? readBytes(paths, paths.schema, signal) : null,
|
|
2258
|
+
sourcesPresent ? countSources(paths, signal) : 0,
|
|
2259
|
+
pagesPresent ? countFiles(paths.pages, ".md", paths, signal) : 0,
|
|
2260
|
+
indexPresent ? this.indexStatus(paths, signal) : EMPTY_INDEX_STATUS
|
|
2261
|
+
]);
|
|
2262
|
+
throwIfAborted(signal);
|
|
2263
|
+
return {
|
|
2264
|
+
initialized: schemaPresent && sourcesPresent && pagesPresent && indexPresent,
|
|
2265
|
+
sourceCount,
|
|
2266
|
+
pageCount,
|
|
2267
|
+
schemaText: schemaBytes === null ? null : decodeUtf8(schemaBytes),
|
|
2268
|
+
index
|
|
2269
|
+
};
|
|
2270
|
+
}, signal, false);
|
|
2271
|
+
}
|
|
2272
|
+
async addSource(input, signal) {
|
|
2273
|
+
if (this.disposed) return Promise.reject(new LlmWikiError("NOT_INITIALIZED", "The llmwiki service has been disposed."));
|
|
2274
|
+
throwIfAborted(signal);
|
|
2275
|
+
validateText(input.name, "Source name");
|
|
2276
|
+
const mediaType = input.mediaType ?? "text/plain; charset=utf-8";
|
|
2277
|
+
validateText(mediaType, "Source media type");
|
|
2278
|
+
if (input.origin !== void 0) validateText(input.origin, "Source origin");
|
|
2279
|
+
if (input.content.length === 0) throw new LlmWikiError("INVALID_PAGE", "Source content must not be empty.");
|
|
2280
|
+
const content = encodeUtf8(input.content);
|
|
2281
|
+
if (content.byteLength > this[configKey].maxSourceBytes) throw limit("Source content exceeds maxSourceBytes.");
|
|
2282
|
+
const id = sourceId(hash(content));
|
|
2283
|
+
return this.enqueue(async (paths) => {
|
|
2284
|
+
const result = await paths.rootDirectory.directory(["sources"], { signal }, async (sources) => {
|
|
2285
|
+
const allocation = await sources.createDirectory(id, signal);
|
|
2286
|
+
try {
|
|
2287
|
+
const result = await sources.directory([id], { signal }, async (directory) => {
|
|
2288
|
+
const identity = await directory.identity();
|
|
2289
|
+
if (identity.dev !== allocation.identity.dev || identity.ino !== allocation.identity.ino) unsafe();
|
|
2290
|
+
const children = await directory.list(signal);
|
|
2291
|
+
if (children.some((child) => child.name === "metadata.json")) {
|
|
2292
|
+
const existing = await this.readSourceRecord(paths, id, signal);
|
|
2293
|
+
if (children.length !== 2 || children[0]?.name !== "content" || children[1]?.name !== "metadata.json") unsafe();
|
|
2294
|
+
for (const child of children) if (!child.stat.isFile() || !await directory.validate(child, signal)) unsafe();
|
|
2295
|
+
for (const snapshot of existing.snapshots) await revalidateCatalogSnapshot(snapshot, paths, signal);
|
|
2296
|
+
throwIfAborted(signal);
|
|
2297
|
+
return {
|
|
2298
|
+
id,
|
|
2299
|
+
deduplicated: true,
|
|
2300
|
+
metadata: existing.metadata
|
|
2301
|
+
};
|
|
2302
|
+
}
|
|
2303
|
+
let contentPresent = false;
|
|
2304
|
+
for (const child of children) {
|
|
2305
|
+
if (!child.stat.isFile() || child.name !== "content" && !isAtomicTemporaryName(child.name, "content") && !isAtomicTemporaryName(child.name, "metadata.json")) unsafe();
|
|
2306
|
+
if (child.name === "content") {
|
|
2307
|
+
if (child.stat.size !== BigInt(content.byteLength)) throw new LlmWikiError("INVALID_PAGE", "Source record content does not match its immutable identity.");
|
|
2308
|
+
const read = await directory.read(child.name, signal);
|
|
2309
|
+
if (!read || !sameFileSnapshot(child.stat, read.snapshot.stat)) unsafe();
|
|
2310
|
+
if (hash(read.bytes) !== id || !read.bytes.equals(content)) throw new LlmWikiError("INVALID_PAGE", "Source record content does not match its immutable identity.");
|
|
2311
|
+
contentPresent = true;
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
const currentChildren = await directory.list(signal);
|
|
2315
|
+
if (currentChildren.length !== children.length || currentChildren.some((child, index) => {
|
|
2316
|
+
const previous = children[index];
|
|
2317
|
+
return child.name !== previous?.name || !sameFileSnapshot(child.stat, previous.stat);
|
|
2318
|
+
})) unsafe();
|
|
2319
|
+
for (const child of children) if (!await directory.validate(child, signal)) unsafe();
|
|
2320
|
+
throwIfAborted(signal);
|
|
2321
|
+
for (const child of children) if (child.name !== "content") {
|
|
2322
|
+
throwIfAborted(signal);
|
|
2323
|
+
await directory.unlink(child.name, child.stat);
|
|
2324
|
+
}
|
|
2325
|
+
const metadata = {
|
|
2326
|
+
id,
|
|
2327
|
+
name: input.name,
|
|
2328
|
+
mediaType,
|
|
2329
|
+
byteCount: content.byteLength,
|
|
2330
|
+
capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2331
|
+
...input.origin === void 0 ? {} : { origin: input.origin }
|
|
2332
|
+
};
|
|
2333
|
+
let ownedContent;
|
|
2334
|
+
try {
|
|
2335
|
+
if (!contentPresent) {
|
|
2336
|
+
await atomicWriteFile(directory, "content", content, { signal });
|
|
2337
|
+
const written = await directory.read("content", signal);
|
|
2338
|
+
if (!written?.bytes.equals(content)) unsafe();
|
|
2339
|
+
ownedContent = written.snapshot;
|
|
2340
|
+
}
|
|
2341
|
+
throwIfAborted(signal);
|
|
2342
|
+
await atomicWriteFile(directory, "metadata.json", canonicalJson(metadata), { signal });
|
|
2343
|
+
return {
|
|
2344
|
+
id,
|
|
2345
|
+
deduplicated: false,
|
|
2346
|
+
metadata
|
|
2347
|
+
};
|
|
2348
|
+
} catch (cause) {
|
|
2349
|
+
if (ownedContent) await directory.unlink("content", ownedContent.stat).catch(() => void 0);
|
|
2350
|
+
throw cause;
|
|
2351
|
+
}
|
|
2352
|
+
});
|
|
2353
|
+
if (result === null) unsafe();
|
|
2354
|
+
return result;
|
|
2355
|
+
} catch (cause) {
|
|
2356
|
+
if (allocation.created) await sources.removeCreatedSource(id, allocation.identity, []).catch(() => void 0);
|
|
2357
|
+
throw cause;
|
|
2358
|
+
}
|
|
2359
|
+
});
|
|
2360
|
+
if (result === null) unsafe();
|
|
2361
|
+
return result;
|
|
2362
|
+
}, signal);
|
|
2363
|
+
}
|
|
2364
|
+
async readSource(id, range, signal) {
|
|
2365
|
+
sourceId(id);
|
|
2366
|
+
return this.enqueue(async (paths) => {
|
|
2367
|
+
const record = await this.readSourceRecord(paths, id, signal);
|
|
2368
|
+
const offset = range?.offset ?? 0;
|
|
2369
|
+
const limitValue = range?.limit ?? this[configKey].maxSourceBytes;
|
|
2370
|
+
if (!Number.isSafeInteger(offset) || offset < 0 || offset > record.content.byteLength) throw limit("Source byte offset is outside the source.");
|
|
2371
|
+
if (!Number.isSafeInteger(limitValue) || limitValue < 1 || limitValue > this[configKey].maxSourceBytes) throw limit("Source byte limit is outside the configured range.");
|
|
2372
|
+
const { start, end } = alignedRange(record.content, offset, limitValue);
|
|
2373
|
+
if (offset < record.content.byteLength && end === start) throw limit(INCOMPLETE_UTF8_RANGE);
|
|
2374
|
+
return {
|
|
2375
|
+
id,
|
|
2376
|
+
content: decodeUtf8(record.content.subarray(start, end)),
|
|
2377
|
+
metadata: record.metadata,
|
|
2378
|
+
byteStart: start,
|
|
2379
|
+
byteEnd: end,
|
|
2380
|
+
byteCount: record.content.byteLength
|
|
2381
|
+
};
|
|
2382
|
+
}, signal);
|
|
2383
|
+
}
|
|
2384
|
+
async listSources(request = {}, signal) {
|
|
2385
|
+
throwIfAborted(signal);
|
|
2386
|
+
const limitValue = catalogLimit(request, this[configKey].maxResults);
|
|
2387
|
+
const after = decodeCursor(request.cursor, "sources");
|
|
2388
|
+
return this.enqueue(async (paths) => {
|
|
2389
|
+
if (!wikiRootPresent(paths, signal) || !await regularDirectory(paths.sources, paths, signal)) return {
|
|
2390
|
+
items: [],
|
|
2391
|
+
nextCursor: null
|
|
2392
|
+
};
|
|
2393
|
+
const items = [];
|
|
2394
|
+
const snapshots = [];
|
|
2395
|
+
let hasMore = false;
|
|
2396
|
+
const root = await readSafeCatalogDirectory(paths.sources, paths, signal);
|
|
2397
|
+
throwIfAborted(signal);
|
|
2398
|
+
snapshots.push(root.snapshot);
|
|
2399
|
+
const discovered = root.entries.sort((a, b) => compareCodeUnits(a.name, b.name));
|
|
2400
|
+
for (const entry of discovered) {
|
|
2401
|
+
throwIfAborted(signal);
|
|
2402
|
+
if (entry.isSymbolicLink()) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Symbolic links are not allowed in the wiki.");
|
|
2403
|
+
if (!entry.isDirectory()) throw new LlmWikiError("UNSAFE_FILESYSTEM", "The sources tree may contain only regular source directories.");
|
|
2404
|
+
if (!isSourceId(entry.name)) throw catalogCorrupt("The source catalog contains an invalid record.");
|
|
2405
|
+
const id = sourceId(entry.name);
|
|
2406
|
+
const recordPath = join(paths.sources, entry.name);
|
|
2407
|
+
const recordDirectory = await readSafeCatalogDirectory(recordPath, paths, signal);
|
|
2408
|
+
throwIfAborted(signal);
|
|
2409
|
+
snapshots.push(recordDirectory.snapshot);
|
|
2410
|
+
const children = recordDirectory.entries.sort((a, b) => compareCodeUnits(a.name, b.name));
|
|
2411
|
+
for (const child of children) {
|
|
2412
|
+
throwIfAborted(signal);
|
|
2413
|
+
if (child.isSymbolicLink() || !child.isFile()) throw new LlmWikiError("UNSAFE_FILESYSTEM", "Source records must contain only regular files.");
|
|
2414
|
+
}
|
|
2415
|
+
if (children.length !== 2 || children[0]?.name !== "content" || children[1]?.name !== "metadata.json") throw catalogCorrupt("The source catalog contains an invalid record.");
|
|
2416
|
+
const contentSnapshot = await snapshotSafeCatalogFile(join(recordPath, "content"), paths, signal);
|
|
2417
|
+
throwIfAborted(signal);
|
|
2418
|
+
const metadataSnapshot = await snapshotSafeCatalogFile(join(recordPath, "metadata.json"), paths, signal);
|
|
2419
|
+
throwIfAborted(signal);
|
|
2420
|
+
snapshots.push(contentSnapshot, metadataSnapshot);
|
|
2421
|
+
try {
|
|
2422
|
+
const record = await this.readSourceRecord(paths, id, signal);
|
|
2423
|
+
throwIfAborted(signal);
|
|
2424
|
+
hasMore ||= appendCatalogItem(items, record.metadata.id, after, limitValue, () => ({
|
|
2425
|
+
id: record.metadata.id,
|
|
2426
|
+
name: record.metadata.name,
|
|
2427
|
+
mediaType: record.metadata.mediaType,
|
|
2428
|
+
byteCount: record.metadata.byteCount,
|
|
2429
|
+
capturedAt: record.metadata.capturedAt,
|
|
2430
|
+
...record.metadata.origin === void 0 ? {} : { origin: record.metadata.origin }
|
|
2431
|
+
}));
|
|
2432
|
+
} catch (cause) {
|
|
2433
|
+
if (cause instanceof LlmWikiError && (cause.code === "ABORTED" || cause.code === "UNSAFE_FILESYSTEM")) throw cause;
|
|
2434
|
+
if (!isMissing(cause) && typeof cause.code === "string" && !(cause instanceof LlmWikiError)) throw cause;
|
|
2435
|
+
await revalidateCatalogSnapshot(recordDirectory.snapshot, paths, signal);
|
|
2436
|
+
throwIfAborted(signal);
|
|
2437
|
+
await revalidateCatalogSnapshot(contentSnapshot, paths, signal);
|
|
2438
|
+
throwIfAborted(signal);
|
|
2439
|
+
await revalidateCatalogSnapshot(metadataSnapshot, paths, signal);
|
|
2440
|
+
throwIfAborted(signal);
|
|
2441
|
+
throw catalogCorrupt("The source catalog contains an invalid record.");
|
|
2442
|
+
}
|
|
2443
|
+
}
|
|
2444
|
+
for (const snapshot of snapshots) {
|
|
2445
|
+
throwIfAborted(signal);
|
|
2446
|
+
await revalidateCatalogSnapshot(snapshot, paths, signal);
|
|
2447
|
+
throwIfAborted(signal);
|
|
2448
|
+
}
|
|
2449
|
+
return finishCatalogPage("sources", items, hasMore);
|
|
2450
|
+
}, signal, false);
|
|
2451
|
+
}
|
|
2452
|
+
async listPages(request = {}, signal) {
|
|
2453
|
+
throwIfAborted(signal);
|
|
2454
|
+
const limitValue = catalogLimit(request, this[configKey].maxResults);
|
|
2455
|
+
const after = decodeCursor(request.cursor, "pages");
|
|
2456
|
+
return this.enqueue(async (paths) => {
|
|
2457
|
+
if (!wikiRootPresent(paths, signal) || !await regularDirectory(paths.pages, paths, signal)) return {
|
|
2458
|
+
items: [],
|
|
2459
|
+
nextCursor: null
|
|
2460
|
+
};
|
|
2461
|
+
const files = [];
|
|
2462
|
+
const snapshots = [];
|
|
2463
|
+
await collectCatalogPageFiles(paths.pages, paths, files, snapshots, signal);
|
|
2464
|
+
throwIfAborted(signal);
|
|
2465
|
+
files.sort((left, right) => compareCodeUnits(left.logicalPath, right.logicalPath));
|
|
2466
|
+
const items = [];
|
|
2467
|
+
let hasMore = false;
|
|
2468
|
+
for (const file of files) {
|
|
2469
|
+
throwIfAborted(signal);
|
|
2470
|
+
const logical = file.logicalPath;
|
|
2471
|
+
if (!logical.endsWith(".md") || !isPageId(logical.slice(0, -3))) {
|
|
2472
|
+
for (const snapshot of snapshots) {
|
|
2473
|
+
throwIfAborted(signal);
|
|
2474
|
+
await revalidateCatalogSnapshot(snapshot, paths, signal);
|
|
2475
|
+
throwIfAborted(signal);
|
|
2476
|
+
}
|
|
2477
|
+
throw catalogCorrupt("The page catalog contains an invalid record.");
|
|
2478
|
+
}
|
|
2479
|
+
try {
|
|
2480
|
+
const read = await readSafeCatalogFile(file.path, paths, signal);
|
|
2481
|
+
throwIfAborted(signal);
|
|
2482
|
+
if (!sameStableSnapshot(file.snapshot.stats, read.snapshot.stats)) throw new LlmWikiError("UNSAFE_FILESYSTEM", "The catalog changed while it was being read.");
|
|
2483
|
+
const parsed = parsePageMarkdown(decodeUtf8(read.bytes));
|
|
2484
|
+
const id = pageId(logical.slice(0, -3));
|
|
2485
|
+
hasMore ||= appendCatalogItem(items, id, after, limitValue, () => ({
|
|
2486
|
+
id,
|
|
2487
|
+
title: parsed.metadata.title,
|
|
2488
|
+
summary: parsed.metadata.summary,
|
|
2489
|
+
sources: [...parsed.metadata.sources],
|
|
2490
|
+
byteCount: read.bytes.byteLength,
|
|
2491
|
+
sha256: hash(read.bytes)
|
|
2492
|
+
}));
|
|
2493
|
+
} catch (cause) {
|
|
2494
|
+
if (cause instanceof LlmWikiError && (cause.code === "ABORTED" || cause.code === "UNSAFE_FILESYSTEM")) throw cause;
|
|
2495
|
+
if (!isMissing(cause) && typeof cause.code === "string" && !(cause instanceof LlmWikiError)) throw cause;
|
|
2496
|
+
await revalidateCatalogSnapshot(file.snapshot, paths, signal);
|
|
2497
|
+
throwIfAborted(signal);
|
|
2498
|
+
throw catalogCorrupt("The page catalog contains an invalid record.");
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
for (const snapshot of snapshots) {
|
|
2502
|
+
throwIfAborted(signal);
|
|
2503
|
+
await revalidateCatalogSnapshot(snapshot, paths, signal);
|
|
2504
|
+
throwIfAborted(signal);
|
|
2505
|
+
}
|
|
2506
|
+
return finishCatalogPage("pages", items, hasMore);
|
|
2507
|
+
}, signal, false);
|
|
2508
|
+
}
|
|
2509
|
+
async readSourceRecord(paths, id, signal) {
|
|
2510
|
+
try {
|
|
2511
|
+
const contentPath = paths.sourceContent(id);
|
|
2512
|
+
const metadataPath = paths.sourceMetadata(id);
|
|
2513
|
+
const contentPresent = await regularFile(contentPath, paths, signal);
|
|
2514
|
+
throwIfAborted(signal);
|
|
2515
|
+
const metadataPresent = await regularFile(metadataPath, paths, signal);
|
|
2516
|
+
throwIfAborted(signal);
|
|
2517
|
+
if (!contentPresent || !metadataPresent) throw missing("SOURCE_NOT_FOUND", "Source was not found.");
|
|
2518
|
+
const metadataRead = await readSafeCatalogFile(metadataPath, paths, signal);
|
|
2519
|
+
throwIfAborted(signal);
|
|
2520
|
+
const metadata = parseMetadata(metadataRead.bytes, id);
|
|
2521
|
+
const contentRead = await readSafeCatalogFile(contentPath, paths, signal);
|
|
2522
|
+
throwIfAborted(signal);
|
|
2523
|
+
const snapshots = [metadataRead.snapshot, contentRead.snapshot];
|
|
2524
|
+
try {
|
|
2525
|
+
if (hash(contentRead.bytes) !== id || metadata.byteCount !== contentRead.bytes.byteLength) throw new LlmWikiError("INVALID_PAGE", "Source record content does not match its immutable identity.");
|
|
2526
|
+
decodeUtf8(contentRead.bytes);
|
|
2527
|
+
return {
|
|
2528
|
+
content: contentRead.bytes,
|
|
2529
|
+
metadata,
|
|
2530
|
+
snapshots
|
|
2531
|
+
};
|
|
2532
|
+
} catch (cause) {
|
|
2533
|
+
for (const snapshot of snapshots) {
|
|
2534
|
+
throwIfAborted(signal);
|
|
2535
|
+
await revalidateCatalogSnapshot(snapshot, paths, signal);
|
|
2536
|
+
throwIfAborted(signal);
|
|
2537
|
+
}
|
|
2538
|
+
throw cause;
|
|
2539
|
+
}
|
|
2540
|
+
} catch (cause) {
|
|
2541
|
+
throwIfAborted(signal);
|
|
2542
|
+
if (isMissing(cause)) throw missing("SOURCE_NOT_FOUND", "Source was not found.");
|
|
2543
|
+
throw cause;
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
async readPage(id, signal) {
|
|
2547
|
+
pageId(id);
|
|
2548
|
+
return this.enqueue(async (paths) => {
|
|
2549
|
+
try {
|
|
2550
|
+
const target = paths.page(id);
|
|
2551
|
+
if (!await regularFile(target, paths, signal)) throw missing("PAGE_NOT_FOUND", "Page was not found.");
|
|
2552
|
+
const bytes = await readBytes(paths, target, signal);
|
|
2553
|
+
if (bytes === null) throw missing("PAGE_NOT_FOUND", "Page was not found.");
|
|
2554
|
+
throwIfAborted(signal);
|
|
2555
|
+
const markdown = decodeUtf8(bytes);
|
|
2556
|
+
return {
|
|
2557
|
+
id,
|
|
2558
|
+
markdown,
|
|
2559
|
+
metadata: parsePageMarkdown(markdown).metadata,
|
|
2560
|
+
sha256: hash(bytes)
|
|
2561
|
+
};
|
|
2562
|
+
} catch (cause) {
|
|
2563
|
+
throwIfAborted(signal);
|
|
2564
|
+
if (isMissing(cause)) throw missing("PAGE_NOT_FOUND", "Page was not found.");
|
|
2565
|
+
throw cause;
|
|
2566
|
+
}
|
|
2567
|
+
}, signal);
|
|
2568
|
+
}
|
|
2569
|
+
async upsertPage(input, signal) {
|
|
2570
|
+
if (this.disposed) return Promise.reject(new LlmWikiError("NOT_INITIALIZED", "The llmwiki service has been disposed."));
|
|
2571
|
+
throwIfAborted(signal);
|
|
2572
|
+
const expectedSha256 = input.expectedSha256;
|
|
2573
|
+
if (expectedSha256 !== null && (typeof expectedSha256 !== "string" || !HASH.test(expectedSha256))) throw new LlmWikiError("INVALID_PRECONDITION", "expectedSha256 must be null or a 64-character lowercase hexadecimal SHA-256 hash.");
|
|
2574
|
+
pageId(input.id);
|
|
2575
|
+
return this.enqueue(async (paths) => {
|
|
2576
|
+
const bytes = encodeUtf8(renderPageMarkdown(input, input.body));
|
|
2577
|
+
if (bytes.byteLength > this[configKey].maxPageBytes) throw limit("Page content exceeds maxPageBytes.");
|
|
2578
|
+
for (const id of input.sources) await this.readSourceRecord(paths, id, signal);
|
|
2579
|
+
const segments = wikiRelativeSegments(paths, paths.page(input.id));
|
|
2580
|
+
const receipt = await paths.rootDirectory.directory(segments.slice(0, -1), {
|
|
2581
|
+
create: expectedSha256 === null,
|
|
2582
|
+
signal
|
|
2583
|
+
}, async (directory) => {
|
|
2584
|
+
for (const id of input.sources) await this.readSourceRecord(paths, id, signal);
|
|
2585
|
+
const existing = await directory.read(segments.at(-1), signal);
|
|
2586
|
+
if (expectedSha256 === null ? existing !== null : existing === null || hash(existing.bytes) !== expectedSha256) throw new LlmWikiError("PAGE_CONFLICT", "Page does not match expectedSha256; reread and reconcile before writing.");
|
|
2587
|
+
await atomicWriteFile(directory, segments.at(-1), bytes, { signal });
|
|
2588
|
+
return {
|
|
2589
|
+
id: input.id,
|
|
2590
|
+
created: existing === null,
|
|
2591
|
+
sha256: hash(bytes)
|
|
2592
|
+
};
|
|
2593
|
+
});
|
|
2594
|
+
if (receipt === null) {
|
|
2595
|
+
if (expectedSha256 !== null) throw new LlmWikiError("PAGE_CONFLICT", "Page does not match expectedSha256; reread and reconcile before writing.");
|
|
2596
|
+
unsafe();
|
|
2597
|
+
}
|
|
2598
|
+
return receipt;
|
|
2599
|
+
}, signal);
|
|
2600
|
+
}
|
|
2601
|
+
async lint(signal) {
|
|
2602
|
+
return this.enqueue((paths) => lintWiki(paths, signal, paths.rootDirectory), signal, false);
|
|
2603
|
+
}
|
|
2604
|
+
async search(query, limitValue = this[configKey].maxResults, signal) {
|
|
2605
|
+
throwIfAborted(signal);
|
|
2606
|
+
if (!Number.isSafeInteger(limitValue) || limitValue < 1 || !Number.isSafeInteger(this[configKey].maxResults) || this[configKey].maxResults < 1 || !Number.isSafeInteger(this[configKey].maxSnippetBytes) || this[configKey].maxSnippetBytes < 1) throw limit("Search limits must be positive safe integers.");
|
|
2607
|
+
if (tokenize(query).length === 0) throw new LlmWikiError("INVALID_PAGE", "Search query must contain at least one Unicode letter or number.");
|
|
2608
|
+
return this.enqueue(async (paths) => {
|
|
2609
|
+
return [...searchBuiltIndex(await this.ensureIndex(paths, signal), query, {
|
|
2610
|
+
limit: limitValue,
|
|
2611
|
+
maxResults: this[configKey].maxResults,
|
|
2612
|
+
maxSnippetBytes: this[configKey].maxSnippetBytes,
|
|
2613
|
+
...signal === void 0 ? {} : { signal }
|
|
2614
|
+
})];
|
|
2615
|
+
}, signal);
|
|
2616
|
+
}
|
|
2617
|
+
async reindex(signal) {
|
|
2618
|
+
return this.enqueue(async (paths) => {
|
|
2619
|
+
await countFiles(paths.pages, ".md", paths, signal);
|
|
2620
|
+
await this.indexTargetPresence(paths, signal);
|
|
2621
|
+
const built = await buildSearchIndex(paths, signal, void 0, paths.rootDirectory);
|
|
2622
|
+
await writeIndex(paths, built, signal, paths.rootDirectory);
|
|
2623
|
+
return {
|
|
2624
|
+
pageCount: built.search.pageFingerprints.length,
|
|
2625
|
+
sectionCount: built.search.sections.length,
|
|
2626
|
+
formatVersion: built.search.formatVersion
|
|
2627
|
+
};
|
|
2628
|
+
}, signal);
|
|
2629
|
+
}
|
|
2630
|
+
async indexTargetPresence(paths, signal) {
|
|
2631
|
+
const [searchPresent, statePresent] = await allSettledOnFailure([regularFile(paths.indexFile("search.json"), paths, signal), regularFile(paths.indexFile("state.json"), paths, signal)]);
|
|
2632
|
+
return [searchPresent, statePresent];
|
|
2633
|
+
}
|
|
2634
|
+
async ensureIndex(paths, signal) {
|
|
2635
|
+
await countFiles(paths.pages, ".md", paths, signal);
|
|
2636
|
+
const expected = await buildSearchIndex(paths, signal, void 0, paths.rootDirectory);
|
|
2637
|
+
const [searchPresent, statePresent] = await this.indexTargetPresence(paths, signal);
|
|
2638
|
+
if (searchPresent && statePresent) try {
|
|
2639
|
+
const [searchBytes, stateBytes] = await allSettledOnFailure([readBytes(paths, paths.indexFile("search.json"), signal), readBytes(paths, paths.indexFile("state.json"), signal)]);
|
|
2640
|
+
if (searchBytes === null || stateBytes === null) unsafe();
|
|
2641
|
+
const search = trustedSearchIndex(searchBytes, stateBytes, expected);
|
|
2642
|
+
await validateBuiltIndexSnapshot(paths, expected, signal, paths.rootDirectory);
|
|
2643
|
+
if (search !== null) return search;
|
|
2644
|
+
} catch (cause) {
|
|
2645
|
+
throwIfAborted(signal);
|
|
2646
|
+
if (!(cause instanceof LlmWikiError && cause.code === "INDEX_CORRUPT") && !isMissing(cause)) throw cause;
|
|
2647
|
+
}
|
|
2648
|
+
await this.indexTargetPresence(paths, signal);
|
|
2649
|
+
await writeIndex(paths, expected, signal, paths.rootDirectory);
|
|
2650
|
+
return expected.search;
|
|
2651
|
+
}
|
|
2652
|
+
async indexStatus(paths, signal) {
|
|
2653
|
+
const [searchPresent, statePresent] = await this.indexTargetPresence(paths, signal);
|
|
2654
|
+
if (!searchPresent && !statePresent) return EMPTY_INDEX_STATUS;
|
|
2655
|
+
if (!searchPresent || !statePresent) return {
|
|
2656
|
+
present: true,
|
|
2657
|
+
fresh: false,
|
|
2658
|
+
formatVersion: null,
|
|
2659
|
+
sectionCount: 0
|
|
2660
|
+
};
|
|
2661
|
+
try {
|
|
2662
|
+
const [searchBytes, stateBytes] = await allSettledOnFailure([readBytes(paths, paths.indexFile("search.json"), signal), readBytes(paths, paths.indexFile("state.json"), signal)]);
|
|
2663
|
+
if (searchBytes === null || stateBytes === null) unsafe();
|
|
2664
|
+
parseSearchIndex(searchBytes);
|
|
2665
|
+
parseIndexState(stateBytes);
|
|
2666
|
+
let expected;
|
|
2667
|
+
try {
|
|
2668
|
+
expected = await buildSearchIndex(paths, signal, void 0, paths.rootDirectory);
|
|
2669
|
+
} catch (cause) {
|
|
2670
|
+
throwIfAborted(signal);
|
|
2671
|
+
if (cause instanceof LlmWikiError && (cause.code === "INVALID_PAGE" || cause.code === "INVALID_PATH")) return {
|
|
2672
|
+
present: true,
|
|
2673
|
+
fresh: false,
|
|
2674
|
+
formatVersion: 1,
|
|
2675
|
+
sectionCount: 0
|
|
2676
|
+
};
|
|
2677
|
+
throw cause;
|
|
2678
|
+
}
|
|
2679
|
+
const search = trustedSearchIndex(searchBytes, stateBytes, expected);
|
|
2680
|
+
await validateBuiltIndexSnapshot(paths, expected, signal, paths.rootDirectory);
|
|
2681
|
+
return {
|
|
2682
|
+
present: true,
|
|
2683
|
+
fresh: search !== null,
|
|
2684
|
+
formatVersion: 1,
|
|
2685
|
+
sectionCount: expected.search.sections.length
|
|
2686
|
+
};
|
|
2687
|
+
} catch (cause) {
|
|
2688
|
+
throwIfAborted(signal);
|
|
2689
|
+
if (isMissing(cause) || cause instanceof LlmWikiError && cause.code === "INDEX_CORRUPT") return {
|
|
2690
|
+
present: true,
|
|
2691
|
+
fresh: false,
|
|
2692
|
+
formatVersion: null,
|
|
2693
|
+
sectionCount: 0
|
|
2694
|
+
};
|
|
2695
|
+
throw cause;
|
|
2696
|
+
}
|
|
2697
|
+
}
|
|
2698
|
+
};
|
|
2699
|
+
//#endregion
|
|
2700
|
+
//#region src/presentation.ts
|
|
2701
|
+
const PRESENTATION = Object.freeze({
|
|
2702
|
+
llmwiki_status: {
|
|
2703
|
+
title: "Inspect wiki status",
|
|
2704
|
+
kind: "read"
|
|
2705
|
+
},
|
|
2706
|
+
llmwiki_add_source: {
|
|
2707
|
+
title: "Preserve wiki source",
|
|
2708
|
+
kind: "edit"
|
|
2709
|
+
},
|
|
2710
|
+
llmwiki_list_sources: {
|
|
2711
|
+
title: "List wiki sources",
|
|
2712
|
+
kind: "read"
|
|
2713
|
+
},
|
|
2714
|
+
llmwiki_read_source: {
|
|
2715
|
+
title: "Read wiki source",
|
|
2716
|
+
kind: "read"
|
|
2717
|
+
},
|
|
2718
|
+
llmwiki_search: {
|
|
2719
|
+
title: "Search wiki",
|
|
2720
|
+
kind: "search"
|
|
2721
|
+
},
|
|
2722
|
+
llmwiki_list_pages: {
|
|
2723
|
+
title: "List wiki pages",
|
|
2724
|
+
kind: "read"
|
|
2725
|
+
},
|
|
2726
|
+
llmwiki_read_page: {
|
|
2727
|
+
title: "Read wiki page",
|
|
2728
|
+
kind: "read"
|
|
2729
|
+
},
|
|
2730
|
+
llmwiki_upsert_page: {
|
|
2731
|
+
title: "Update wiki page",
|
|
2732
|
+
kind: "edit"
|
|
2733
|
+
},
|
|
2734
|
+
llmwiki_lint: {
|
|
2735
|
+
title: "Lint wiki",
|
|
2736
|
+
kind: "read"
|
|
2737
|
+
}
|
|
2738
|
+
});
|
|
2739
|
+
function salientInput(name, args) {
|
|
2740
|
+
switch (name) {
|
|
2741
|
+
case "llmwiki_add_source": return typeof args.name === "string" ? args.name : void 0;
|
|
2742
|
+
case "llmwiki_read_source": return typeof args.id === "string" ? args.id : void 0;
|
|
2743
|
+
case "llmwiki_search": return typeof args.query === "string" ? args.query : void 0;
|
|
2744
|
+
case "llmwiki_read_page":
|
|
2745
|
+
case "llmwiki_upsert_page": return typeof args.id === "string" ? args.id : void 0;
|
|
2746
|
+
default: return;
|
|
2747
|
+
}
|
|
2748
|
+
}
|
|
2749
|
+
function presentLlmWikiCall(name, args) {
|
|
2750
|
+
const presentation = PRESENTATION[name];
|
|
2751
|
+
return {
|
|
2752
|
+
card: "generic",
|
|
2753
|
+
title: presentation.title,
|
|
2754
|
+
kind: presentation.kind,
|
|
2755
|
+
...salientInput(name, args) === void 0 ? {} : { rawInput: salientInput(name, args) }
|
|
2756
|
+
};
|
|
2757
|
+
}
|
|
2758
|
+
function presentLlmWikiResult(name, _args, result) {
|
|
2759
|
+
const presentation = PRESENTATION[name];
|
|
2760
|
+
return {
|
|
2761
|
+
card: "generic",
|
|
2762
|
+
title: result.isError ? `${presentation.title} failed` : presentation.title
|
|
2763
|
+
};
|
|
2764
|
+
}
|
|
2765
|
+
//#endregion
|
|
2766
|
+
//#region src/tools.ts
|
|
2767
|
+
const requiredString = () => ({
|
|
2768
|
+
type: "string",
|
|
2769
|
+
required: true
|
|
2770
|
+
});
|
|
2771
|
+
const requiredInteger = () => ({
|
|
2772
|
+
type: "integer",
|
|
2773
|
+
required: true
|
|
2774
|
+
});
|
|
2775
|
+
const requiredNumber = () => ({
|
|
2776
|
+
type: "number",
|
|
2777
|
+
required: true
|
|
2778
|
+
});
|
|
2779
|
+
const requiredBoolean = () => ({
|
|
2780
|
+
type: "boolean",
|
|
2781
|
+
required: true
|
|
2782
|
+
});
|
|
2783
|
+
const requiredStringArray = () => ({
|
|
2784
|
+
type: "array",
|
|
2785
|
+
required: true,
|
|
2786
|
+
items: { type: "string" }
|
|
2787
|
+
});
|
|
2788
|
+
const closed = (properties) => ({
|
|
2789
|
+
type: "object",
|
|
2790
|
+
additionalProperties: false,
|
|
2791
|
+
properties
|
|
2792
|
+
});
|
|
2793
|
+
const requiredClosed = (properties) => ({
|
|
2794
|
+
...closed(properties),
|
|
2795
|
+
required: true
|
|
2796
|
+
});
|
|
2797
|
+
const requiredNullableString = {
|
|
2798
|
+
required: true,
|
|
2799
|
+
oneOf: [{ type: "string" }, { type: "null" }]
|
|
2800
|
+
};
|
|
2801
|
+
const requiredNullableInteger = {
|
|
2802
|
+
required: true,
|
|
2803
|
+
oneOf: [{ type: "integer" }, { type: "null" }]
|
|
2804
|
+
};
|
|
2805
|
+
const optionalNullableString = { oneOf: [{ type: "string" }, { type: "null" }] };
|
|
2806
|
+
const optionalString = { type: "string" };
|
|
2807
|
+
const catalogParameters = {
|
|
2808
|
+
limit: {
|
|
2809
|
+
type: "integer",
|
|
2810
|
+
description: "Optional catalog page size; defaults to deployment maxResults and is bounded by it."
|
|
2811
|
+
},
|
|
2812
|
+
cursor: {
|
|
2813
|
+
type: "string",
|
|
2814
|
+
description: "Optional opaque cursor returned by the same catalog tool."
|
|
2815
|
+
}
|
|
2816
|
+
};
|
|
2817
|
+
const sourceCatalogPageSchema = closed({
|
|
2818
|
+
items: {
|
|
2819
|
+
type: "array",
|
|
2820
|
+
required: true,
|
|
2821
|
+
items: closed({
|
|
2822
|
+
id: requiredString(),
|
|
2823
|
+
name: requiredString(),
|
|
2824
|
+
mediaType: requiredString(),
|
|
2825
|
+
byteCount: requiredInteger(),
|
|
2826
|
+
capturedAt: requiredString(),
|
|
2827
|
+
origin: optionalString
|
|
2828
|
+
})
|
|
2829
|
+
},
|
|
2830
|
+
nextCursor: requiredNullableString
|
|
2831
|
+
});
|
|
2832
|
+
const pageCatalogPageSchema = closed({
|
|
2833
|
+
items: {
|
|
2834
|
+
type: "array",
|
|
2835
|
+
required: true,
|
|
2836
|
+
items: closed({
|
|
2837
|
+
id: requiredString(),
|
|
2838
|
+
title: requiredString(),
|
|
2839
|
+
summary: requiredString(),
|
|
2840
|
+
sources: requiredStringArray(),
|
|
2841
|
+
byteCount: requiredInteger(),
|
|
2842
|
+
sha256: requiredString()
|
|
2843
|
+
})
|
|
2844
|
+
},
|
|
2845
|
+
nextCursor: requiredNullableString
|
|
2846
|
+
});
|
|
2847
|
+
const indexSchema = requiredClosed({
|
|
2848
|
+
present: requiredBoolean(),
|
|
2849
|
+
fresh: requiredBoolean(),
|
|
2850
|
+
formatVersion: requiredNullableInteger,
|
|
2851
|
+
sectionCount: requiredInteger()
|
|
2852
|
+
});
|
|
2853
|
+
const metadataSchema = requiredClosed({
|
|
2854
|
+
id: requiredString(),
|
|
2855
|
+
name: requiredString(),
|
|
2856
|
+
mediaType: requiredString(),
|
|
2857
|
+
byteCount: requiredInteger(),
|
|
2858
|
+
capturedAt: requiredString(),
|
|
2859
|
+
origin: optionalNullableString
|
|
2860
|
+
});
|
|
2861
|
+
const render = (_args, value) => [{
|
|
2862
|
+
type: "text",
|
|
2863
|
+
text: JSON.stringify(value)
|
|
2864
|
+
}];
|
|
2865
|
+
function stableFailure(cause) {
|
|
2866
|
+
if (isLlmWikiError(cause)) throw new LlmWikiError(cause.code, cause.message);
|
|
2867
|
+
throw cause;
|
|
2868
|
+
}
|
|
2869
|
+
async function call(operation) {
|
|
2870
|
+
try {
|
|
2871
|
+
return await operation();
|
|
2872
|
+
} catch (cause) {
|
|
2873
|
+
stableFailure(cause);
|
|
2874
|
+
}
|
|
2875
|
+
}
|
|
2876
|
+
function presentation(name) {
|
|
2877
|
+
return {
|
|
2878
|
+
presentCall: (args) => presentLlmWikiCall(name, args),
|
|
2879
|
+
presentResult: (args, result) => presentLlmWikiResult(name, args, result)
|
|
2880
|
+
};
|
|
2881
|
+
}
|
|
2882
|
+
function registerLlmWikiTools(ctx) {
|
|
2883
|
+
ctx.tools.register(defineTool({
|
|
2884
|
+
name: "llmwiki_status",
|
|
2885
|
+
description: "Discover wiki initialization, schema, durable record counts, and derived-index freshness before relying on wiki knowledge. Read-only; no approval or filesystem capability is requested.",
|
|
2886
|
+
parameters: {},
|
|
2887
|
+
output: {
|
|
2888
|
+
schema: closed({
|
|
2889
|
+
initialized: requiredBoolean(),
|
|
2890
|
+
sourceCount: requiredInteger(),
|
|
2891
|
+
pageCount: requiredInteger(),
|
|
2892
|
+
schemaText: requiredNullableString,
|
|
2893
|
+
index: indexSchema
|
|
2894
|
+
}),
|
|
2895
|
+
render
|
|
2896
|
+
},
|
|
2897
|
+
execute: (_args, exec) => call(async () => {
|
|
2898
|
+
const status = await ctx.llmwiki.status(exec.signal);
|
|
2899
|
+
return {
|
|
2900
|
+
initialized: status.initialized,
|
|
2901
|
+
sourceCount: status.sourceCount,
|
|
2902
|
+
pageCount: status.pageCount,
|
|
2903
|
+
schemaText: status.schemaText,
|
|
2904
|
+
index: {
|
|
2905
|
+
present: status.index.present,
|
|
2906
|
+
fresh: status.index.fresh,
|
|
2907
|
+
formatVersion: status.index.formatVersion,
|
|
2908
|
+
sectionCount: status.index.sectionCount
|
|
2909
|
+
}
|
|
2910
|
+
};
|
|
2911
|
+
}),
|
|
2912
|
+
isConcurrencySafe: () => true,
|
|
2913
|
+
...presentation("llmwiki_status")
|
|
2914
|
+
}));
|
|
2915
|
+
ctx.tools.register(defineTool({
|
|
2916
|
+
name: "llmwiki_add_source",
|
|
2917
|
+
description: "Preserve exact UTF-8 evidence supplied in content as an immutable source record. This mutates wiki storage and should be used only with authorized evidence; it cannot read host paths. Completion returns the durable source artifact ID and dedupe state.",
|
|
2918
|
+
parameters: {
|
|
2919
|
+
name: {
|
|
2920
|
+
type: "string",
|
|
2921
|
+
required: true,
|
|
2922
|
+
description: "Non-empty human-readable source name."
|
|
2923
|
+
},
|
|
2924
|
+
content: {
|
|
2925
|
+
type: "string",
|
|
2926
|
+
required: true,
|
|
2927
|
+
description: "Exact UTF-8 source content containing at least one byte; whitespace-only content is valid; bounded by deployment maxSourceBytes."
|
|
2928
|
+
},
|
|
2929
|
+
mediaType: {
|
|
2930
|
+
type: "string",
|
|
2931
|
+
description: "Optional non-empty media type; defaults to UTF-8 plain text."
|
|
2932
|
+
},
|
|
2933
|
+
origin: {
|
|
2934
|
+
type: "string",
|
|
2935
|
+
description: "Optional provenance label or URL containing at least one non-whitespace character; never used as a host path."
|
|
2936
|
+
}
|
|
2937
|
+
},
|
|
2938
|
+
output: {
|
|
2939
|
+
schema: closed({
|
|
2940
|
+
id: requiredString(),
|
|
2941
|
+
deduplicated: requiredBoolean(),
|
|
2942
|
+
metadata: metadataSchema
|
|
2943
|
+
}),
|
|
2944
|
+
render
|
|
2945
|
+
},
|
|
2946
|
+
execute: (args, exec) => call(async () => {
|
|
2947
|
+
const receipt = await ctx.llmwiki.addSource({
|
|
2948
|
+
name: args.name,
|
|
2949
|
+
content: args.content,
|
|
2950
|
+
...args.mediaType === void 0 ? {} : { mediaType: args.mediaType },
|
|
2951
|
+
...args.origin === void 0 ? {} : { origin: args.origin }
|
|
2952
|
+
}, exec.signal);
|
|
2953
|
+
return {
|
|
2954
|
+
id: receipt.id,
|
|
2955
|
+
deduplicated: receipt.deduplicated,
|
|
2956
|
+
metadata: {
|
|
2957
|
+
id: receipt.metadata.id,
|
|
2958
|
+
name: receipt.metadata.name,
|
|
2959
|
+
mediaType: receipt.metadata.mediaType,
|
|
2960
|
+
byteCount: receipt.metadata.byteCount,
|
|
2961
|
+
capturedAt: receipt.metadata.capturedAt,
|
|
2962
|
+
...receipt.metadata.origin === void 0 ? {} : { origin: receipt.metadata.origin }
|
|
2963
|
+
}
|
|
2964
|
+
};
|
|
2965
|
+
}),
|
|
2966
|
+
...presentation("llmwiki_add_source")
|
|
2967
|
+
}));
|
|
2968
|
+
ctx.tools.register(defineTool({
|
|
2969
|
+
name: "llmwiki_list_sources",
|
|
2970
|
+
description: "List immutable source records in deterministic ID order for recovery and inventory. Read-only; returns bounded safe metadata without content or host paths.",
|
|
2971
|
+
parameters: catalogParameters,
|
|
2972
|
+
output: {
|
|
2973
|
+
schema: sourceCatalogPageSchema,
|
|
2974
|
+
render
|
|
2975
|
+
},
|
|
2976
|
+
execute: (args, exec) => call(async () => {
|
|
2977
|
+
const page = await ctx.llmwiki.listSources({
|
|
2978
|
+
...args.limit === void 0 ? {} : { limit: args.limit },
|
|
2979
|
+
...args.cursor === void 0 ? {} : { cursor: args.cursor }
|
|
2980
|
+
}, exec.signal);
|
|
2981
|
+
return {
|
|
2982
|
+
items: page.items.map((item) => ({
|
|
2983
|
+
id: item.id,
|
|
2984
|
+
name: item.name,
|
|
2985
|
+
mediaType: item.mediaType,
|
|
2986
|
+
byteCount: item.byteCount,
|
|
2987
|
+
capturedAt: item.capturedAt,
|
|
2988
|
+
...item.origin === void 0 ? {} : { origin: item.origin }
|
|
2989
|
+
})),
|
|
2990
|
+
nextCursor: page.nextCursor
|
|
2991
|
+
};
|
|
2992
|
+
}),
|
|
2993
|
+
isConcurrencySafe: () => true,
|
|
2994
|
+
...presentation("llmwiki_list_sources")
|
|
2995
|
+
}));
|
|
2996
|
+
ctx.tools.register(defineTool({
|
|
2997
|
+
name: "llmwiki_read_source",
|
|
2998
|
+
description: "Read immutable source evidence by exact source ID, optionally using a configured byte-bounded range, and return provenance metadata with the content artifact. First access can create the configured host root, wiki directories, and default schema outside DSH filesystem approval, even if the requested source is missing; existing source records are not modified. A non-EOF range must fit at least one complete UTF-8 code point or returns LIMIT_EXCEEDED instructing the caller to increase limit.",
|
|
2999
|
+
parameters: {
|
|
3000
|
+
id: {
|
|
3001
|
+
type: "string",
|
|
3002
|
+
required: true,
|
|
3003
|
+
description: "Exact 64-character lowercase hexadecimal source ID."
|
|
3004
|
+
},
|
|
3005
|
+
offset: {
|
|
3006
|
+
type: "integer",
|
|
3007
|
+
description: "Optional non-negative zero-based UTF-8 byte offset; defaults to 0."
|
|
3008
|
+
},
|
|
3009
|
+
limit: {
|
|
3010
|
+
type: "integer",
|
|
3011
|
+
description: "Optional positive maximum bytes; defaults to deployment maxSourceBytes and is bounded by it. Increase it if no complete UTF-8 code point fits."
|
|
3012
|
+
}
|
|
3013
|
+
},
|
|
3014
|
+
output: {
|
|
3015
|
+
schema: closed({
|
|
3016
|
+
id: requiredString(),
|
|
3017
|
+
content: requiredString(),
|
|
3018
|
+
metadata: metadataSchema,
|
|
3019
|
+
byteStart: requiredInteger(),
|
|
3020
|
+
byteEnd: requiredInteger(),
|
|
3021
|
+
byteCount: requiredInteger()
|
|
3022
|
+
}),
|
|
3023
|
+
render
|
|
3024
|
+
},
|
|
3025
|
+
execute: (args, exec) => call(async () => {
|
|
3026
|
+
const range = args.offset === void 0 && args.limit === void 0 ? void 0 : {
|
|
3027
|
+
...args.offset === void 0 ? {} : { offset: args.offset },
|
|
3028
|
+
...args.limit === void 0 ? {} : { limit: args.limit }
|
|
3029
|
+
};
|
|
3030
|
+
const source = await ctx.llmwiki.readSource(sourceId(args.id), range, exec.signal);
|
|
3031
|
+
return {
|
|
3032
|
+
id: source.id,
|
|
3033
|
+
content: source.content,
|
|
3034
|
+
metadata: {
|
|
3035
|
+
id: source.metadata.id,
|
|
3036
|
+
name: source.metadata.name,
|
|
3037
|
+
mediaType: source.metadata.mediaType,
|
|
3038
|
+
byteCount: source.metadata.byteCount,
|
|
3039
|
+
capturedAt: source.metadata.capturedAt,
|
|
3040
|
+
...source.metadata.origin === void 0 ? {} : { origin: source.metadata.origin }
|
|
3041
|
+
},
|
|
3042
|
+
byteStart: source.byteStart,
|
|
3043
|
+
byteEnd: source.byteEnd,
|
|
3044
|
+
byteCount: source.byteCount
|
|
3045
|
+
};
|
|
3046
|
+
}),
|
|
3047
|
+
isConcurrencySafe: () => true,
|
|
3048
|
+
...presentation("llmwiki_read_source")
|
|
3049
|
+
}));
|
|
3050
|
+
ctx.tools.register(defineTool({
|
|
3051
|
+
name: "llmwiki_search",
|
|
3052
|
+
description: "Search the derived section index first for ranked source-linked page-section matches. May rebuild only stale derived index artifacts; it does not alter durable pages or sources. Results are capped by deployment maxResults and snippets by maxSnippetBytes.",
|
|
3053
|
+
parameters: {
|
|
3054
|
+
query: {
|
|
3055
|
+
type: "string",
|
|
3056
|
+
required: true,
|
|
3057
|
+
description: "Non-empty Unicode lexical query containing at least one letter or number."
|
|
3058
|
+
},
|
|
3059
|
+
limit: {
|
|
3060
|
+
type: "integer",
|
|
3061
|
+
description: "Optional positive result cap up to 100, additionally bounded by deployment maxResults."
|
|
3062
|
+
}
|
|
3063
|
+
},
|
|
3064
|
+
output: {
|
|
3065
|
+
schema: {
|
|
3066
|
+
type: "array",
|
|
3067
|
+
items: closed({
|
|
3068
|
+
pageId: requiredString(),
|
|
3069
|
+
title: requiredString(),
|
|
3070
|
+
headingTrail: requiredStringArray(),
|
|
3071
|
+
startLine: requiredInteger(),
|
|
3072
|
+
score: requiredNumber(),
|
|
3073
|
+
snippet: requiredString(),
|
|
3074
|
+
sourceIds: requiredStringArray()
|
|
3075
|
+
})
|
|
3076
|
+
},
|
|
3077
|
+
render
|
|
3078
|
+
},
|
|
3079
|
+
execute: (args, exec) => call(async () => {
|
|
3080
|
+
return (await ctx.llmwiki.search(args.query, args.limit, exec.signal)).map((hit) => ({
|
|
3081
|
+
pageId: hit.pageId,
|
|
3082
|
+
title: hit.title,
|
|
3083
|
+
headingTrail: [...hit.headingTrail],
|
|
3084
|
+
startLine: hit.startLine,
|
|
3085
|
+
score: hit.score,
|
|
3086
|
+
snippet: hit.snippet,
|
|
3087
|
+
sourceIds: [...hit.sourceIds]
|
|
3088
|
+
}));
|
|
3089
|
+
}),
|
|
3090
|
+
...presentation("llmwiki_search")
|
|
3091
|
+
}));
|
|
3092
|
+
ctx.tools.register(defineTool({
|
|
3093
|
+
name: "llmwiki_list_pages",
|
|
3094
|
+
description: "List durable wiki pages in deterministic ID order for recovery and inventory. Read-only; returns bounded metadata and exact byte hashes without page content.",
|
|
3095
|
+
parameters: catalogParameters,
|
|
3096
|
+
output: {
|
|
3097
|
+
schema: pageCatalogPageSchema,
|
|
3098
|
+
render
|
|
3099
|
+
},
|
|
3100
|
+
execute: (args, exec) => call(async () => {
|
|
3101
|
+
const page = await ctx.llmwiki.listPages({
|
|
3102
|
+
...args.limit === void 0 ? {} : { limit: args.limit },
|
|
3103
|
+
...args.cursor === void 0 ? {} : { cursor: args.cursor }
|
|
3104
|
+
}, exec.signal);
|
|
3105
|
+
return {
|
|
3106
|
+
items: page.items.map((item) => ({
|
|
3107
|
+
id: item.id,
|
|
3108
|
+
title: item.title,
|
|
3109
|
+
summary: item.summary,
|
|
3110
|
+
sources: [...item.sources],
|
|
3111
|
+
byteCount: item.byteCount,
|
|
3112
|
+
sha256: item.sha256
|
|
3113
|
+
})),
|
|
3114
|
+
nextCursor: page.nextCursor
|
|
3115
|
+
};
|
|
3116
|
+
}),
|
|
3117
|
+
isConcurrencySafe: () => true,
|
|
3118
|
+
...presentation("llmwiki_list_pages")
|
|
3119
|
+
}));
|
|
3120
|
+
ctx.tools.register(defineTool({
|
|
3121
|
+
name: "llmwiki_read_page",
|
|
3122
|
+
description: "Read one synthesized source-linked wiki page by normalized logical page ID; inspect its cited immutable source records when evaluating claim support. First access can create the configured host root, wiki directories, and default schema outside DSH filesystem approval, even if the requested page is missing; existing pages are not modified.",
|
|
3123
|
+
parameters: { id: {
|
|
3124
|
+
type: "string",
|
|
3125
|
+
required: true,
|
|
3126
|
+
description: "Non-empty normalized POSIX page ID without a leading slash or .md suffix."
|
|
3127
|
+
} },
|
|
3128
|
+
output: {
|
|
3129
|
+
schema: closed({
|
|
3130
|
+
id: requiredString(),
|
|
3131
|
+
markdown: requiredString(),
|
|
3132
|
+
sha256: requiredString(),
|
|
3133
|
+
metadata: requiredClosed({
|
|
3134
|
+
title: requiredString(),
|
|
3135
|
+
summary: requiredString(),
|
|
3136
|
+
sources: requiredStringArray()
|
|
3137
|
+
})
|
|
3138
|
+
}),
|
|
3139
|
+
render
|
|
3140
|
+
},
|
|
3141
|
+
execute: (args, exec) => call(async () => {
|
|
3142
|
+
const page = await ctx.llmwiki.readPage(pageId(args.id), exec.signal);
|
|
3143
|
+
return {
|
|
3144
|
+
id: page.id,
|
|
3145
|
+
markdown: page.markdown,
|
|
3146
|
+
sha256: page.sha256,
|
|
3147
|
+
metadata: {
|
|
3148
|
+
title: page.metadata.title,
|
|
3149
|
+
summary: page.metadata.summary,
|
|
3150
|
+
sources: [...page.metadata.sources]
|
|
3151
|
+
}
|
|
3152
|
+
};
|
|
3153
|
+
}),
|
|
3154
|
+
isConcurrencySafe: () => true,
|
|
3155
|
+
...presentation("llmwiki_read_page")
|
|
3156
|
+
}));
|
|
3157
|
+
ctx.tools.register(defineTool({
|
|
3158
|
+
name: "llmwiki_upsert_page",
|
|
3159
|
+
description: "Atomically create or update a synthesized durable Markdown page with a required concurrency precondition: null creates only; the exact current raw-byte SHA-256 updates only. PAGE_CONFLICT requires rereading and reconciling, not automatic retry. This mutation requires existing preserved source IDs but does not verify claim-level support. Use only when the user request authorizes maintenance. Returns the page artifact ID, creation state, and content hash.",
|
|
3160
|
+
parameters: {
|
|
3161
|
+
id: {
|
|
3162
|
+
type: "string",
|
|
3163
|
+
required: true,
|
|
3164
|
+
description: "Non-empty normalized POSIX page ID without a leading slash or .md suffix."
|
|
3165
|
+
},
|
|
3166
|
+
expectedSha256: {
|
|
3167
|
+
...requiredNullableString,
|
|
3168
|
+
description: "Required: null for a new page, or the exact current lowercase SHA-256 from llmwiki_read_page or llmwiki_list_pages for an update. On PAGE_CONFLICT, reread and reconcile before another write."
|
|
3169
|
+
},
|
|
3170
|
+
title: {
|
|
3171
|
+
type: "string",
|
|
3172
|
+
required: true,
|
|
3173
|
+
description: "Non-empty concise page title."
|
|
3174
|
+
},
|
|
3175
|
+
summary: {
|
|
3176
|
+
type: "string",
|
|
3177
|
+
required: true,
|
|
3178
|
+
description: "Non-empty concise source-linked summary."
|
|
3179
|
+
},
|
|
3180
|
+
sources: {
|
|
3181
|
+
type: "array",
|
|
3182
|
+
required: true,
|
|
3183
|
+
items: { type: "string" },
|
|
3184
|
+
description: "Non-empty sorted unique 64-character lowercase hexadecimal IDs of existing immutable source records; never invent IDs. Existence is verified, semantic support is not."
|
|
3185
|
+
},
|
|
3186
|
+
body: {
|
|
3187
|
+
type: "string",
|
|
3188
|
+
required: true,
|
|
3189
|
+
description: "Non-empty Markdown body bounded by deployment maxPageBytes after canonical rendering."
|
|
3190
|
+
}
|
|
3191
|
+
},
|
|
3192
|
+
output: {
|
|
3193
|
+
schema: closed({
|
|
3194
|
+
id: requiredString(),
|
|
3195
|
+
created: requiredBoolean(),
|
|
3196
|
+
sha256: requiredString()
|
|
3197
|
+
}),
|
|
3198
|
+
render
|
|
3199
|
+
},
|
|
3200
|
+
execute: (args, exec) => call(async () => {
|
|
3201
|
+
const receipt = await ctx.llmwiki.upsertPage({
|
|
3202
|
+
id: pageId(args.id),
|
|
3203
|
+
expectedSha256: args.expectedSha256,
|
|
3204
|
+
title: args.title,
|
|
3205
|
+
summary: args.summary,
|
|
3206
|
+
sources: args.sources.map(sourceId),
|
|
3207
|
+
body: args.body
|
|
3208
|
+
}, exec.signal);
|
|
3209
|
+
return {
|
|
3210
|
+
id: receipt.id,
|
|
3211
|
+
created: receipt.created,
|
|
3212
|
+
sha256: receipt.sha256
|
|
3213
|
+
};
|
|
3214
|
+
}),
|
|
3215
|
+
...presentation("llmwiki_upsert_page")
|
|
3216
|
+
}));
|
|
3217
|
+
ctx.tools.register(defineTool({
|
|
3218
|
+
name: "llmwiki_lint",
|
|
3219
|
+
description: "Run deterministic model-free read-only structural validation. Reports filesystem, integrity, link, canonical-format, and index diagnostics and progress counts; never makes semantic judgments or fixes, deletes, or rewrites artifacts.",
|
|
3220
|
+
parameters: {},
|
|
3221
|
+
output: {
|
|
3222
|
+
schema: closed({
|
|
3223
|
+
diagnostics: {
|
|
3224
|
+
type: "array",
|
|
3225
|
+
required: true,
|
|
3226
|
+
items: closed({
|
|
3227
|
+
code: requiredString(),
|
|
3228
|
+
severity: {
|
|
3229
|
+
type: "string",
|
|
3230
|
+
required: true,
|
|
3231
|
+
enum: ["error", "warning"]
|
|
3232
|
+
},
|
|
3233
|
+
path: requiredString(),
|
|
3234
|
+
line: requiredNullableInteger,
|
|
3235
|
+
message: requiredString()
|
|
3236
|
+
})
|
|
3237
|
+
},
|
|
3238
|
+
errorCount: requiredInteger(),
|
|
3239
|
+
warningCount: requiredInteger(),
|
|
3240
|
+
filesExamined: requiredInteger()
|
|
3241
|
+
}),
|
|
3242
|
+
render
|
|
3243
|
+
},
|
|
3244
|
+
execute: (_args, exec) => call(async () => {
|
|
3245
|
+
const report = await ctx.llmwiki.lint(exec.signal);
|
|
3246
|
+
return {
|
|
3247
|
+
diagnostics: report.diagnostics.map((diagnostic) => ({
|
|
3248
|
+
code: diagnostic.code,
|
|
3249
|
+
severity: diagnostic.severity,
|
|
3250
|
+
path: diagnostic.path,
|
|
3251
|
+
line: diagnostic.line ?? null,
|
|
3252
|
+
message: diagnostic.message
|
|
3253
|
+
})),
|
|
3254
|
+
errorCount: report.errorCount,
|
|
3255
|
+
warningCount: report.warningCount,
|
|
3256
|
+
filesExamined: report.filesExamined
|
|
3257
|
+
};
|
|
3258
|
+
}),
|
|
3259
|
+
isConcurrencySafe: () => true,
|
|
3260
|
+
...presentation("llmwiki_lint")
|
|
3261
|
+
}));
|
|
3262
|
+
}
|
|
3263
|
+
//#endregion
|
|
3264
|
+
//#region src/index.ts
|
|
3265
|
+
const name = "llmwiki";
|
|
3266
|
+
const inject = [
|
|
3267
|
+
"tools",
|
|
3268
|
+
"commands",
|
|
3269
|
+
"systemPrompt"
|
|
3270
|
+
];
|
|
3271
|
+
function apply(ctx, config = {}) {
|
|
3272
|
+
const resolvedConfig = resolveConfig(config);
|
|
3273
|
+
new LlmWikiService(ctx, resolvedConfig);
|
|
3274
|
+
registerLlmWikiPrompt(ctx);
|
|
3275
|
+
registerLlmWikiTools(ctx);
|
|
3276
|
+
registerLlmWikiCommand(ctx, resolvedConfig);
|
|
3277
|
+
}
|
|
3278
|
+
//#endregion
|
|
3279
|
+
export { Config, LLMWIKI_ERROR_CODES, LlmWikiError, LlmWikiService, apply, inject, isLlmWikiError, isPageId, isSourceId, name, pageId, sourceId };
|
|
3280
|
+
|
|
3281
|
+
//# sourceMappingURL=index.js.map
|