@hraness/wordcell 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -15
- package/dist/agent-context.js +1 -0
- package/dist/agent-guide-audit.js +1 -0
- package/dist/attachments.js +1 -0
- package/dist/authoring.js +1 -0
- package/dist/authority-4sp9d552.js +663 -0
- package/dist/benchmark.js +1 -0
- package/dist/capture.js +1 -0
- package/dist/cli.js +3781 -32
- package/dist/clip/acquire.js +1 -0
- package/dist/clip/args.js +1 -0
- package/dist/clip/bounded-byte-buffer.js +1 -0
- package/dist/clip/browser-profiles.js +1 -0
- package/dist/clip/bundle-reader.js +1 -0
- package/dist/clip/cli.js +1 -0
- package/dist/clip/cookies.js +1 -0
- package/dist/clip/doctor.js +1 -0
- package/dist/clip/jobs.js +1 -0
- package/dist/clip/network-proxy.js +1 -0
- package/dist/clip/network.js +1 -0
- package/dist/clip/persist.js +1 -0
- package/dist/clip/refresh.js +1 -0
- package/dist/clip/terminal.js +1 -0
- package/dist/evaluation-builder.js +13 -5
- package/dist/evaluation-kb.js +11 -4
- package/dist/evaluation.js +1 -0
- package/dist/git.js +1 -0
- package/dist/graph-authority.js +29 -0
- package/dist/graph-percolation.js +16 -0
- package/dist/graph.js +1 -0
- package/dist/index-0k2x4nn9.js +393 -0
- package/dist/{index-n05s3wsb.js → index-115b07ap.js} +156 -541
- package/dist/index-11621h23.js +160 -0
- package/dist/{index-tkpxp48t.js → index-51bap0mq.js} +2 -2
- package/dist/{index-8a08v6x8.js → index-59ssmqqm.js} +1 -1
- package/dist/index-bcfn9xah.js +472 -0
- package/dist/index-bgfzwt4h.js +51 -0
- package/dist/{index-bs0yf4pp.js → index-pbr6rv47.js} +2 -2
- package/dist/{index-m7et4rz4.js → index-vdw5yjfa.js} +61 -9
- package/dist/index-z1w83f81.js +4 -0
- package/dist/index.js +61 -34
- package/dist/navigation.js +1 -0
- package/dist/pdf.js +1 -0
- package/dist/percolate.js +1 -0
- package/dist/portfolio.js +11 -4
- package/dist/query.js +1 -0
- package/dist/repository-memory.js +1 -0
- package/dist/sdk.js +10 -3
- package/dist/search-rules.js +1 -0
- package/dist/search.js +1 -0
- package/dist/semantic.js +3 -1
- package/dist/source-inbox.js +1 -0
- package/dist/untrusted-content.js +1 -0
- package/dist/url-intelligence.js +1 -0
- package/dist/workflow.js +1 -0
- package/dist/workflows/decision-context.js +11 -4
- package/dist/workflows/explain-change.js +1 -0
- package/dist/workflows/index.js +11 -4
- package/dist/workflows/plan-radar.js +1 -0
- package/package.json +24 -6
- package/skills/wordcell/SKILL.md +2 -2
- package/skills/wordcell/references/percolate.md +4 -0
- package/skills/wordcell/references/query.md +6 -0
- package/src/cli-program.ts +45 -2
- package/src/graph-authority-model.ts +116 -0
- package/src/graph-authority.ts +219 -0
- package/src/graph-cache-filesystem.ts +4 -0
- package/src/graph-cli.ts +96 -0
- package/src/graph-facts.ts +206 -0
- package/src/graph-percolation.ts +39 -0
- package/src/graph-query.ts +122 -0
- package/src/index.ts +2 -0
- package/src/oh/authority.ts +193 -0
- package/src/oh/programs.ts +99 -0
- package/src/oh/schema.ts +40 -0
- package/src/oh/snapshot.ts +114 -0
- package/src/oh/validation.ts +61 -0
- package/src/sdk.ts +51 -6
- package/dist/index-sy0zety7.js +0 -3630
- package/dist/kb-alias.js +0 -55
- package/src/kb-alias.ts +0 -14
|
@@ -1,433 +1,48 @@
|
|
|
1
1
|
// @bun
|
|
2
|
+
import {
|
|
3
|
+
MAX_NOTE_UTF8_BYTES,
|
|
4
|
+
MAX_SCANNED_NOTES,
|
|
5
|
+
MAX_VAULT_UTF8_BYTES,
|
|
6
|
+
VaultScanBudgetError,
|
|
7
|
+
scanVault
|
|
8
|
+
} from "./index-0k2x4nn9.js";
|
|
2
9
|
import {
|
|
3
10
|
fuseRankedCandidates,
|
|
4
11
|
validateSearchQuery
|
|
5
12
|
} from "./index-why54q5p.js";
|
|
6
|
-
import {
|
|
7
|
-
MAX_ANALYZED_NOTES,
|
|
8
|
-
analyzeVault,
|
|
9
|
-
isCanonicalNoteId,
|
|
10
|
-
lookupNote,
|
|
11
|
-
normalizeVaultPath,
|
|
12
|
-
parseNote,
|
|
13
|
-
renderCatalog,
|
|
14
|
-
replaceCatalog
|
|
15
|
-
} from "./index-ekpwvbra.js";
|
|
16
13
|
|
|
17
14
|
// src/semantic.ts
|
|
18
15
|
import { createHash as createHash2 } from "crypto";
|
|
19
|
-
import { constants as
|
|
16
|
+
import { constants as constants2 } from "fs";
|
|
20
17
|
import {
|
|
21
18
|
chmod,
|
|
22
|
-
lstat as
|
|
19
|
+
lstat as lstat2,
|
|
23
20
|
mkdir as mkdir2,
|
|
24
21
|
mkdtemp,
|
|
25
|
-
open as
|
|
26
|
-
realpath as
|
|
27
|
-
rm as
|
|
22
|
+
open as open2,
|
|
23
|
+
realpath as realpath2,
|
|
24
|
+
rm as rm2,
|
|
28
25
|
stat
|
|
29
26
|
} from "fs/promises";
|
|
30
27
|
import { homedir, tmpdir } from "os";
|
|
31
|
-
import { dirname as
|
|
28
|
+
import { dirname as dirname2, isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2, sep as sep2 } from "path";
|
|
32
29
|
|
|
33
30
|
// src/semantic-runtime.ts
|
|
34
|
-
import { createHash, randomUUID
|
|
31
|
+
import { createHash, randomUUID } from "crypto";
|
|
35
32
|
import {
|
|
36
|
-
constants
|
|
33
|
+
constants
|
|
37
34
|
} from "fs";
|
|
38
|
-
import {
|
|
39
|
-
lstat as lstat2,
|
|
40
|
-
mkdir,
|
|
41
|
-
open as open2,
|
|
42
|
-
opendir,
|
|
43
|
-
realpath as realpath2,
|
|
44
|
-
rename as rename2,
|
|
45
|
-
rm as rm2,
|
|
46
|
-
writeFile
|
|
47
|
-
} from "fs/promises";
|
|
48
|
-
import { basename as basename2, dirname as dirname2, isAbsolute, join as join2, relative as relative2, resolve as resolve2, sep as sep2 } from "path";
|
|
49
|
-
|
|
50
|
-
// src/vault.ts
|
|
51
|
-
import { randomUUID } from "crypto";
|
|
52
|
-
import { constants } from "fs";
|
|
53
35
|
import {
|
|
54
36
|
lstat,
|
|
37
|
+
mkdir,
|
|
55
38
|
open,
|
|
56
|
-
|
|
39
|
+
opendir,
|
|
57
40
|
realpath,
|
|
58
41
|
rename,
|
|
59
|
-
rm
|
|
42
|
+
rm,
|
|
43
|
+
writeFile
|
|
60
44
|
} from "fs/promises";
|
|
61
|
-
import { basename, dirname, join, relative, resolve, sep } from "path";
|
|
62
|
-
var MAX_SCANNED_NOTES = MAX_ANALYZED_NOTES;
|
|
63
|
-
var MAX_NOTE_UTF8_BYTES = 16 * 1024 * 1024;
|
|
64
|
-
var MAX_VAULT_UTF8_BYTES = 256 * 1024 * 1024;
|
|
65
|
-
|
|
66
|
-
class VaultScanBudgetError extends RangeError {
|
|
67
|
-
kind;
|
|
68
|
-
limit;
|
|
69
|
-
constructor(kind, limit, message) {
|
|
70
|
-
super(message);
|
|
71
|
-
this.name = "VaultScanBudgetError";
|
|
72
|
-
this.kind = kind;
|
|
73
|
-
this.limit = limit;
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
var defaultIgnoredDirectories = new Set([
|
|
77
|
-
".git",
|
|
78
|
-
".next",
|
|
79
|
-
".turbo",
|
|
80
|
-
".vercel",
|
|
81
|
-
"coverage",
|
|
82
|
-
"dist",
|
|
83
|
-
"node_modules"
|
|
84
|
-
]);
|
|
85
|
-
async function markdownFiles(directory, ignoredDirectories = defaultIgnoredDirectories) {
|
|
86
|
-
const files = [];
|
|
87
|
-
const entries = await readdir(directory, { withFileTypes: true });
|
|
88
|
-
for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) {
|
|
89
|
-
if (entry.name.startsWith("."))
|
|
90
|
-
continue;
|
|
91
|
-
const entryPath = join(directory, entry.name);
|
|
92
|
-
if (entry.isDirectory()) {
|
|
93
|
-
if (ignoredDirectories.has(entry.name))
|
|
94
|
-
continue;
|
|
95
|
-
files.push(...await markdownFiles(entryPath, ignoredDirectories));
|
|
96
|
-
continue;
|
|
97
|
-
}
|
|
98
|
-
if (entry.isFile() && entry.name.endsWith(".md") && entry.name !== "AGENTS.md") {
|
|
99
|
-
files.push(entryPath);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
return files;
|
|
103
|
-
}
|
|
104
|
-
function checkedScanLimit(value, hardMaximum, option) {
|
|
105
|
-
const limit = value ?? hardMaximum;
|
|
106
|
-
if (!Number.isSafeInteger(limit) || limit < 0 || limit > hardMaximum) {
|
|
107
|
-
throw new RangeError(`${option} must be a safe integer from 0 through ${hardMaximum}.`);
|
|
108
|
-
}
|
|
109
|
-
return limit;
|
|
110
|
-
}
|
|
111
|
-
function normalizedRawNoteId(rawId) {
|
|
112
|
-
return normalizeVaultPath(rawId).normalize("NFC");
|
|
113
|
-
}
|
|
114
|
-
function validateScannedNotePaths(root, paths) {
|
|
115
|
-
const files = paths.map((absolutePath) => {
|
|
116
|
-
const vaultPath = relative(root, absolutePath).split(sep).join("/");
|
|
117
|
-
return {
|
|
118
|
-
absolutePath,
|
|
119
|
-
vaultPath,
|
|
120
|
-
rawId: vaultPath.slice(0, -3)
|
|
121
|
-
};
|
|
122
|
-
});
|
|
123
|
-
const pathByNormalizedId = new Map;
|
|
124
|
-
for (const file of files) {
|
|
125
|
-
const normalizedId = normalizedRawNoteId(file.rawId);
|
|
126
|
-
const collision = pathByNormalizedId.get(normalizedId);
|
|
127
|
-
if (collision !== undefined && collision !== file.vaultPath) {
|
|
128
|
-
throw new Error(`Vault note paths ${JSON.stringify(collision)} and ` + `${JSON.stringify(file.vaultPath)} normalize to the same note ID ` + `${JSON.stringify(normalizedId)}.`);
|
|
129
|
-
}
|
|
130
|
-
pathByNormalizedId.set(normalizedId, file.vaultPath);
|
|
131
|
-
}
|
|
132
|
-
for (const file of files) {
|
|
133
|
-
if (isCanonicalNoteId(file.rawId))
|
|
134
|
-
continue;
|
|
135
|
-
if (file.rawId !== file.rawId.normalize("NFC")) {
|
|
136
|
-
throw new Error(`Vault note path ${JSON.stringify(file.vaultPath)} is not NFC; ` + `its extensionless note ID must be exactly ` + `${JSON.stringify(file.rawId.normalize("NFC"))}.`);
|
|
137
|
-
}
|
|
138
|
-
if (file.rawId.includes("\\")) {
|
|
139
|
-
throw new Error(`Vault note path ${JSON.stringify(file.vaultPath)} contains a backslash; ` + "note IDs must use exact vault-root directory separators.");
|
|
140
|
-
}
|
|
141
|
-
throw new Error(`Vault note path ${JSON.stringify(file.vaultPath)} must have an exact ` + "canonical extensionless vault-root note ID.");
|
|
142
|
-
}
|
|
143
|
-
return files;
|
|
144
|
-
}
|
|
145
|
-
function assertScannableNoteFile(vaultPath, metadata) {
|
|
146
|
-
if (metadata.isSymbolicLink()) {
|
|
147
|
-
throw new Error(`Vault note ${JSON.stringify(vaultPath)} must not be a symbolic link.`);
|
|
148
|
-
}
|
|
149
|
-
if (!metadata.isFile()) {
|
|
150
|
-
throw new Error(`Vault note ${JSON.stringify(vaultPath)} must be a regular file.`);
|
|
151
|
-
}
|
|
152
|
-
if (metadata.nlink !== 1n) {
|
|
153
|
-
throw new Error(`Vault note ${JSON.stringify(vaultPath)} must not be hard-linked.`);
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
function noteBytesError(vaultPath, limit) {
|
|
157
|
-
return new VaultScanBudgetError("note-bytes", limit, `Vault note ${JSON.stringify(vaultPath)} exceeds the ${limit}-byte UTF-8 limit.`);
|
|
158
|
-
}
|
|
159
|
-
function totalBytesError(limit) {
|
|
160
|
-
return new VaultScanBudgetError("total-bytes", limit, `Vault scan exceeds the ${limit}-byte cumulative UTF-8 limit.`);
|
|
161
|
-
}
|
|
162
|
-
async function readBoundedNote(handle, vaultPath, maxNoteBytes, remainingTotalBytes, maxTotalBytes) {
|
|
163
|
-
const chunks = [];
|
|
164
|
-
let bytes = 0;
|
|
165
|
-
for (;; ) {
|
|
166
|
-
const remaining = Math.min(maxNoteBytes - bytes, remainingTotalBytes - bytes);
|
|
167
|
-
const buffer = new Uint8Array(Math.min(64 * 1024, Math.max(1, remaining + 1)));
|
|
168
|
-
const result = await handle.read(buffer, 0, buffer.byteLength, null);
|
|
169
|
-
if (result.bytesRead === 0)
|
|
170
|
-
break;
|
|
171
|
-
bytes += result.bytesRead;
|
|
172
|
-
if (bytes > maxNoteBytes)
|
|
173
|
-
throw noteBytesError(vaultPath, maxNoteBytes);
|
|
174
|
-
if (bytes > remainingTotalBytes) {
|
|
175
|
-
throw totalBytesError(maxTotalBytes);
|
|
176
|
-
}
|
|
177
|
-
chunks.push(buffer.slice(0, result.bytesRead));
|
|
178
|
-
}
|
|
179
|
-
const joined = new Uint8Array(bytes);
|
|
180
|
-
let offset = 0;
|
|
181
|
-
for (const chunk of chunks) {
|
|
182
|
-
joined.set(chunk, offset);
|
|
183
|
-
offset += chunk.byteLength;
|
|
184
|
-
}
|
|
185
|
-
try {
|
|
186
|
-
return {
|
|
187
|
-
content: new TextDecoder("utf-8", { fatal: true }).decode(joined),
|
|
188
|
-
bytes
|
|
189
|
-
};
|
|
190
|
-
} catch (error) {
|
|
191
|
-
throw new Error(`Vault note ${JSON.stringify(vaultPath)} is not valid UTF-8.`, { cause: error });
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
async function readVaultNotes(root, ignoredDirectories = defaultIgnoredDirectories, limits = {}) {
|
|
195
|
-
const maxNotes = checkedScanLimit(limits.maxNotes, MAX_SCANNED_NOTES, "maxNotes");
|
|
196
|
-
const maxNoteBytes = checkedScanLimit(limits.maxNoteBytes, MAX_NOTE_UTF8_BYTES, "maxNoteBytes");
|
|
197
|
-
const maxTotalBytes = checkedScanLimit(limits.maxTotalBytes, MAX_VAULT_UTF8_BYTES, "maxTotalBytes");
|
|
198
|
-
const paths = await markdownFiles(root, ignoredDirectories);
|
|
199
|
-
if (paths.length > maxNotes) {
|
|
200
|
-
throw new VaultScanBudgetError("notes", maxNotes, `Vault scan exceeds the ${maxNotes} Markdown note limit.`);
|
|
201
|
-
}
|
|
202
|
-
const files = validateScannedNotePaths(root, paths);
|
|
203
|
-
let declaredTotal = 0n;
|
|
204
|
-
const preflight = [];
|
|
205
|
-
for (const file of files) {
|
|
206
|
-
const metadata = await lstat(file.absolutePath, { bigint: true });
|
|
207
|
-
assertScannableNoteFile(file.vaultPath, metadata);
|
|
208
|
-
if (metadata.size > BigInt(maxNoteBytes)) {
|
|
209
|
-
throw noteBytesError(file.vaultPath, maxNoteBytes);
|
|
210
|
-
}
|
|
211
|
-
declaredTotal += metadata.size;
|
|
212
|
-
if (declaredTotal > BigInt(maxTotalBytes)) {
|
|
213
|
-
throw totalBytesError(maxTotalBytes);
|
|
214
|
-
}
|
|
215
|
-
preflight.push({
|
|
216
|
-
...file,
|
|
217
|
-
device: metadata.dev,
|
|
218
|
-
inode: metadata.ino
|
|
219
|
-
});
|
|
220
|
-
}
|
|
221
|
-
const notes = [];
|
|
222
|
-
let observedTotal = 0;
|
|
223
|
-
for (const file of preflight) {
|
|
224
|
-
const handle = await open(file.absolutePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
225
|
-
try {
|
|
226
|
-
const beforeRead = await handle.stat({ bigint: true });
|
|
227
|
-
assertScannableNoteFile(file.vaultPath, beforeRead);
|
|
228
|
-
if (beforeRead.dev !== file.device || beforeRead.ino !== file.inode) {
|
|
229
|
-
throw new Error(`Vault note ${JSON.stringify(file.vaultPath)} changed during scan; retry.`);
|
|
230
|
-
}
|
|
231
|
-
if (beforeRead.size > BigInt(maxNoteBytes)) {
|
|
232
|
-
throw noteBytesError(file.vaultPath, maxNoteBytes);
|
|
233
|
-
}
|
|
234
|
-
if (BigInt(observedTotal) + beforeRead.size > BigInt(maxTotalBytes)) {
|
|
235
|
-
throw totalBytesError(maxTotalBytes);
|
|
236
|
-
}
|
|
237
|
-
const read = await readBoundedNote(handle, file.vaultPath, maxNoteBytes, maxTotalBytes - observedTotal, maxTotalBytes);
|
|
238
|
-
const afterRead = await handle.stat({ bigint: true });
|
|
239
|
-
if (afterRead.dev !== file.device || afterRead.ino !== file.inode || afterRead.size !== beforeRead.size || afterRead.size !== BigInt(read.bytes)) {
|
|
240
|
-
throw new Error(`Vault note ${JSON.stringify(file.vaultPath)} changed during scan; retry.`);
|
|
241
|
-
}
|
|
242
|
-
observedTotal += read.bytes;
|
|
243
|
-
notes.push(parseNote(file.vaultPath, read.content));
|
|
244
|
-
} finally {
|
|
245
|
-
await handle.close();
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
return notes;
|
|
249
|
-
}
|
|
250
|
-
function confined(root, path) {
|
|
251
|
-
const fromRoot = relative(root, path);
|
|
252
|
-
return fromRoot !== "" && fromRoot !== ".." && !fromRoot.startsWith(`..${sep}`);
|
|
253
|
-
}
|
|
254
|
-
async function assertConfinedIndexParents(root, path) {
|
|
255
|
-
if (!confined(root, path))
|
|
256
|
-
throw new Error("The configured index must be a file inside the vault root.");
|
|
257
|
-
const parent = dirname(path);
|
|
258
|
-
const segments = relative(root, parent).split(sep).filter((segment) => segment !== "");
|
|
259
|
-
let current = root;
|
|
260
|
-
for (const segment of segments) {
|
|
261
|
-
current = join(current, segment);
|
|
262
|
-
const metadata = await lstat(current);
|
|
263
|
-
if (metadata.isSymbolicLink()) {
|
|
264
|
-
throw new Error("The configured index path must not traverse a symbolic link.");
|
|
265
|
-
}
|
|
266
|
-
if (!metadata.isDirectory()) {
|
|
267
|
-
throw new Error("Every configured index parent must be a directory.");
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
const canonicalParent = await realpath(parent);
|
|
271
|
-
if (!confined(root, join(canonicalParent, basename(path)))) {
|
|
272
|
-
throw new Error("The configured index parent resolves outside the vault root.");
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
async function readIndexRevision(root, path, maxNoteBytes = MAX_NOTE_UTF8_BYTES) {
|
|
276
|
-
await assertConfinedIndexParents(root, path);
|
|
277
|
-
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
278
|
-
try {
|
|
279
|
-
const metadata = await handle.stat({ bigint: true });
|
|
280
|
-
if (!metadata.isFile())
|
|
281
|
-
throw new Error("The configured index must be a regular file.");
|
|
282
|
-
if (metadata.nlink !== 1n)
|
|
283
|
-
throw new Error("The configured index must not be hard-linked.");
|
|
284
|
-
const canonicalPath = await realpath(path);
|
|
285
|
-
if (!confined(root, canonicalPath)) {
|
|
286
|
-
throw new Error("The configured index resolves outside the vault root.");
|
|
287
|
-
}
|
|
288
|
-
const vaultPath = relative(root, path).split(sep).join("/");
|
|
289
|
-
if (metadata.size > BigInt(maxNoteBytes)) {
|
|
290
|
-
throw noteBytesError(vaultPath, maxNoteBytes);
|
|
291
|
-
}
|
|
292
|
-
const read = await readBoundedNote(handle, vaultPath, maxNoteBytes, maxNoteBytes, maxNoteBytes);
|
|
293
|
-
const afterRead = await handle.stat({ bigint: true });
|
|
294
|
-
if (afterRead.dev !== metadata.dev || afterRead.ino !== metadata.ino || afterRead.size !== metadata.size || afterRead.size !== BigInt(read.bytes)) {
|
|
295
|
-
throw new Error("The configured index changed during scan; retry.");
|
|
296
|
-
}
|
|
297
|
-
return {
|
|
298
|
-
content: read.content,
|
|
299
|
-
device: metadata.dev,
|
|
300
|
-
inode: metadata.ino,
|
|
301
|
-
mode: Number(metadata.mode & 0o777n)
|
|
302
|
-
};
|
|
303
|
-
} finally {
|
|
304
|
-
await handle.close();
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
function sameRevision(left, right) {
|
|
308
|
-
return left.device === right.device && left.inode === right.inode && left.content === right.content;
|
|
309
|
-
}
|
|
310
|
-
function parsedCatalogMode(value, source) {
|
|
311
|
-
if (value === undefined)
|
|
312
|
-
return;
|
|
313
|
-
if (value === "managed" || value === "authored")
|
|
314
|
-
return value;
|
|
315
|
-
throw new Error(`${source} must be exactly "managed" or "authored".`);
|
|
316
|
-
}
|
|
317
|
-
function declaredCatalogMode(indexNote) {
|
|
318
|
-
const declaration = Object.entries(indexNote.metadata).find(([name]) => name.toLocaleLowerCase("en-US") === "kb_catalog");
|
|
319
|
-
return parsedCatalogMode(declaration?.[1], `The configured index frontmatter property "kb_catalog"`);
|
|
320
|
-
}
|
|
321
|
-
async function atomicReplace(root, path, content, expected) {
|
|
322
|
-
const beforeWrite = await readIndexRevision(root, path);
|
|
323
|
-
if (!sameRevision(beforeWrite, expected)) {
|
|
324
|
-
throw new Error("The configured index changed during refresh; retry without overwriting the editor's changes.");
|
|
325
|
-
}
|
|
326
|
-
const directory = dirname(path);
|
|
327
|
-
const temporaryPath = join(directory, `.${basename(path)}.${process.pid}.${randomUUID()}.tmp`);
|
|
328
|
-
await assertConfinedIndexParents(root, path);
|
|
329
|
-
const handle = await open(temporaryPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, expected.mode);
|
|
330
|
-
let closed = false;
|
|
331
|
-
try {
|
|
332
|
-
await handle.writeFile(content, "utf8");
|
|
333
|
-
await handle.sync();
|
|
334
|
-
await handle.close();
|
|
335
|
-
closed = true;
|
|
336
|
-
const beforeRename = await readIndexRevision(root, path);
|
|
337
|
-
if (!sameRevision(beforeRename, expected)) {
|
|
338
|
-
throw new Error("The configured index changed during refresh; retry without overwriting the editor's changes.");
|
|
339
|
-
}
|
|
340
|
-
await assertConfinedIndexParents(root, path);
|
|
341
|
-
await rename(temporaryPath, path);
|
|
342
|
-
} catch (error) {
|
|
343
|
-
if (!closed)
|
|
344
|
-
await handle.close().catch(() => {
|
|
345
|
-
return;
|
|
346
|
-
});
|
|
347
|
-
await rm(temporaryPath, { force: true }).catch(() => {
|
|
348
|
-
return;
|
|
349
|
-
});
|
|
350
|
-
throw error;
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
async function snapshot(rootInput, options, writeIndex) {
|
|
354
|
-
const requestedRoot = resolve(rootInput);
|
|
355
|
-
const root = await realpath(requestedRoot);
|
|
356
|
-
const rootMetadata = await lstat(root);
|
|
357
|
-
if (!rootMetadata.isDirectory())
|
|
358
|
-
throw new Error("The vault root must be a directory.");
|
|
359
|
-
const indexPath = resolve(root, options.index ?? "index.md");
|
|
360
|
-
const relativeIndex = relative(root, indexPath);
|
|
361
|
-
if (!confined(root, indexPath)) {
|
|
362
|
-
throw new Error("The configured index must be a file inside the vault root.");
|
|
363
|
-
}
|
|
364
|
-
if (!indexPath.toLowerCase().endsWith(".md")) {
|
|
365
|
-
throw new Error("The configured index must be a Markdown file.");
|
|
366
|
-
}
|
|
367
|
-
const vaultIndexPath = relativeIndex.split(sep).join("/");
|
|
368
|
-
const catalogNoteId = vaultIndexPath.toLowerCase().endsWith(".md") ? vaultIndexPath.slice(0, -3) : vaultIndexPath;
|
|
369
|
-
const notes = await readVaultNotes(root, options.ignoredDirectories, {
|
|
370
|
-
...options.maxNotes === undefined ? {} : { maxNotes: options.maxNotes },
|
|
371
|
-
...options.maxNoteBytes === undefined ? {} : { maxNoteBytes: options.maxNoteBytes },
|
|
372
|
-
...options.maxTotalBytes === undefined ? {} : { maxTotalBytes: options.maxTotalBytes }
|
|
373
|
-
});
|
|
374
|
-
const indexRevision = await readIndexRevision(root, indexPath, options.maxNoteBytes ?? MAX_NOTE_UTF8_BYTES);
|
|
375
|
-
const currentIndex = indexRevision.content;
|
|
376
|
-
const indexNote = parseNote(vaultIndexPath, currentIndex);
|
|
377
|
-
const catalogMode = parsedCatalogMode(options.catalogMode, "ScanVaultOptions.catalogMode") ?? declaredCatalogMode(indexNote) ?? "managed";
|
|
378
|
-
let index = "authored";
|
|
379
|
-
if (catalogMode === "managed") {
|
|
380
|
-
const expectedIndex = replaceCatalog(currentIndex, renderCatalog(notes, catalogNoteId));
|
|
381
|
-
const stale = currentIndex !== expectedIndex;
|
|
382
|
-
index = stale ? "stale" : "current";
|
|
383
|
-
if (writeIndex && stale) {
|
|
384
|
-
await atomicReplace(root, indexPath, expectedIndex, indexRevision);
|
|
385
|
-
index = "updated";
|
|
386
|
-
const parsed = parseNote(vaultIndexPath, expectedIndex);
|
|
387
|
-
const noteIndex = notes.findIndex((note) => note.path === vaultIndexPath);
|
|
388
|
-
if (noteIndex === -1)
|
|
389
|
-
notes.push(parsed);
|
|
390
|
-
else
|
|
391
|
-
notes[noteIndex] = parsed;
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
const mentionScope = options.mentionScope;
|
|
395
|
-
const mentionIds = new Set;
|
|
396
|
-
if (typeof mentionScope === "string") {
|
|
397
|
-
const lookup = lookupNote(notes, mentionScope);
|
|
398
|
-
if (lookup.kind === "found")
|
|
399
|
-
mentionIds.add(lookup.note.id);
|
|
400
|
-
else if (lookup.kind === "ambiguous") {
|
|
401
|
-
for (const note of lookup.candidates)
|
|
402
|
-
mentionIds.add(note.id);
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
const mentionScopePredicate = mentionScope === undefined ? undefined : (note) => mentionScope !== false && mentionIds.has(note.id);
|
|
406
|
-
return {
|
|
407
|
-
root,
|
|
408
|
-
indexPath,
|
|
409
|
-
catalogMode,
|
|
410
|
-
index,
|
|
411
|
-
notes,
|
|
412
|
-
analysis: analyzeVault(notes, {
|
|
413
|
-
catalogNoteId,
|
|
414
|
-
...options.includeInSuggestions === undefined ? {} : { includeInSuggestions: options.includeInSuggestions },
|
|
415
|
-
...mentionScopePredicate === undefined ? {} : { mentionScope: mentionScopePredicate },
|
|
416
|
-
...options.maxNotes === undefined ? {} : { maxNotes: options.maxNotes },
|
|
417
|
-
...options.maxConnectionObservations === undefined ? {} : { maxConnectionObservations: options.maxConnectionObservations },
|
|
418
|
-
...options.maxMentionPairs === undefined ? {} : { maxMentionPairs: options.maxMentionPairs },
|
|
419
|
-
...options.maxMentions === undefined ? {} : { maxMentions: options.maxMentions }
|
|
420
|
-
})
|
|
421
|
-
};
|
|
422
|
-
}
|
|
423
|
-
async function scanVault(root = ".", options = {}) {
|
|
424
|
-
return snapshot(root, options, false);
|
|
425
|
-
}
|
|
426
|
-
async function refreshVault(root = ".", options = {}) {
|
|
427
|
-
return snapshot(root, options, true);
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
// src/semantic-runtime.ts
|
|
45
|
+
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "path";
|
|
431
46
|
var LEASE_VERSION = 1;
|
|
432
47
|
var PROJECTION_VERSION = 2;
|
|
433
48
|
var DEFAULT_LEASE_WAIT_MS = 30000;
|
|
@@ -507,11 +122,11 @@ function validatedIndexIdentity(identity) {
|
|
|
507
122
|
return validated;
|
|
508
123
|
}
|
|
509
124
|
async function boundedMetadataText(path, maximum, label) {
|
|
510
|
-
const pathBefore = await
|
|
125
|
+
const pathBefore = await lstat(path, { bigint: true });
|
|
511
126
|
if (pathBefore.isSymbolicLink() || !pathBefore.isFile() || pathBefore.nlink !== 1n) {
|
|
512
127
|
throw new Error(`${label} must be a regular, singly linked file.`);
|
|
513
128
|
}
|
|
514
|
-
const handle = await
|
|
129
|
+
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
515
130
|
try {
|
|
516
131
|
const before = await handle.stat({ bigint: true });
|
|
517
132
|
if (!before.isFile() || before.nlink !== 1n || before.dev !== pathBefore.dev || before.ino !== pathBefore.ino) {
|
|
@@ -537,7 +152,7 @@ async function boundedMetadataText(path, maximum, label) {
|
|
|
537
152
|
const after = await handle.stat({ bigint: true });
|
|
538
153
|
let pathAfter;
|
|
539
154
|
try {
|
|
540
|
-
pathAfter = await
|
|
155
|
+
pathAfter = await lstat(path, { bigint: true });
|
|
541
156
|
} catch {
|
|
542
157
|
throw new Error(`${label} changed while it was being read; retry.`);
|
|
543
158
|
}
|
|
@@ -557,7 +172,7 @@ async function boundedMetadataText(path, maximum, label) {
|
|
|
557
172
|
}
|
|
558
173
|
async function safeDirectory(path, label) {
|
|
559
174
|
try {
|
|
560
|
-
const metadata = await
|
|
175
|
+
const metadata = await lstat(path);
|
|
561
176
|
if (metadata.isSymbolicLink())
|
|
562
177
|
throw new Error(`${label} must not be a symbolic link.`);
|
|
563
178
|
if (!metadata.isDirectory())
|
|
@@ -571,17 +186,17 @@ async function safeDirectory(path, label) {
|
|
|
571
186
|
}
|
|
572
187
|
async function canonicalProspectiveDirectory(path) {
|
|
573
188
|
const missing = [];
|
|
574
|
-
let candidate =
|
|
189
|
+
let candidate = resolve(path);
|
|
575
190
|
for (;; ) {
|
|
576
191
|
try {
|
|
577
|
-
return
|
|
192
|
+
return resolve(await realpath(candidate), ...missing.toReversed());
|
|
578
193
|
} catch (error) {
|
|
579
194
|
if (errorCode(error) !== "ENOENT")
|
|
580
195
|
throw error;
|
|
581
|
-
const parent =
|
|
196
|
+
const parent = dirname(candidate);
|
|
582
197
|
if (parent === candidate)
|
|
583
198
|
throw error;
|
|
584
|
-
missing.push(
|
|
199
|
+
missing.push(basename(candidate));
|
|
585
200
|
candidate = parent;
|
|
586
201
|
}
|
|
587
202
|
}
|
|
@@ -589,7 +204,7 @@ async function canonicalProspectiveDirectory(path) {
|
|
|
589
204
|
async function validateSemanticDatabaseFile(path) {
|
|
590
205
|
let metadata;
|
|
591
206
|
try {
|
|
592
|
-
metadata = await
|
|
207
|
+
metadata = await lstat(path, { bigint: true });
|
|
593
208
|
} catch (error) {
|
|
594
209
|
if (errorCode(error) === "ENOENT")
|
|
595
210
|
return;
|
|
@@ -601,7 +216,7 @@ async function validateSemanticDatabaseFile(path) {
|
|
|
601
216
|
if (!metadata.isFile() || metadata.nlink !== 1n) {
|
|
602
217
|
throw new Error("Semantic database must be a regular, singly linked file.");
|
|
603
218
|
}
|
|
604
|
-
const handle = await
|
|
219
|
+
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
605
220
|
try {
|
|
606
221
|
const current = await handle.stat({ bigint: true });
|
|
607
222
|
if (!current.isFile() || current.nlink !== 1n || current.dev !== metadata.dev || current.ino !== metadata.ino) {
|
|
@@ -612,12 +227,12 @@ async function validateSemanticDatabaseFile(path) {
|
|
|
612
227
|
}
|
|
613
228
|
}
|
|
614
229
|
async function canonicalSemanticDatabasePath(database) {
|
|
615
|
-
const requested =
|
|
616
|
-
const canonicalParent = await canonicalProspectiveDirectory(
|
|
617
|
-
const canonical =
|
|
230
|
+
const requested = resolve(database);
|
|
231
|
+
const canonicalParent = await canonicalProspectiveDirectory(dirname(requested));
|
|
232
|
+
const canonical = join(canonicalParent, basename(requested));
|
|
618
233
|
await validateSemanticDatabaseFile(canonical);
|
|
619
234
|
try {
|
|
620
|
-
return await
|
|
235
|
+
return await realpath(canonical);
|
|
621
236
|
} catch (error) {
|
|
622
237
|
if (errorCode(error) === "ENOENT")
|
|
623
238
|
return canonical;
|
|
@@ -626,14 +241,14 @@ async function canonicalSemanticDatabasePath(database) {
|
|
|
626
241
|
}
|
|
627
242
|
async function settledSemanticDatabasePath(database) {
|
|
628
243
|
const canonical = await canonicalSemanticDatabasePath(database);
|
|
629
|
-
await mkdir(
|
|
630
|
-
const settledParent = await
|
|
631
|
-
if (settledParent !==
|
|
244
|
+
await mkdir(dirname(canonical), { recursive: true, mode: 448 });
|
|
245
|
+
const settledParent = await realpath(dirname(canonical));
|
|
246
|
+
if (settledParent !== dirname(canonical)) {
|
|
632
247
|
throw new Error("Semantic database parent changed while settling its identity; retry.");
|
|
633
248
|
}
|
|
634
249
|
let created;
|
|
635
250
|
try {
|
|
636
|
-
created = await
|
|
251
|
+
created = await open(canonical, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW | constants.O_NONBLOCK, 384);
|
|
637
252
|
} catch (error) {
|
|
638
253
|
if (errorCode(error) !== "EEXIST")
|
|
639
254
|
throw error;
|
|
@@ -641,7 +256,7 @@ async function settledSemanticDatabasePath(database) {
|
|
|
641
256
|
await created?.close();
|
|
642
257
|
}
|
|
643
258
|
await validateSemanticDatabaseFile(canonical);
|
|
644
|
-
return await
|
|
259
|
+
return await realpath(canonical);
|
|
645
260
|
}
|
|
646
261
|
function pathIsWithin(root, candidate) {
|
|
647
262
|
return candidate === root || confinedPath(root, candidate);
|
|
@@ -659,7 +274,7 @@ function assertSemanticCacheOutsideVault(root, database) {
|
|
|
659
274
|
async function resolveSemanticDatabase(database, root) {
|
|
660
275
|
const [canonicalDatabase, canonicalRoot] = await Promise.all([
|
|
661
276
|
canonicalSemanticDatabasePath(database),
|
|
662
|
-
|
|
277
|
+
realpath(root)
|
|
663
278
|
]);
|
|
664
279
|
assertSemanticCacheOutsideVault(canonicalRoot, canonicalDatabase);
|
|
665
280
|
return canonicalDatabase;
|
|
@@ -677,7 +292,7 @@ function parsedOwner(value) {
|
|
|
677
292
|
}
|
|
678
293
|
async function readOwner(path) {
|
|
679
294
|
try {
|
|
680
|
-
return parsedOwner(JSON.parse(await boundedMetadataText(
|
|
295
|
+
return parsedOwner(JSON.parse(await boundedMetadataText(join(path, OWNER_NAME), MAX_OWNER_BYTES, "Semantic lease owner")));
|
|
681
296
|
} catch (error) {
|
|
682
297
|
if (error instanceof RangeError)
|
|
683
298
|
throw error;
|
|
@@ -701,15 +316,15 @@ async function recoverDeadLease(path) {
|
|
|
701
316
|
const confirmed = await readOwner(path);
|
|
702
317
|
if (confirmed?.pid !== owner.pid || confirmed.token !== owner.token)
|
|
703
318
|
return false;
|
|
704
|
-
const tombstone = `${path}.dead-${owner.token}-${
|
|
319
|
+
const tombstone = `${path}.dead-${owner.token}-${randomUUID()}`;
|
|
705
320
|
try {
|
|
706
|
-
await
|
|
321
|
+
await rename(path, tombstone);
|
|
707
322
|
} catch (error) {
|
|
708
323
|
if (["ENOENT", "EEXIST", "ENOTEMPTY"].includes(errorCode(error) ?? ""))
|
|
709
324
|
return false;
|
|
710
325
|
throw error;
|
|
711
326
|
}
|
|
712
|
-
await
|
|
327
|
+
await rm(tombstone, { recursive: true, force: true });
|
|
713
328
|
return true;
|
|
714
329
|
}
|
|
715
330
|
async function acquireWriterLeaseState(database, options) {
|
|
@@ -720,7 +335,7 @@ async function acquireWriterLeaseState(database, options) {
|
|
|
720
335
|
const owner = {
|
|
721
336
|
version: LEASE_VERSION,
|
|
722
337
|
pid: process.pid,
|
|
723
|
-
token:
|
|
338
|
+
token: randomUUID(),
|
|
724
339
|
acquiredAt: new Date().toISOString()
|
|
725
340
|
};
|
|
726
341
|
const startedAt = Date.now();
|
|
@@ -729,14 +344,14 @@ async function acquireWriterLeaseState(database, options) {
|
|
|
729
344
|
const claim = `${path}.claim-${process.pid}-${owner.token}`;
|
|
730
345
|
await mkdir(claim, { mode: 448 });
|
|
731
346
|
try {
|
|
732
|
-
await writeFile(
|
|
347
|
+
await writeFile(join(claim, OWNER_NAME), `${JSON.stringify(owner)}
|
|
733
348
|
`, {
|
|
734
349
|
encoding: "utf8",
|
|
735
350
|
flag: "wx",
|
|
736
351
|
mode: 384
|
|
737
352
|
});
|
|
738
353
|
try {
|
|
739
|
-
await
|
|
354
|
+
await rename(claim, path);
|
|
740
355
|
const lease = { path, owner, database: canonicalDatabase };
|
|
741
356
|
try {
|
|
742
357
|
await validateSemanticDatabaseFile(canonicalDatabase);
|
|
@@ -750,13 +365,13 @@ async function acquireWriterLeaseState(database, options) {
|
|
|
750
365
|
throw error;
|
|
751
366
|
}
|
|
752
367
|
} finally {
|
|
753
|
-
await
|
|
368
|
+
await rm(claim, { recursive: true, force: true });
|
|
754
369
|
}
|
|
755
370
|
if (await recoverDeadLease(path))
|
|
756
371
|
continue;
|
|
757
372
|
const elapsed = Date.now() - startedAt;
|
|
758
373
|
if (elapsed >= waitMs) {
|
|
759
|
-
throw new Error(`Timed out after ${waitMs}ms waiting for the semantic writer lease for ${JSON.stringify(
|
|
374
|
+
throw new Error(`Timed out after ${waitMs}ms waiting for the semantic writer lease for ${JSON.stringify(resolve(database))}.`);
|
|
760
375
|
}
|
|
761
376
|
await Bun.sleep(Math.min(pollMs, waitMs - elapsed));
|
|
762
377
|
}
|
|
@@ -769,13 +384,13 @@ async function releaseWriterLease(lease) {
|
|
|
769
384
|
return;
|
|
770
385
|
const tombstone = `${lease.path}.release-${lease.owner.token}`;
|
|
771
386
|
try {
|
|
772
|
-
await
|
|
387
|
+
await rename(lease.path, tombstone);
|
|
773
388
|
} catch (error) {
|
|
774
389
|
if (errorCode(error) === "ENOENT")
|
|
775
390
|
return;
|
|
776
391
|
throw error;
|
|
777
392
|
}
|
|
778
|
-
await
|
|
393
|
+
await rm(tombstone, { recursive: true, force: true });
|
|
779
394
|
}
|
|
780
395
|
async function acquireSemanticWriterLease(database, options = {}) {
|
|
781
396
|
const lease = await acquireWriterLeaseState(database, options);
|
|
@@ -790,8 +405,8 @@ async function acquireSemanticWriterLease(database, options = {}) {
|
|
|
790
405
|
};
|
|
791
406
|
}
|
|
792
407
|
function confinedPath(root, path) {
|
|
793
|
-
const candidate =
|
|
794
|
-
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${
|
|
408
|
+
const candidate = relative(root, path);
|
|
409
|
+
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep}`) && !isAbsolute(candidate);
|
|
795
410
|
}
|
|
796
411
|
function validatedProjectionNotes(notes) {
|
|
797
412
|
if (notes.length > MAX_SCANNED_NOTES) {
|
|
@@ -827,20 +442,20 @@ function validatedProjectionNotes(notes) {
|
|
|
827
442
|
return { notes: validated, totalBytes };
|
|
828
443
|
}
|
|
829
444
|
async function atomicWrite(path, content) {
|
|
830
|
-
const temporary = `${path}.${process.pid}.${
|
|
445
|
+
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
831
446
|
let handle;
|
|
832
447
|
try {
|
|
833
|
-
handle = await
|
|
448
|
+
handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 384);
|
|
834
449
|
await handle.writeFile(content, "utf8");
|
|
835
450
|
await handle.sync();
|
|
836
451
|
await handle.close();
|
|
837
452
|
handle = undefined;
|
|
838
|
-
await
|
|
453
|
+
await rename(temporary, path);
|
|
839
454
|
} finally {
|
|
840
455
|
await handle?.close().catch(() => {
|
|
841
456
|
return;
|
|
842
457
|
});
|
|
843
|
-
await
|
|
458
|
+
await rm(temporary, { force: true }).catch(() => {
|
|
844
459
|
return;
|
|
845
460
|
});
|
|
846
461
|
}
|
|
@@ -849,7 +464,7 @@ function snapshotCacheOwner(database) {
|
|
|
849
464
|
return {
|
|
850
465
|
version: SNAPSHOT_OWNER_VERSION,
|
|
851
466
|
kind: SNAPSHOT_OWNER_KIND,
|
|
852
|
-
databaseIdentity:
|
|
467
|
+
databaseIdentity: resolve(database)
|
|
853
468
|
};
|
|
854
469
|
}
|
|
855
470
|
async function assertOwnedSnapshotCache(database, snapshotDirectory) {
|
|
@@ -859,7 +474,7 @@ async function assertOwnedSnapshotCache(database, snapshotDirectory) {
|
|
|
859
474
|
const expected = snapshotCacheOwner(database);
|
|
860
475
|
let actual;
|
|
861
476
|
try {
|
|
862
|
-
actual = JSON.parse(await boundedMetadataText(
|
|
477
|
+
actual = JSON.parse(await boundedMetadataText(join(snapshotDirectory, SNAPSHOT_OWNER_NAME), MAX_OWNER_BYTES, "Semantic snapshot cache owner"));
|
|
863
478
|
} catch {
|
|
864
479
|
throw new Error(`Semantic snapshot cache ${JSON.stringify(snapshotDirectory)} is unowned or has an incompatible ownership marker; remove this disposable directory explicitly before retrying.`);
|
|
865
480
|
}
|
|
@@ -871,20 +486,20 @@ async function assertOwnedSnapshotCache(database, snapshotDirectory) {
|
|
|
871
486
|
async function ensureOwnedSnapshotCache(database, snapshotDirectory) {
|
|
872
487
|
if (await assertOwnedSnapshotCache(database, snapshotDirectory) === "owned")
|
|
873
488
|
return;
|
|
874
|
-
const temporary = `${snapshotDirectory}.initialize-${process.pid}-${
|
|
489
|
+
const temporary = `${snapshotDirectory}.initialize-${process.pid}-${randomUUID()}`;
|
|
875
490
|
await mkdir(temporary, { mode: 448 });
|
|
876
491
|
try {
|
|
877
|
-
await writeFile(
|
|
492
|
+
await writeFile(join(temporary, SNAPSHOT_OWNER_NAME), `${JSON.stringify(snapshotCacheOwner(database))}
|
|
878
493
|
`, { encoding: "utf8", flag: "wx", mode: 384 });
|
|
879
494
|
try {
|
|
880
|
-
await
|
|
495
|
+
await rename(temporary, snapshotDirectory);
|
|
881
496
|
return;
|
|
882
497
|
} catch (error) {
|
|
883
498
|
if (!["EEXIST", "ENOTEMPTY"].includes(errorCode(error) ?? ""))
|
|
884
499
|
throw error;
|
|
885
500
|
}
|
|
886
501
|
} finally {
|
|
887
|
-
await
|
|
502
|
+
await rm(temporary, { recursive: true, force: true });
|
|
888
503
|
}
|
|
889
504
|
await assertOwnedSnapshotCache(database, snapshotDirectory);
|
|
890
505
|
}
|
|
@@ -893,11 +508,11 @@ async function existingGenerationMatches(generationPath, manifest, manifestText)
|
|
|
893
508
|
if (await safeDirectory(generationPath, "Semantic projection generation") === "absent") {
|
|
894
509
|
return false;
|
|
895
510
|
}
|
|
896
|
-
if (await boundedMetadataText(
|
|
511
|
+
if (await boundedMetadataText(join(generationPath, MANIFEST_NAME), MAX_MANIFEST_BYTES, "Semantic projection manifest") !== manifestText) {
|
|
897
512
|
return false;
|
|
898
513
|
}
|
|
899
514
|
for (const entry of manifest.notes) {
|
|
900
|
-
const path =
|
|
515
|
+
const path = resolve(generationPath, ...entry.path.split("/"));
|
|
901
516
|
if (!confinedPath(generationPath, path))
|
|
902
517
|
return false;
|
|
903
518
|
const content = await boundedMetadataText(path, MAX_NOTE_UTF8_BYTES, `Semantic projection note ${JSON.stringify(entry.path)}`);
|
|
@@ -905,13 +520,13 @@ async function existingGenerationMatches(generationPath, manifest, manifestText)
|
|
|
905
520
|
return false;
|
|
906
521
|
}
|
|
907
522
|
}
|
|
908
|
-
const expectedNotes = new Set(manifest.notes.map(({ path }) =>
|
|
523
|
+
const expectedNotes = new Set(manifest.notes.map(({ path }) => join(...path.split("/"))));
|
|
909
524
|
const expectedDirectories = new Set;
|
|
910
525
|
for (const path of expectedNotes) {
|
|
911
|
-
let parent =
|
|
526
|
+
let parent = dirname(path);
|
|
912
527
|
while (parent !== ".") {
|
|
913
528
|
expectedDirectories.add(parent);
|
|
914
|
-
parent =
|
|
529
|
+
parent = dirname(parent);
|
|
915
530
|
}
|
|
916
531
|
}
|
|
917
532
|
const pending = [generationPath];
|
|
@@ -927,8 +542,8 @@ async function existingGenerationMatches(generationPath, manifest, manifestText)
|
|
|
927
542
|
entries += 1;
|
|
928
543
|
if (entries > maximumEntries || entry.isSymbolicLink())
|
|
929
544
|
return false;
|
|
930
|
-
const absolute =
|
|
931
|
-
const path =
|
|
545
|
+
const absolute = join(directoryPath, entry.name);
|
|
546
|
+
const path = relative(generationPath, absolute);
|
|
932
547
|
if (entry.isDirectory()) {
|
|
933
548
|
if (!expectedDirectories.has(path))
|
|
934
549
|
return false;
|
|
@@ -939,7 +554,7 @@ async function existingGenerationMatches(generationPath, manifest, manifestText)
|
|
|
939
554
|
return false;
|
|
940
555
|
if (path === MANIFEST_NAME || expectedNotes.has(path))
|
|
941
556
|
continue;
|
|
942
|
-
if (
|
|
557
|
+
if (dirname(path) === "." && entry.name.startsWith(READER_PREFIX) && entry.name.endsWith(".json")) {
|
|
943
558
|
continue;
|
|
944
559
|
}
|
|
945
560
|
return false;
|
|
@@ -965,9 +580,9 @@ async function materializeGeneration(snapshotDirectory, generationPath, manifest
|
|
|
965
580
|
if (await activeGenerationReaders(generationPath) > 0) {
|
|
966
581
|
throw new Error("Semantic projection cache verification failed while this generation still has active readers; close those sessions and retry the repair.");
|
|
967
582
|
}
|
|
968
|
-
await
|
|
583
|
+
await rm(generationPath, { recursive: true, force: true });
|
|
969
584
|
}
|
|
970
|
-
const temporary =
|
|
585
|
+
const temporary = join(snapshotDirectory, `.temporary-${process.pid}-${randomUUID()}`);
|
|
971
586
|
await mkdir(temporary, { recursive: true, mode: 448 });
|
|
972
587
|
try {
|
|
973
588
|
for (const entry of manifest.notes) {
|
|
@@ -980,21 +595,21 @@ async function materializeGeneration(snapshotDirectory, generationPath, manifest
|
|
|
980
595
|
if (bytes !== entry.bytes || sha256 !== entry.sha256) {
|
|
981
596
|
throw new Error(`Semantic projection note ${JSON.stringify(entry.path)} changed after validation.`);
|
|
982
597
|
}
|
|
983
|
-
const destination =
|
|
598
|
+
const destination = resolve(temporary, ...entry.path.split("/"));
|
|
984
599
|
if (!confinedPath(temporary, destination)) {
|
|
985
600
|
throw new Error(`Semantic projection path ${JSON.stringify(entry.path)} escapes its generation.`);
|
|
986
601
|
}
|
|
987
|
-
await mkdir(
|
|
602
|
+
await mkdir(dirname(destination), { recursive: true, mode: 448 });
|
|
988
603
|
await writeFile(destination, note.content, {
|
|
989
604
|
encoding: "utf8",
|
|
990
605
|
flag: "wx",
|
|
991
606
|
mode: 384
|
|
992
607
|
});
|
|
993
608
|
}
|
|
994
|
-
await atomicWrite(
|
|
995
|
-
await
|
|
609
|
+
await atomicWrite(join(temporary, MANIFEST_NAME), manifestText);
|
|
610
|
+
await rename(temporary, generationPath);
|
|
996
611
|
} catch (error) {
|
|
997
|
-
await
|
|
612
|
+
await rm(temporary, { recursive: true, force: true });
|
|
998
613
|
throw error;
|
|
999
614
|
}
|
|
1000
615
|
}
|
|
@@ -1021,7 +636,7 @@ async function activeGenerationReaders(generation) {
|
|
|
1021
636
|
if (readerEntries > MAX_CACHE_DIRECTORY_ENTRIES) {
|
|
1022
637
|
throw new RangeError("Semantic projection generation has too many reader entries.");
|
|
1023
638
|
}
|
|
1024
|
-
const path =
|
|
639
|
+
const path = join(generation, entry.name);
|
|
1025
640
|
let owner;
|
|
1026
641
|
try {
|
|
1027
642
|
owner = parsedOwner(JSON.parse(await boundedMetadataText(path, MAX_OWNER_BYTES, "Semantic projection reader")));
|
|
@@ -1033,7 +648,7 @@ async function activeGenerationReaders(generation) {
|
|
|
1033
648
|
if (owner !== null && processIsAlive(owner.pid)) {
|
|
1034
649
|
active += 1;
|
|
1035
650
|
} else {
|
|
1036
|
-
await
|
|
651
|
+
await rm(path, { force: true });
|
|
1037
652
|
}
|
|
1038
653
|
}
|
|
1039
654
|
} finally {
|
|
@@ -1060,7 +675,7 @@ async function activeReaderGenerations(database) {
|
|
|
1060
675
|
}
|
|
1061
676
|
if (!entry.isDirectory() || !entry.name.startsWith(GENERATION_PREFIX))
|
|
1062
677
|
continue;
|
|
1063
|
-
if (await activeGenerationReaders(
|
|
678
|
+
if (await activeGenerationReaders(join(snapshotDirectory, entry.name)) > 0) {
|
|
1064
679
|
active.add(entry.name);
|
|
1065
680
|
}
|
|
1066
681
|
}
|
|
@@ -1118,15 +733,15 @@ async function cleanUnusedGenerations(snapshotDirectory, currentGeneration) {
|
|
|
1118
733
|
}
|
|
1119
734
|
if (!entry.isDirectory())
|
|
1120
735
|
continue;
|
|
1121
|
-
const path =
|
|
736
|
+
const path = join(snapshotDirectory, entry.name);
|
|
1122
737
|
if (entry.name.startsWith(".temporary-")) {
|
|
1123
|
-
await
|
|
738
|
+
await rm(path, { recursive: true, force: true });
|
|
1124
739
|
continue;
|
|
1125
740
|
}
|
|
1126
741
|
if (!entry.name.startsWith(GENERATION_PREFIX) || entry.name === currentGeneration)
|
|
1127
742
|
continue;
|
|
1128
743
|
if (await activeGenerationReaders(path) === 0) {
|
|
1129
|
-
await
|
|
744
|
+
await rm(path, { recursive: true, force: true });
|
|
1130
745
|
}
|
|
1131
746
|
}
|
|
1132
747
|
} finally {
|
|
@@ -1137,7 +752,7 @@ async function cleanUnusedGenerations(snapshotDirectory, currentGeneration) {
|
|
|
1137
752
|
}
|
|
1138
753
|
async function describeSemanticProjection(database, root, notes, indexIdentity) {
|
|
1139
754
|
const [canonicalRoot, prospectiveDatabase] = await Promise.all([
|
|
1140
|
-
|
|
755
|
+
realpath(root),
|
|
1141
756
|
canonicalSemanticDatabasePath(database)
|
|
1142
757
|
]);
|
|
1143
758
|
assertSemanticCacheOutsideVault(canonicalRoot, prospectiveDatabase);
|
|
@@ -1154,7 +769,7 @@ async function describeSemanticProjection(database, root, notes, indexIdentity)
|
|
|
1154
769
|
})).digest("hex");
|
|
1155
770
|
const generation = `${GENERATION_PREFIX}${identity.slice(0, 32)}`;
|
|
1156
771
|
const snapshotDirectory = `${canonicalDatabase}.snapshot`;
|
|
1157
|
-
const generationPath =
|
|
772
|
+
const generationPath = join(snapshotDirectory, generation);
|
|
1158
773
|
const manifest = {
|
|
1159
774
|
version: PROJECTION_VERSION,
|
|
1160
775
|
indexIdentity: validatedIdentity,
|
|
@@ -1188,10 +803,10 @@ async function prepareSemanticProjection(description, notes) {
|
|
|
1188
803
|
const reader = {
|
|
1189
804
|
version: LEASE_VERSION,
|
|
1190
805
|
pid: process.pid,
|
|
1191
|
-
token:
|
|
806
|
+
token: randomUUID(),
|
|
1192
807
|
acquiredAt: new Date().toISOString()
|
|
1193
808
|
};
|
|
1194
|
-
const readerPath =
|
|
809
|
+
const readerPath = join(generationPath, readerOwnerName(reader));
|
|
1195
810
|
await writeFile(readerPath, `${JSON.stringify(reader)}
|
|
1196
811
|
`, {
|
|
1197
812
|
encoding: "utf8",
|
|
@@ -1199,10 +814,10 @@ async function prepareSemanticProjection(description, notes) {
|
|
|
1199
814
|
mode: 384
|
|
1200
815
|
});
|
|
1201
816
|
try {
|
|
1202
|
-
await atomicWrite(
|
|
817
|
+
await atomicWrite(join(snapshotDirectory, MANIFEST_NAME), manifestText);
|
|
1203
818
|
await cleanUnusedGenerations(snapshotDirectory, manifest.generation);
|
|
1204
819
|
} catch (error) {
|
|
1205
|
-
await
|
|
820
|
+
await rm(readerPath, { force: true });
|
|
1206
821
|
throw error;
|
|
1207
822
|
}
|
|
1208
823
|
let released = false;
|
|
@@ -1213,7 +828,7 @@ async function prepareSemanticProjection(description, notes) {
|
|
|
1213
828
|
if (released)
|
|
1214
829
|
return;
|
|
1215
830
|
released = true;
|
|
1216
|
-
await
|
|
831
|
+
await rm(readerPath, { force: true });
|
|
1217
832
|
}
|
|
1218
833
|
};
|
|
1219
834
|
}
|
|
@@ -1257,7 +872,7 @@ var semanticIndexIdentity = {
|
|
|
1257
872
|
};
|
|
1258
873
|
var qmdModuleSpecifier = "@tobilu/qmd";
|
|
1259
874
|
async function sha256EmbeddingModelFile(path) {
|
|
1260
|
-
const handle = await
|
|
875
|
+
const handle = await open2(path, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
1261
876
|
try {
|
|
1262
877
|
const metadata = await handle.stat();
|
|
1263
878
|
if (!metadata.isFile())
|
|
@@ -1301,14 +916,14 @@ async function verifiedIndexEmbeddingModelSource(path, dependencies) {
|
|
|
1301
916
|
if (path === undefined) {
|
|
1302
917
|
return Object.freeze({ source: recommendedEmbeddingModel, release: () => Promise.resolve() });
|
|
1303
918
|
}
|
|
1304
|
-
const sourcePath =
|
|
1305
|
-
const directory = await mkdtemp(
|
|
1306
|
-
const destinationPath =
|
|
919
|
+
const sourcePath = resolve2(path);
|
|
920
|
+
const directory = await mkdtemp(join2(tmpdir(), "hraness-wordcell-embedding-model-"));
|
|
921
|
+
const destinationPath = join2(directory, "pinned-model.gguf");
|
|
1307
922
|
let source;
|
|
1308
923
|
let destination;
|
|
1309
924
|
try {
|
|
1310
|
-
source = await
|
|
1311
|
-
destination = await
|
|
925
|
+
source = await open2(sourcePath, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
926
|
+
destination = await open2(destinationPath, constants2.O_WRONLY | constants2.O_CREAT | constants2.O_EXCL | constants2.O_NOFOLLOW, 256);
|
|
1312
927
|
const before = await source.stat();
|
|
1313
928
|
if (!before.isFile())
|
|
1314
929
|
throw new TypeError("The embedding model must be a regular file.");
|
|
@@ -1352,7 +967,7 @@ async function verifiedIndexEmbeddingModelSource(path, dependencies) {
|
|
|
1352
967
|
if (released)
|
|
1353
968
|
return;
|
|
1354
969
|
released = true;
|
|
1355
|
-
await
|
|
970
|
+
await rm2(directory, { recursive: true, force: true });
|
|
1356
971
|
}
|
|
1357
972
|
});
|
|
1358
973
|
} catch (error) {
|
|
@@ -1362,7 +977,7 @@ async function verifiedIndexEmbeddingModelSource(path, dependencies) {
|
|
|
1362
977
|
await source?.close().catch(() => {
|
|
1363
978
|
return;
|
|
1364
979
|
});
|
|
1365
|
-
await
|
|
980
|
+
await rm2(directory, { recursive: true, force: true });
|
|
1366
981
|
throw error;
|
|
1367
982
|
}
|
|
1368
983
|
}
|
|
@@ -1457,16 +1072,16 @@ async function sessionEmbeddingModelSource(options, dependencies) {
|
|
|
1457
1072
|
function cacheHome(dependencies) {
|
|
1458
1073
|
const configured = dependencies.cacheHome ?? process.env.XDG_CACHE_HOME;
|
|
1459
1074
|
if (configured !== undefined && configured.trim() !== "") {
|
|
1460
|
-
return isAbsolute2(configured) ? configured :
|
|
1075
|
+
return isAbsolute2(configured) ? configured : resolve2(configured);
|
|
1461
1076
|
}
|
|
1462
|
-
return
|
|
1077
|
+
return join2(homedir(), ".cache");
|
|
1463
1078
|
}
|
|
1464
1079
|
function semanticDatabasePath(root, dependencies = {}) {
|
|
1465
|
-
const identity = createHash2("sha256").update(
|
|
1466
|
-
return
|
|
1080
|
+
const identity = createHash2("sha256").update(resolve2(root)).digest("hex").slice(0, 20);
|
|
1081
|
+
return join2(cacheHome(dependencies), "hraness-kb", "indexes", `${identity}.sqlite`);
|
|
1467
1082
|
}
|
|
1468
1083
|
async function resolvedDirectory(path) {
|
|
1469
|
-
const root = await
|
|
1084
|
+
const root = await realpath2(resolve2(path));
|
|
1470
1085
|
if (!(await stat(root)).isDirectory())
|
|
1471
1086
|
throw new Error("Knowledge-base root must be a directory.");
|
|
1472
1087
|
return root;
|
|
@@ -1724,18 +1339,18 @@ async function writeAll(handle, bytes, length, position) {
|
|
|
1724
1339
|
async function copyStableSnapshotFile(source, destination, label, maximumBytes, expected) {
|
|
1725
1340
|
if (!Number.isSafeInteger(expected.bytes) || expected.bytes < 0 || expected.bytes > maximumBytes || !SHA256.test(expected.sha256))
|
|
1726
1341
|
throw new TypeError(`${label} seal is invalid.`);
|
|
1727
|
-
const pathBefore = await
|
|
1342
|
+
const pathBefore = await lstat2(source, { bigint: true });
|
|
1728
1343
|
if (pathBefore.isSymbolicLink() || !pathBefore.isFile() || pathBefore.nlink !== 1n || pathBefore.size > BigInt(maximumBytes) || pathBefore.size !== BigInt(expected.bytes)) {
|
|
1729
1344
|
throw new Error(`${label} must be one bounded, singly linked regular file.`);
|
|
1730
1345
|
}
|
|
1731
|
-
const sourceHandle = await
|
|
1346
|
+
const sourceHandle = await open2(source, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
1732
1347
|
let destinationHandle;
|
|
1733
1348
|
try {
|
|
1734
1349
|
const before = await sourceHandle.stat({ bigint: true });
|
|
1735
1350
|
if (!sameStableFileMetadata(pathBefore, before)) {
|
|
1736
1351
|
throw new Error(`${label} changed before its isolated read snapshot was opened.`);
|
|
1737
1352
|
}
|
|
1738
|
-
destinationHandle = await
|
|
1353
|
+
destinationHandle = await open2(destination, constants2.O_WRONLY | constants2.O_CREAT | constants2.O_EXCL | constants2.O_NOFOLLOW, 384);
|
|
1739
1354
|
const buffer = new Uint8Array(1024 * 1024);
|
|
1740
1355
|
const hash = createHash2("sha256");
|
|
1741
1356
|
let copied = 0;
|
|
@@ -1752,7 +1367,7 @@ async function copyStableSnapshotFile(source, destination, label, maximumBytes,
|
|
|
1752
1367
|
await destinationHandle.sync();
|
|
1753
1368
|
const [after, pathAfter] = await Promise.all([
|
|
1754
1369
|
sourceHandle.stat({ bigint: true }),
|
|
1755
|
-
|
|
1370
|
+
lstat2(source, { bigint: true })
|
|
1756
1371
|
]);
|
|
1757
1372
|
if (copied !== Number(before.size) || !sameStableFileMetadata(before, after) || !sameStableFileMetadata(after, pathAfter)) {
|
|
1758
1373
|
throw new Error(`${label} changed while its isolated read snapshot was copied.`);
|
|
@@ -1771,7 +1386,7 @@ async function copyStableSnapshotFile(source, destination, label, maximumBytes,
|
|
|
1771
1386
|
}
|
|
1772
1387
|
async function assertAbsentSnapshotSidecar(path, label) {
|
|
1773
1388
|
try {
|
|
1774
|
-
await
|
|
1389
|
+
await lstat2(path);
|
|
1775
1390
|
} catch (error) {
|
|
1776
1391
|
if (missingFile(error))
|
|
1777
1392
|
return;
|
|
@@ -1780,17 +1395,17 @@ async function assertAbsentSnapshotSidecar(path, label) {
|
|
|
1780
1395
|
throw new Error(`${label} appeared while the isolated read snapshot was copied.`);
|
|
1781
1396
|
}
|
|
1782
1397
|
async function createIsolatedQmdDatabaseSnapshot(database, seal) {
|
|
1783
|
-
const directory = await mkdtemp(
|
|
1398
|
+
const directory = await mkdtemp(join2(tmpdir(), "hraness-wordcell-qmd-reader."));
|
|
1784
1399
|
await chmod(directory, 448);
|
|
1785
1400
|
let cleaned = false;
|
|
1786
1401
|
const cleanup = async () => {
|
|
1787
1402
|
if (cleaned)
|
|
1788
1403
|
return;
|
|
1789
1404
|
cleaned = true;
|
|
1790
|
-
await
|
|
1405
|
+
await rm2(directory, { recursive: true, force: true });
|
|
1791
1406
|
};
|
|
1792
1407
|
try {
|
|
1793
|
-
const isolatedDatabase =
|
|
1408
|
+
const isolatedDatabase = join2(directory, "snapshot.sqlite");
|
|
1794
1409
|
if (seal.wal !== null || seal.shm !== null || seal.journal !== null) {
|
|
1795
1410
|
throw new TypeError("Strict warm semantic snapshots require checkpointed SQLite state.");
|
|
1796
1411
|
}
|
|
@@ -1823,7 +1438,7 @@ function aggregateCloseFailures(settlements, label) {
|
|
|
1823
1438
|
throw new AggregateError(failures.map(({ reason }) => reason), `${label} did not close cleanly.`);
|
|
1824
1439
|
}
|
|
1825
1440
|
}
|
|
1826
|
-
async function closeIsolatedStore(close,
|
|
1441
|
+
async function closeIsolatedStore(close, snapshot, label) {
|
|
1827
1442
|
const settlements = [];
|
|
1828
1443
|
try {
|
|
1829
1444
|
await close();
|
|
@@ -1832,29 +1447,29 @@ async function closeIsolatedStore(close, snapshot2, label) {
|
|
|
1832
1447
|
settlements.push({ status: "rejected", reason });
|
|
1833
1448
|
}
|
|
1834
1449
|
try {
|
|
1835
|
-
await
|
|
1450
|
+
await snapshot.cleanup();
|
|
1836
1451
|
settlements.push({ status: "fulfilled", value: undefined });
|
|
1837
1452
|
} catch (reason) {
|
|
1838
1453
|
settlements.push({ status: "rejected", reason });
|
|
1839
1454
|
}
|
|
1840
1455
|
aggregateCloseFailures(settlements, label);
|
|
1841
1456
|
}
|
|
1842
|
-
function isolatedWarmSearchStore(store,
|
|
1457
|
+
function isolatedWarmSearchStore(store, snapshot) {
|
|
1843
1458
|
let closePromise;
|
|
1844
1459
|
return Object.freeze({
|
|
1845
1460
|
...store,
|
|
1846
1461
|
close: () => {
|
|
1847
|
-
closePromise ??= closeIsolatedStore(store.close,
|
|
1462
|
+
closePromise ??= closeIsolatedStore(store.close, snapshot, "Isolated QMD warm store");
|
|
1848
1463
|
return closePromise;
|
|
1849
1464
|
}
|
|
1850
1465
|
});
|
|
1851
1466
|
}
|
|
1852
|
-
function isolatedAttestationStore(store,
|
|
1467
|
+
function isolatedAttestationStore(store, snapshot) {
|
|
1853
1468
|
let closePromise;
|
|
1854
1469
|
return Object.freeze({
|
|
1855
1470
|
...store,
|
|
1856
1471
|
close: () => {
|
|
1857
|
-
closePromise ??= closeIsolatedStore(store.close,
|
|
1472
|
+
closePromise ??= closeIsolatedStore(store.close, snapshot, "Isolated QMD attestation store");
|
|
1858
1473
|
return closePromise;
|
|
1859
1474
|
}
|
|
1860
1475
|
});
|
|
@@ -2011,7 +1626,7 @@ async function openedSemanticAttestationStore(value) {
|
|
|
2011
1626
|
}
|
|
2012
1627
|
}
|
|
2013
1628
|
async function openStore(root, database, embeddingModelSource, dependencies, requireStoreLocalVectorBoundary = false) {
|
|
2014
|
-
await mkdir2(
|
|
1629
|
+
await mkdir2(dirname2(database), { recursive: true });
|
|
2015
1630
|
const created = await (dependencies.createStore ?? defaultCreateStore)({
|
|
2016
1631
|
dbPath: database,
|
|
2017
1632
|
config: storeConfig(root, embeddingModelSource)
|
|
@@ -2022,7 +1637,7 @@ async function openStore(root, database, embeddingModelSource, dependencies, req
|
|
|
2022
1637
|
});
|
|
2023
1638
|
}
|
|
2024
1639
|
async function assertExactWarmProjectionFile(path, expected, label) {
|
|
2025
|
-
const handle = await
|
|
1640
|
+
const handle = await open2(path, constants2.O_RDONLY | constants2.O_NOFOLLOW);
|
|
2026
1641
|
try {
|
|
2027
1642
|
const before = await handle.stat();
|
|
2028
1643
|
if (!before.isFile() || before.nlink !== 1 || before.size !== expected.byteLength) {
|
|
@@ -2040,14 +1655,14 @@ async function assertExactWarmProjectionFile(path, expected, label) {
|
|
|
2040
1655
|
async function assertExistingWarmProjection(description, notes) {
|
|
2041
1656
|
let canonicalGeneration;
|
|
2042
1657
|
try {
|
|
2043
|
-
canonicalGeneration = await
|
|
1658
|
+
canonicalGeneration = await realpath2(description.generationPath);
|
|
2044
1659
|
} catch (error) {
|
|
2045
1660
|
throw new Error("The immutable warm semantic projection is absent.", { cause: error });
|
|
2046
1661
|
}
|
|
2047
1662
|
if (canonicalGeneration !== description.generationPath) {
|
|
2048
1663
|
throw new Error("The immutable warm semantic projection changed identity.");
|
|
2049
1664
|
}
|
|
2050
|
-
await assertExactWarmProjectionFile(
|
|
1665
|
+
await assertExactWarmProjectionFile(join2(canonicalGeneration, "manifest.json"), Buffer.from(description.manifestText, "utf8"), "Semantic projection manifest");
|
|
2051
1666
|
const notesByPath = new Map(notes.map((note) => [note.path, note]));
|
|
2052
1667
|
if (notesByPath.size !== description.manifest.notes.length) {
|
|
2053
1668
|
throw new Error("The immutable warm semantic projection note population drifted.");
|
|
@@ -2061,38 +1676,38 @@ async function assertExistingWarmProjection(description, notes) {
|
|
|
2061
1676
|
if (expected.byteLength !== entry.bytes || createHash2("sha256").update(expected).digest("hex") !== entry.sha256) {
|
|
2062
1677
|
throw new Error(`Semantic projection note ${JSON.stringify(entry.path)} drifted.`);
|
|
2063
1678
|
}
|
|
2064
|
-
await assertExactWarmProjectionFile(
|
|
1679
|
+
await assertExactWarmProjectionFile(resolve2(canonicalGeneration, ...entry.path.split("/")), expected, `Semantic projection note ${JSON.stringify(entry.path)}`);
|
|
2065
1680
|
}
|
|
2066
1681
|
}
|
|
2067
1682
|
async function openWarmSearchStore(database, databaseSnapshotSeal, embeddingModelSource, dependencies) {
|
|
2068
|
-
const
|
|
1683
|
+
const snapshot = await createIsolatedQmdDatabaseSnapshot(database, databaseSnapshotSeal);
|
|
2069
1684
|
try {
|
|
2070
1685
|
const created = await (dependencies.createWarmSearchStore ?? defaultCreateWarmSearchStore)({
|
|
2071
|
-
dbPath:
|
|
1686
|
+
dbPath: snapshot.database,
|
|
2072
1687
|
embeddingModelSource
|
|
2073
1688
|
});
|
|
2074
1689
|
const store = await openedWarmSearchStore(created, dependencies.now ?? performance.now.bind(performance));
|
|
2075
|
-
return isolatedWarmSearchStore(store,
|
|
1690
|
+
return isolatedWarmSearchStore(store, snapshot);
|
|
2076
1691
|
} catch (error) {
|
|
2077
|
-
await
|
|
1692
|
+
await snapshot.cleanup();
|
|
2078
1693
|
throw error;
|
|
2079
1694
|
}
|
|
2080
1695
|
}
|
|
2081
1696
|
async function openAttestationStore(database, databaseSnapshotSeal, dependencies) {
|
|
2082
|
-
const
|
|
1697
|
+
const snapshot = await createIsolatedQmdDatabaseSnapshot(database, databaseSnapshotSeal);
|
|
2083
1698
|
try {
|
|
2084
|
-
const created = await (dependencies.createAttestationStore ?? defaultCreateAttestationStore)({ dbPath:
|
|
1699
|
+
const created = await (dependencies.createAttestationStore ?? defaultCreateAttestationStore)({ dbPath: snapshot.database });
|
|
2085
1700
|
const store = await openedSemanticAttestationStore(created);
|
|
2086
|
-
return isolatedAttestationStore(store,
|
|
1701
|
+
return isolatedAttestationStore(store, snapshot);
|
|
2087
1702
|
} catch (error) {
|
|
2088
|
-
await
|
|
1703
|
+
await snapshot.cleanup();
|
|
2089
1704
|
throw error;
|
|
2090
1705
|
}
|
|
2091
1706
|
}
|
|
2092
1707
|
async function checkpointSemanticWarmCache(options, dependencies = {}) {
|
|
2093
1708
|
const root = await resolvedDirectory(options.root);
|
|
2094
1709
|
const database = await resolveSemanticDatabase(databaseFor(root, options.database, dependencies), root);
|
|
2095
|
-
const before = await
|
|
1710
|
+
const before = await lstat2(database);
|
|
2096
1711
|
if (before.isSymbolicLink() || !before.isFile() || before.nlink !== 1) {
|
|
2097
1712
|
throw new TypeError("The semantic database to checkpoint must be a singly linked regular file.");
|
|
2098
1713
|
}
|
|
@@ -2123,7 +1738,7 @@ async function checkpointSemanticWarmCache(options, dependencies = {}) {
|
|
|
2123
1738
|
for (const sidecar of [`${database}-wal`, `${database}-shm`]) {
|
|
2124
1739
|
let metadata;
|
|
2125
1740
|
try {
|
|
2126
|
-
metadata = await
|
|
1741
|
+
metadata = await lstat2(sidecar);
|
|
2127
1742
|
} catch (error) {
|
|
2128
1743
|
if (missingFile(error))
|
|
2129
1744
|
continue;
|
|
@@ -2132,7 +1747,7 @@ async function checkpointSemanticWarmCache(options, dependencies = {}) {
|
|
|
2132
1747
|
if (metadata.isSymbolicLink() || !metadata.isFile() || metadata.nlink !== 1) {
|
|
2133
1748
|
throw new TypeError("Semantic checkpoint sidecars must be singly linked regular files.");
|
|
2134
1749
|
}
|
|
2135
|
-
await
|
|
1750
|
+
await rm2(sidecar);
|
|
2136
1751
|
}
|
|
2137
1752
|
await Promise.all([
|
|
2138
1753
|
assertAbsentSnapshotSidecar(`${database}-wal`, "Semantic database WAL"),
|
|
@@ -2188,23 +1803,23 @@ async function openSemanticWarmSearchSession(options, dependencies = {}) {
|
|
|
2188
1803
|
if (!databaseState.isFile()) {
|
|
2189
1804
|
throw new Error("The warm semantic database must already be a regular file.");
|
|
2190
1805
|
}
|
|
2191
|
-
const
|
|
2192
|
-
const description = await describeSemanticProjection(database, root,
|
|
2193
|
-
await assertExistingWarmProjection(description,
|
|
1806
|
+
const snapshot = await semanticSnapshot(root, dependencies);
|
|
1807
|
+
const description = await describeSemanticProjection(database, root, snapshot.notes, semanticIndexIdentity);
|
|
1808
|
+
await assertExistingWarmProjection(description, snapshot.notes);
|
|
2194
1809
|
store = await openWarmSearchStore(database, options.databaseSnapshotSeal, embeddingModel.source, dependencies);
|
|
2195
1810
|
const pendingEmbeddings = await store.pendingEmbeddingCount();
|
|
2196
1811
|
if (pendingEmbeddings !== 0) {
|
|
2197
1812
|
throw new Error(`Warm semantic cache is not ready: ${pendingEmbeddings} embedding input(s) remain pending.`);
|
|
2198
1813
|
}
|
|
2199
|
-
const notesByPath = new Map(
|
|
1814
|
+
const notesByPath = new Map(snapshot.notes.map((note) => [note.path, note]));
|
|
2200
1815
|
const contentHashesByPath = new Map(description.manifest.notes.map(({ path, sha256 }) => [path, sha256]));
|
|
2201
|
-
const notesByQmdPath = qmdNoteLookup(
|
|
2202
|
-
const connectionsById = new Map(
|
|
1816
|
+
const notesByQmdPath = qmdNoteLookup(snapshot.notes, contentHashesByPath);
|
|
1817
|
+
const connectionsById = new Map(snapshot.analysis.noteConnections.map((connection) => [connection.id, connection]));
|
|
2203
1818
|
const update = Object.freeze({
|
|
2204
1819
|
collections: 1,
|
|
2205
1820
|
indexed: 0,
|
|
2206
1821
|
updated: 0,
|
|
2207
|
-
unchanged:
|
|
1822
|
+
unchanged: snapshot.notes.length,
|
|
2208
1823
|
removed: 0,
|
|
2209
1824
|
needsEmbedding: 0
|
|
2210
1825
|
});
|
|
@@ -2271,7 +1886,7 @@ async function openSemanticWarmSearchSession(options, dependencies = {}) {
|
|
|
2271
1886
|
function databaseFor(root, requested, dependencies) {
|
|
2272
1887
|
if (requested === undefined)
|
|
2273
1888
|
return semanticDatabasePath(root, dependencies);
|
|
2274
|
-
return
|
|
1889
|
+
return resolve2(requested);
|
|
2275
1890
|
}
|
|
2276
1891
|
async function embedChanged(store, update, force) {
|
|
2277
1892
|
if (!force && update.needsEmbedding === 0)
|
|
@@ -2291,11 +1906,11 @@ async function indexSemanticVault(options, dependencies = {}) {
|
|
|
2291
1906
|
try {
|
|
2292
1907
|
const root = await resolvedDirectory(options.root);
|
|
2293
1908
|
const databaseCandidate = await resolveSemanticDatabase(databaseFor(root, options.database, dependencies), root);
|
|
2294
|
-
const
|
|
2295
|
-
const description = await describeSemanticProjection(databaseCandidate, root,
|
|
1909
|
+
const snapshot = await semanticSnapshot(root, dependencies);
|
|
1910
|
+
const description = await describeSemanticProjection(databaseCandidate, root, snapshot.notes, semanticIndexIdentity);
|
|
2296
1911
|
const database = description.database;
|
|
2297
1912
|
return await withSemanticGenerationWriterLease(database, description.manifest.generation, async () => {
|
|
2298
|
-
const projection = await prepareSemanticProjection(description,
|
|
1913
|
+
const projection = await prepareSemanticProjection(description, snapshot.notes);
|
|
2299
1914
|
let store;
|
|
2300
1915
|
try {
|
|
2301
1916
|
store = await openStore(projection.root, database, embeddingModel.source, dependencies);
|
|
@@ -2387,15 +2002,15 @@ async function resolvedSearchNote(projectionRoot, result, notesByPath, notesByQm
|
|
|
2387
2002
|
return null;
|
|
2388
2003
|
let filepath;
|
|
2389
2004
|
try {
|
|
2390
|
-
filepath = await
|
|
2005
|
+
filepath = await realpath2(resolve2(result.filepath));
|
|
2391
2006
|
} catch {
|
|
2392
2007
|
return null;
|
|
2393
2008
|
}
|
|
2394
|
-
const candidate =
|
|
2395
|
-
if (candidate === "" || candidate === ".." || candidate.startsWith(`..${
|
|
2009
|
+
const candidate = relative2(projectionRoot, filepath);
|
|
2010
|
+
if (candidate === "" || candidate === ".." || candidate.startsWith(`..${sep2}`) || isAbsolute2(candidate)) {
|
|
2396
2011
|
return null;
|
|
2397
2012
|
}
|
|
2398
|
-
const note = notesByPath.get(candidate.split(
|
|
2013
|
+
const note = notesByPath.get(candidate.split(sep2).join("/"));
|
|
2399
2014
|
return note !== undefined && contentHashesByPath.get(note.path) === result.hash ? note : null;
|
|
2400
2015
|
}
|
|
2401
2016
|
function queryOffset(body, query, suggested) {
|
|
@@ -2597,12 +2212,12 @@ async function executeSemanticSearch(context, options) {
|
|
|
2597
2212
|
async function openSemanticSearchSession(options, dependencies = {}) {
|
|
2598
2213
|
const root = await resolvedDirectory(options.root);
|
|
2599
2214
|
const databaseCandidate = await resolveSemanticDatabase(databaseFor(root, options.database, dependencies), root);
|
|
2600
|
-
const
|
|
2601
|
-
const description = await describeSemanticProjection(databaseCandidate, root,
|
|
2602
|
-
const notesByPath = new Map(
|
|
2215
|
+
const snapshot = await semanticSnapshot(root, dependencies);
|
|
2216
|
+
const description = await describeSemanticProjection(databaseCandidate, root, snapshot.notes, semanticIndexIdentity);
|
|
2217
|
+
const notesByPath = new Map(snapshot.notes.map((note) => [note.path, note]));
|
|
2603
2218
|
const contentHashesByPath = new Map(description.manifest.notes.map(({ path, sha256 }) => [path, sha256]));
|
|
2604
|
-
const notesByQmdPath = qmdNoteLookup(
|
|
2605
|
-
const connectionsById = new Map(
|
|
2219
|
+
const notesByQmdPath = qmdNoteLookup(snapshot.notes, contentHashesByPath);
|
|
2220
|
+
const connectionsById = new Map(snapshot.analysis.noteConnections.map((connection) => [connection.id, connection]));
|
|
2606
2221
|
const requireStoreLocalVectorBoundary = requiredStoreLocalVectorBoundary(options.requireStoreLocalVectorBoundary);
|
|
2607
2222
|
const embeddingModel = await sessionEmbeddingModelSource(options, dependencies);
|
|
2608
2223
|
const database = description.database;
|
|
@@ -2610,7 +2225,7 @@ async function openSemanticSearchSession(options, dependencies = {}) {
|
|
|
2610
2225
|
let initialized;
|
|
2611
2226
|
try {
|
|
2612
2227
|
initialized = await withSemanticGenerationWriterLease(database, description.manifest.generation, async () => {
|
|
2613
|
-
const projection2 = await prepareSemanticProjection(description,
|
|
2228
|
+
const projection2 = await prepareSemanticProjection(description, snapshot.notes);
|
|
2614
2229
|
let store2;
|
|
2615
2230
|
try {
|
|
2616
2231
|
store2 = await openStore(projection2.root, database, embeddingModel.source, dependencies, requireStoreLocalVectorBoundary);
|
|
@@ -2725,4 +2340,4 @@ async function searchSemanticVault(options, dependencies = {}) {
|
|
|
2725
2340
|
}
|
|
2726
2341
|
}
|
|
2727
2342
|
|
|
2728
|
-
export {
|
|
2343
|
+
export { recommendedEmbeddingModel, recommendedEmbeddingModelSha256, MAX_EMBEDDING_MODEL_BYTES, MAX_SEMANTIC_DATABASE_IDENTITY_BYTES, qmdIndexerVersion, sha256EmbeddingModelFile, createVerifiedEmbeddingModelLease, semanticDatabasePath, checkpointSemanticWarmCache, attestSemanticWarmCache, openSemanticWarmSearchSession, indexSemanticVault, openSemanticSearchSession, searchSemanticVault };
|