@mfdaves/okf-mcp 0.9.1 → 0.13.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +100 -0
  2. package/README.md +452 -26
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/distribution/reference-bundle.md +10 -3
  5. package/okf/okf-mcp/index.md +2 -1
  6. package/okf/okf-mcp/interfaces/http-authoring-api.md +11 -2
  7. package/okf/okf-mcp/interfaces/mcp-http.md +64 -0
  8. package/okf/okf-mcp/interfaces/mcp-tools.md +25 -6
  9. package/okf/okf-mcp/overview/okf-mcp.md +16 -6
  10. package/okf/okf-mcp/policies/authoring-safety.md +28 -5
  11. package/okf/okf-mcp/runtime/indexer.md +14 -1
  12. package/okf/okf-mcp/runtime/mcp-server.md +16 -8
  13. package/okf/okf-mcp/specs/producer-api.md +16 -2
  14. package/package.json +17 -5
  15. package/server.json +38 -2
  16. package/src/assets.js +165 -73
  17. package/src/authoring/candidates.js +434 -0
  18. package/src/authoring/git-preflight.js +225 -0
  19. package/src/authoring/git-publish.js +783 -0
  20. package/src/authoring/live.js +757 -0
  21. package/src/authoring/paths.js +112 -0
  22. package/src/authoring/policy-hook.js +221 -0
  23. package/src/authoring/proposal-policy.js +159 -0
  24. package/src/authoring/receipts.js +373 -0
  25. package/src/authoring/render.js +38 -0
  26. package/src/authoring/shared.js +75 -0
  27. package/src/authoring/staging.js +216 -0
  28. package/src/authoring.js +386 -287
  29. package/src/cache/files.js +147 -0
  30. package/src/cache/lock.js +201 -0
  31. package/src/cache/paths.js +197 -0
  32. package/src/cache/store.js +703 -0
  33. package/src/catalog-text.js +19 -0
  34. package/src/cli/commands.js +448 -0
  35. package/src/cli/http-options.js +193 -0
  36. package/src/cli/options.js +554 -0
  37. package/src/cli/usage.js +96 -0
  38. package/src/cli.js +18 -932
  39. package/src/computation.js +99 -61
  40. package/src/context/assemble.js +567 -0
  41. package/src/context/cli-options.js +88 -0
  42. package/src/context/score.js +234 -0
  43. package/src/context/sections.js +274 -0
  44. package/src/embed/interface.js +160 -0
  45. package/src/embed/openai.js +715 -0
  46. package/src/embed/prepare.js +182 -0
  47. package/src/embed/units.js +96 -0
  48. package/src/error-details.js +53 -0
  49. package/src/git-source.js +112 -52
  50. package/src/graph.js +95 -37
  51. package/src/http-server.js +36 -17
  52. package/src/index.js +15 -0
  53. package/src/indexer/assemble.js +566 -0
  54. package/src/indexer/build.js +290 -0
  55. package/src/indexer/cache.js +81 -0
  56. package/src/indexer/edges.js +430 -0
  57. package/src/indexer/index.js +44 -0
  58. package/src/indexer/paths.js +178 -0
  59. package/src/indexer/resolve.js +282 -0
  60. package/src/indexer/update.js +720 -0
  61. package/src/indexer.js +3 -1068
  62. package/src/live-authoring.js +4 -2329
  63. package/src/markdown.js +21 -16
  64. package/src/mcp/auth.js +409 -0
  65. package/src/mcp/http.js +317 -0
  66. package/src/mcp/principal.js +59 -0
  67. package/src/mcp/registry.js +356 -0
  68. package/src/mcp/resources.js +45 -0
  69. package/src/mcp/server.js +567 -0
  70. package/src/mcp/tools/catalog.js +247 -0
  71. package/src/mcp/tools/computation.js +83 -0
  72. package/src/mcp/tools/context.js +133 -0
  73. package/src/mcp/tools/graph.js +167 -0
  74. package/src/mcp/tools/live-authoring.js +316 -0
  75. package/src/mcp/tools/migration.js +86 -0
  76. package/src/mcp/tools/producers.js +79 -0
  77. package/src/mcp/tools/proposals.js +225 -0
  78. package/src/mcp/tools/provenance.js +87 -0
  79. package/src/mcp/tools/remote.js +79 -0
  80. package/src/mcp/tools/search.js +242 -0
  81. package/src/mcp-server.js +3 -1611
  82. package/src/migration.js +431 -253
  83. package/src/parser.js +245 -130
  84. package/src/plugins.js +71 -48
  85. package/src/policy/write-policy.js +713 -0
  86. package/src/producer-publisher.js +202 -78
  87. package/src/producers.js +311 -103
  88. package/src/project.js +346 -93
  89. package/src/remote.js +96 -61
  90. package/src/search/cli-options.js +107 -0
  91. package/src/search/confidence.js +61 -0
  92. package/src/search/dense.js +304 -0
  93. package/src/search/embedding-cli.js +57 -0
  94. package/src/search/embedding-config.js +217 -0
  95. package/src/search/fusion.js +249 -0
  96. package/src/search/hybrid.js +526 -0
  97. package/src/search/languages.js +200 -0
  98. package/src/search/lexical.js +695 -0
  99. package/src/search/query.js +620 -0
  100. package/src/search/stem/among.js +43 -0
  101. package/src/search/stem/base-stemmer.js +311 -0
  102. package/src/search/stem/en-regions.js +456 -0
  103. package/src/search/stem/en-suffixes.js +664 -0
  104. package/src/search/stem/en-tables.js +176 -0
  105. package/src/search/stem/en.js +238 -0
  106. package/src/search/stem/it-regions.js +459 -0
  107. package/src/search/stem/it-suffixes.js +471 -0
  108. package/src/search/stem/it-tables.js +257 -0
  109. package/src/search/stem/it.js +155 -0
  110. package/src/search-index.js +2 -81
  111. package/src/search.js +2 -252
  112. package/src/store.js +134 -81
  113. package/src/tool-packs.js +346 -0
  114. package/src/v02.js +301 -105
  115. package/src/validation.js +24 -20
@@ -0,0 +1,703 @@
1
+ "use strict";
2
+
3
+ // Vector cache outside the bundle (docs/plan-1.0.md R-9, O-3, S-1, S-2).
4
+ // Vectors are derived data: the Markdown files stay the source of truth, so
5
+ // the cache can be deleted at any moment and is rebuilt from the content
6
+ // hashes of the units. Where it may live is decided in ./paths.js, the lock
7
+ // protocol in ./lock.js, the file primitives in ./files.js.
8
+ //
9
+ // Layout of a cache directory:
10
+ //
11
+ // manifest.json { format, formatVersion, embedderId, dimension, count,
12
+ // generation, files: { units, vectors },
13
+ // digests: { units, vectors },
14
+ // retired: [{ generation, retiredAt }], updatedAt }
15
+ // units.<generation>.json [{ id, hash }] in row order; hash = sha256 of the unit text
16
+ // vectors.<generation>.f32 little-endian float32, row-major, count x dimension
17
+ // lock { token, pid, createdAt, generation } during a write
18
+ //
19
+ // Rows are keyed by hash: a second unit with identical text shares the row.
20
+ // Every write holds the lock, produces a new generation of the two data files
21
+ // and then replaces the manifest atomically, so a reader never sees rows of
22
+ // one generation paired with vectors of another; the manifest's SHA-256
23
+ // digests and the finiteness of every stored value are verified on open. A
24
+ // reader loads its generation into memory once and never goes back to the
25
+ // files by name. The manifest records when each superseded generation was
26
+ // retired; garbage collection after a committed write removes at once the
27
+ // generations this process wrote earlier, retired foreign generations once
28
+ // their retirement is older than the grace period, and never-committed
29
+ // orphans once their files are. A different embedder, dimension or format, a
30
+ // corrupt or tampered file, or a missing directory opens an empty cache and
31
+ // reports why.
32
+ //
33
+ // The directory is re-verified (real path unchanged since open, not a link,
34
+ // outside the root) by every mutation: its creation (ancestors checked), each
35
+ // O_EXCL attempt at the lock, each write (again right before the temporary
36
+ // file is renamed into place and before it is cleaned up), rename, link and
37
+ // unlink. Node has no openat, so this narrows the window between the check
38
+ // and the system call on every platform and cannot close it entirely.
39
+ // Writes of one store instance are serialised; across processes the lock
40
+ // serialises them, and a crashed writer blocks others until lockStaleMs.
41
+
42
+ const crypto = require("node:crypto");
43
+ const fs = require("node:fs");
44
+ const os = require("node:os");
45
+ const path = require("node:path");
46
+
47
+ const { createExclusive, mtimeBefore, readRegularFile, writeAtomic } = require("./files");
48
+ const {
49
+ GENERATION_PATTERN,
50
+ LOCK_FILE,
51
+ LOCK_PARKED_PREFIXES,
52
+ acquireLock,
53
+ newLockToken,
54
+ readLock,
55
+ releaseLock,
56
+ } = require("./lock");
57
+ const {
58
+ VectorCacheError,
59
+ assertNotSymlink,
60
+ assertSafeCachePath,
61
+ isInside,
62
+ realCacheDir,
63
+ realpathOfExistingAncestors,
64
+ realpathOrResolve,
65
+ resolveCacheDir,
66
+ } = require("./paths");
67
+
68
+ const VECTOR_CACHE_FORMAT = "okf-vector-cache";
69
+ const VECTOR_CACHE_FORMAT_VERSION = 1;
70
+ const MANIFEST_FILE = "manifest.json";
71
+ const GENERATION_FILE = /^(units|vectors)\.([0-9a-z]{1,16}-[0-9a-f]{8})\.(json|f32)$/;
72
+ const LOCK_STALE_MS = 10 * 60 * 1000;
73
+ const LOCK_WAIT_MS = 2000;
74
+ const GC_GRACE_MS = 15 * 60 * 1000;
75
+ // The retired list keeps every entry younger than the grace period; the cap
76
+ // applies only to entries past it, which are collectable anyway. The parse
77
+ // bound guards against a pathological manifest, not against young entries.
78
+ const MAX_RETIRED = 64;
79
+ const MAX_RETIRED_PARSE = 4096;
80
+ const HASH_PATTERN = /^[a-f0-9]{64}$/;
81
+ const MAX_DIMENSION = 65536;
82
+ const MIN_CAPACITY_ROWS = 64;
83
+
84
+ function movedError() {
85
+ return new VectorCacheError(
86
+ "Cache directory no longer resolves to the location it was opened at.",
87
+ "invalid_cache_dir",
88
+ );
89
+ }
90
+
91
+ function littleEndian(buffer) {
92
+ if (os.endianness() === "LE") return buffer;
93
+ const swapped = Buffer.from(buffer);
94
+ swapped.swap32();
95
+ return swapped;
96
+ }
97
+
98
+ function sha256Hex(buffer) {
99
+ return crypto.createHash("sha256").update(buffer).digest("hex");
100
+ }
101
+
102
+ function newGeneration() {
103
+ return `${Date.now().toString(36)}-${crypto.randomBytes(4).toString("hex")}`;
104
+ }
105
+
106
+ function parseRetired(value) {
107
+ if (!Array.isArray(value)) return [];
108
+ return value
109
+ .filter(
110
+ (entry) =>
111
+ entry &&
112
+ typeof entry.generation === "string" &&
113
+ GENERATION_PATTERN.test(entry.generation) &&
114
+ typeof entry.retiredAt === "string" &&
115
+ Number.isFinite(Date.parse(entry.retiredAt)),
116
+ )
117
+ .map((entry) => ({ generation: entry.generation, retiredAt: entry.retiredAt }))
118
+ .slice(-MAX_RETIRED_PARSE);
119
+ }
120
+
121
+ function parseManifest(buffer, embedderId, dimension) {
122
+ if (!buffer) return { reason: "manifest_missing" };
123
+ let manifest;
124
+ try {
125
+ manifest = JSON.parse(buffer.toString("utf8"));
126
+ } catch {
127
+ return { reason: "manifest_corrupt" };
128
+ }
129
+ if (!manifest || typeof manifest !== "object" || !Number.isSafeInteger(manifest.count)) {
130
+ return { reason: "manifest_corrupt" };
131
+ }
132
+ if (
133
+ manifest.format !== VECTOR_CACHE_FORMAT ||
134
+ manifest.formatVersion !== VECTOR_CACHE_FORMAT_VERSION
135
+ ) {
136
+ return { reason: "format_mismatch" };
137
+ }
138
+ if (manifest.embedderId !== embedderId) return { reason: "embedder_mismatch" };
139
+ if (manifest.dimension !== dimension) return { reason: "dimension_mismatch" };
140
+ const { generation, files, digests } = manifest;
141
+ if (
142
+ manifest.count < 0 ||
143
+ typeof generation !== "string" ||
144
+ !GENERATION_PATTERN.test(generation) ||
145
+ !files ||
146
+ files.units !== `units.${generation}.json` ||
147
+ files.vectors !== `vectors.${generation}.f32` ||
148
+ !digests ||
149
+ !HASH_PATTERN.test(String(digests.units)) ||
150
+ !HASH_PATTERN.test(String(digests.vectors))
151
+ ) {
152
+ return { reason: "manifest_corrupt" };
153
+ }
154
+ return { manifest: { ...manifest, retired: parseRetired(manifest.retired) } };
155
+ }
156
+
157
+ function parseUnits(buffer, count) {
158
+ let rows;
159
+ try {
160
+ rows = JSON.parse(buffer.toString("utf8"));
161
+ } catch {
162
+ return null;
163
+ }
164
+ if (!Array.isArray(rows) || rows.length !== count) return null;
165
+ const seen = new Set();
166
+ const parsed = [];
167
+ for (const row of rows) {
168
+ if (
169
+ !row ||
170
+ typeof row.id !== "string" ||
171
+ typeof row.hash !== "string" ||
172
+ !HASH_PATTERN.test(row.hash) ||
173
+ seen.has(row.hash)
174
+ ) {
175
+ return null;
176
+ }
177
+ seen.add(row.hash);
178
+ parsed.push({ id: row.id, hash: row.hash });
179
+ }
180
+ return parsed;
181
+ }
182
+
183
+ // A vector must have the store's dimension and every value must still be a
184
+ // finite number once stored as float32 (1e100 becomes Infinity).
185
+ function toVector(value, dimension) {
186
+ const source = value instanceof Float32Array ? value : Array.isArray(value) ? value : null;
187
+ if (!source) return { reason: "not_a_vector" };
188
+ if (source.length !== dimension) return { reason: "wrong_length" };
189
+ const vector = new Float32Array(dimension);
190
+ for (let index = 0; index < dimension; index += 1) {
191
+ const entry = source[index];
192
+ if (typeof entry !== "number" || !Number.isFinite(entry)) return { reason: "not_finite" };
193
+ vector[index] = entry;
194
+ if (!Number.isFinite(vector[index])) return { reason: "not_finite_as_float32" };
195
+ }
196
+ return { vector };
197
+ }
198
+
199
+ function allFinite(data, length) {
200
+ for (let index = 0; index < length; index += 1) {
201
+ if (!Number.isFinite(data[index])) return false;
202
+ }
203
+ return true;
204
+ }
205
+
206
+ class VectorCache {
207
+ constructor(dir, embedderId, dimension, options) {
208
+ const config = options || {};
209
+ this.dir = dir;
210
+ this.embedderId = embedderId;
211
+ this.dimension = dimension;
212
+ this.realRoot = config.realRoot || null;
213
+ this.allowInsideRoot = config.allowInsideRoot === true;
214
+ this.lockWaitMs = Number.isSafeInteger(config.lockWaitMs) ? config.lockWaitMs : LOCK_WAIT_MS;
215
+ this.lockStaleMs = Number.isSafeInteger(config.lockStaleMs)
216
+ ? config.lockStaleMs
217
+ : LOCK_STALE_MS;
218
+ this.generation = null;
219
+ this.ownGenerations = new Set();
220
+ this.rows = [];
221
+ this.byHash = new Map();
222
+ this.data = new Float32Array(0);
223
+ this.opened = null;
224
+ this.writeQueue = Promise.resolve();
225
+ }
226
+
227
+ get size() {
228
+ return this.rows.length;
229
+ }
230
+
231
+ load() {
232
+ let directory;
233
+ try {
234
+ directory = fs.lstatSync(this.dir);
235
+ } catch {
236
+ this.opened = { rebuilt: true, reason: "missing", rows: 0 };
237
+ return this.opened;
238
+ }
239
+ if (directory.isSymbolicLink()) {
240
+ throw new VectorCacheError(
241
+ "Cache directory must not be a symbolic link.",
242
+ "invalid_cache_dir",
243
+ );
244
+ }
245
+ if (!directory.isDirectory()) {
246
+ throw new VectorCacheError("Cache directory path is not a directory.", "invalid_cache_dir");
247
+ }
248
+ let outcome;
249
+ try {
250
+ outcome = this.readFiles();
251
+ } catch {
252
+ outcome = { reason: "unreadable" };
253
+ }
254
+ if (outcome.reason) {
255
+ this.opened = { rebuilt: true, reason: outcome.reason, rows: 0 };
256
+ return this.opened;
257
+ }
258
+ this.rows = outcome.rows;
259
+ this.data = outcome.data;
260
+ this.generation = outcome.generation;
261
+ this.byHash = new Map(this.rows.map((row, index) => [row.hash, index]));
262
+ this.opened = { rebuilt: false, reason: null, rows: this.rows.length };
263
+ return this.opened;
264
+ }
265
+
266
+ // Everything a reader needs is in memory after this; the files are never
267
+ // consulted again by name, so a later commit by another process cannot
268
+ // pull a generation out from under an open store.
269
+ readFiles() {
270
+ const parsed = parseManifest(
271
+ readRegularFile(path.join(this.dir, MANIFEST_FILE)),
272
+ this.embedderId,
273
+ this.dimension,
274
+ );
275
+ if (parsed.reason) return parsed;
276
+ const { manifest } = parsed;
277
+ const count = manifest.count;
278
+ const unitBytes = readRegularFile(path.join(this.dir, manifest.files.units));
279
+ if (!unitBytes) return { reason: "units_corrupt" };
280
+ if (sha256Hex(unitBytes) !== manifest.digests.units) return { reason: "integrity_mismatch" };
281
+ const rows = parseUnits(unitBytes, count);
282
+ if (!rows) return { reason: "units_corrupt" };
283
+ const bytes = readRegularFile(path.join(this.dir, manifest.files.vectors));
284
+ if (!bytes) return { reason: "vectors_corrupt" };
285
+ if (sha256Hex(bytes) !== manifest.digests.vectors) return { reason: "integrity_mismatch" };
286
+ if (bytes.byteLength !== count * this.dimension * 4) return { reason: "vectors_corrupt" };
287
+ const data = new Float32Array(Math.max(count, MIN_CAPACITY_ROWS) * this.dimension);
288
+ new Uint8Array(data.buffer, 0, bytes.byteLength).set(littleEndian(bytes));
289
+ if (!allFinite(data, count * this.dimension)) return { reason: "vectors_corrupt" };
290
+ return { rows, data, generation: manifest.generation };
291
+ }
292
+
293
+ has(hash) {
294
+ return this.byHash.has(hash);
295
+ }
296
+
297
+ // A read-only view into the store, valid as a snapshot until the same hash
298
+ // is written again.
299
+ get(hash) {
300
+ const row = this.byHash.get(hash);
301
+ if (row === undefined) return null;
302
+ return this.data.subarray(row * this.dimension, (row + 1) * this.dimension);
303
+ }
304
+
305
+ all() {
306
+ return this.rows.map((row, index) => ({
307
+ id: row.id,
308
+ hash: row.hash,
309
+ vector: this.data.subarray(index * this.dimension, (index + 1) * this.dimension),
310
+ }));
311
+ }
312
+
313
+ ensureCapacity(additionalRows) {
314
+ const needed = (this.rows.length + additionalRows) * this.dimension;
315
+ if (needed <= this.data.length) return;
316
+ const capacity = Math.max(needed, this.data.length * 2, MIN_CAPACITY_ROWS * this.dimension);
317
+ const grown = new Float32Array(capacity);
318
+ grown.set(this.data.subarray(0, this.rows.length * this.dimension));
319
+ this.data = grown;
320
+ }
321
+
322
+ // Validates every entry before touching memory, then upserts by hash and
323
+ // persists. Resolves to how many rows were added and how many replaced.
324
+ async putMany(entries) {
325
+ if (!Array.isArray(entries)) {
326
+ throw new VectorCacheError("putMany expects an array of entries.", "invalid_cache_entry");
327
+ }
328
+ const prepared = entries.map((entry, position) => {
329
+ if (
330
+ !entry ||
331
+ typeof entry.id !== "string" ||
332
+ !entry.id ||
333
+ typeof entry.hash !== "string" ||
334
+ !HASH_PATTERN.test(entry.hash)
335
+ ) {
336
+ throw new VectorCacheError(
337
+ `Cache entry ${position} must carry an id and a sha256 hash.`,
338
+ "invalid_cache_entry",
339
+ { position },
340
+ );
341
+ }
342
+ const converted = toVector(entry.vector, this.dimension);
343
+ if (!converted.vector) {
344
+ throw new VectorCacheError(
345
+ `Cache entry ${position} must carry a vector of ${this.dimension} finite float32 values.`,
346
+ "invalid_vector",
347
+ { position, reason: converted.reason },
348
+ );
349
+ }
350
+ return { id: entry.id, hash: entry.hash, vector: converted.vector };
351
+ });
352
+ const result = { added: 0, replaced: 0 };
353
+ if (prepared.length === 0) return result;
354
+ this.ensureCapacity(prepared.length);
355
+ for (const entry of prepared) {
356
+ let row = this.byHash.get(entry.hash);
357
+ if (row === undefined) {
358
+ row = this.rows.length;
359
+ this.rows.push({ id: entry.id, hash: entry.hash });
360
+ this.byHash.set(entry.hash, row);
361
+ result.added += 1;
362
+ } else {
363
+ this.rows[row] = { id: entry.id, hash: entry.hash };
364
+ result.replaced += 1;
365
+ }
366
+ this.data.set(entry.vector, row * this.dimension);
367
+ }
368
+ await this.flush();
369
+ return result;
370
+ }
371
+
372
+ // Drops every row whose hash is not in keepHashes; resolves to how many went.
373
+ async prune(keepHashes) {
374
+ const keep = keepHashes instanceof Set ? keepHashes : new Set(keepHashes || []);
375
+ const survivors = [];
376
+ for (let index = 0; index < this.rows.length; index += 1) {
377
+ if (keep.has(this.rows[index].hash)) survivors.push(index);
378
+ }
379
+ const removed = this.rows.length - survivors.length;
380
+ if (removed === 0) return 0;
381
+ const compacted = new Float32Array(
382
+ Math.max(survivors.length, MIN_CAPACITY_ROWS) * this.dimension,
383
+ );
384
+ const rows = [];
385
+ survivors.forEach((source, target) => {
386
+ compacted.set(
387
+ this.data.subarray(source * this.dimension, (source + 1) * this.dimension),
388
+ target * this.dimension,
389
+ );
390
+ rows.push(this.rows[source]);
391
+ });
392
+ this.rows = rows;
393
+ this.data = compacted;
394
+ this.byHash = new Map(rows.map((row, index) => [row.hash, index]));
395
+ await this.flush();
396
+ return removed;
397
+ }
398
+
399
+ // The directory must still be the one opened: real path unchanged, not a
400
+ // link, outside the root. Cheap enough to run before every operation group.
401
+ verifyLocation() {
402
+ assertNotSymlink(this.dir);
403
+ let actual;
404
+ try {
405
+ actual = fs.realpathSync(this.dir);
406
+ } catch {
407
+ throw movedError();
408
+ }
409
+ this.assertOpenedLocation(actual);
410
+ }
411
+
412
+ // Before the directory exists: its existing ancestors must still resolve
413
+ // to where they did at open, which also keeps it outside the root.
414
+ verifyAncestors() {
415
+ assertNotSymlink(this.dir);
416
+ this.assertOpenedLocation(realpathOfExistingAncestors(this.dir));
417
+ }
418
+
419
+ assertOpenedLocation(actual) {
420
+ if (actual !== this.dir) throw movedError();
421
+ if (this.realRoot && !this.allowInsideRoot && isInside(this.realRoot, actual, path)) {
422
+ throw new VectorCacheError(
423
+ "Cache directory resolves inside the bundle root.",
424
+ "cache_inside_root",
425
+ );
426
+ }
427
+ }
428
+
429
+ locationIntact() {
430
+ try {
431
+ this.verifyLocation();
432
+ return true;
433
+ } catch {
434
+ return false;
435
+ }
436
+ }
437
+
438
+ // Recreates the directory when it was deleted underneath a running process,
439
+ // after checking that its ancestors still lead where they did at open.
440
+ ensureWritableDirectory() {
441
+ this.guardedMkdir();
442
+ this.verifyLocation();
443
+ }
444
+
445
+ // Seams for tests that inject failures; production never replaces them.
446
+ makeDirectory(dir) {
447
+ fs.mkdirSync(dir, { recursive: true });
448
+ }
449
+
450
+ writeFile(target, content) {
451
+ writeAtomic(target, content, { verify: () => this.verifyLocation() });
452
+ }
453
+
454
+ createFile(target, content) {
455
+ return createExclusive(target, content);
456
+ }
457
+
458
+ removeFile(target) {
459
+ fs.unlinkSync(target);
460
+ }
461
+
462
+ renameFile(from, to) {
463
+ fs.renameSync(from, to);
464
+ }
465
+
466
+ linkFile(from, to) {
467
+ fs.linkSync(from, to);
468
+ }
469
+
470
+ // Every mutation of the directory goes through these: each verifies the
471
+ // location first, so no call site can skip the check. writeFile verifies
472
+ // again inside writeAtomic, before the rename and before cleanup.
473
+ guardedMkdir() {
474
+ this.verifyAncestors();
475
+ this.makeDirectory(this.dir);
476
+ }
477
+
478
+ guardedCreate(target, content) {
479
+ this.verifyLocation();
480
+ return this.createFile(target, content);
481
+ }
482
+
483
+ guardedWrite(target, content) {
484
+ this.verifyLocation();
485
+ return this.writeFile(target, content);
486
+ }
487
+
488
+ guardedRemove(target) {
489
+ this.verifyLocation();
490
+ this.removeFile(target);
491
+ }
492
+
493
+ guardedRename(from, to) {
494
+ this.verifyLocation();
495
+ this.renameFile(from, to);
496
+ }
497
+
498
+ guardedLink(from, to) {
499
+ this.verifyLocation();
500
+ this.linkFile(from, to);
501
+ }
502
+
503
+ hasGenerationFiles(entry) {
504
+ return (
505
+ fs.existsSync(path.join(this.dir, `units.${entry.generation}.json`)) ||
506
+ fs.existsSync(path.join(this.dir, `vectors.${entry.generation}.f32`))
507
+ );
508
+ }
509
+
510
+ // The committed manifest as it is on disk right now, read under the lock,
511
+ // so retirement is recorded for whatever generation this write supersedes.
512
+ committedState() {
513
+ const parsed = parseManifest(
514
+ readRegularFile(path.join(this.dir, MANIFEST_FILE)),
515
+ this.embedderId,
516
+ this.dimension,
517
+ );
518
+ if (parsed.reason) return { generation: null, retired: [] };
519
+ return { generation: parsed.manifest.generation, retired: parsed.manifest.retired };
520
+ }
521
+
522
+ // Writes of one instance run one after another.
523
+ flush() {
524
+ const run = this.writeQueue.then(() => this.flushNow());
525
+ this.writeQueue = run.catch(() => {});
526
+ return run;
527
+ }
528
+
529
+ // Writes a new generation of the data files under the lock, replaces the
530
+ // manifest atomically as the commit point, then collects garbage.
531
+ async flushNow() {
532
+ this.ensureWritableDirectory();
533
+ const generation = newGeneration();
534
+ const token = newLockToken();
535
+ const io = {
536
+ create: (target, content) => this.guardedCreate(target, content),
537
+ rename: (from, to) => this.guardedRename(from, to),
538
+ remove: (target) => this.guardedRemove(target),
539
+ link: (from, to) => this.guardedLink(from, to),
540
+ };
541
+ const lockPath = await acquireLock(this.dir, {
542
+ token,
543
+ generation,
544
+ waitMs: this.lockWaitMs,
545
+ staleMs: this.lockStaleMs,
546
+ io,
547
+ });
548
+ try {
549
+ this.verifyLocation();
550
+ this.ownGenerations.add(generation);
551
+ const previous = this.committedState();
552
+ const now = new Date().toISOString();
553
+ const cutoff = Date.now() - GC_GRACE_MS;
554
+ const kept = previous.retired.filter(
555
+ (entry) => entry.generation !== generation && this.hasGenerationFiles(entry),
556
+ );
557
+ // Entries younger than the grace period are never forgotten; the cap
558
+ // applies to entries past it, and a forgotten entry's files are
559
+ // collected in this same step.
560
+ const past = kept.filter((entry) => Date.parse(entry.retiredAt) < cutoff);
561
+ const forgotten = past.slice(0, Math.max(0, past.length - MAX_RETIRED));
562
+ const retired = past
563
+ .slice(forgotten.length)
564
+ .concat(kept.filter((entry) => Date.parse(entry.retiredAt) >= cutoff))
565
+ .concat(
566
+ // An own generation is collected right after this commit, so only
567
+ // a foreign one is worth recording.
568
+ previous.generation &&
569
+ previous.generation !== generation &&
570
+ !this.ownGenerations.has(previous.generation)
571
+ ? [{ generation: previous.generation, retiredAt: now }]
572
+ : [],
573
+ );
574
+ const count = this.rows.length;
575
+ const files = { units: `units.${generation}.json`, vectors: `vectors.${generation}.f32` };
576
+ const unitBytes = Buffer.from(`${JSON.stringify(this.rows)}\n`, "utf8");
577
+ const vectorBytes = littleEndian(
578
+ Buffer.from(this.data.buffer, this.data.byteOffset, count * this.dimension * 4),
579
+ );
580
+ await this.guardedWrite(path.join(this.dir, files.units), unitBytes);
581
+ await this.guardedWrite(path.join(this.dir, files.vectors), vectorBytes);
582
+ const manifest = {
583
+ format: VECTOR_CACHE_FORMAT,
584
+ formatVersion: VECTOR_CACHE_FORMAT_VERSION,
585
+ embedderId: this.embedderId,
586
+ dimension: this.dimension,
587
+ count,
588
+ generation,
589
+ files,
590
+ digests: { units: sha256Hex(unitBytes), vectors: sha256Hex(vectorBytes) },
591
+ retired,
592
+ updatedAt: now,
593
+ };
594
+ await this.guardedWrite(
595
+ path.join(this.dir, MANIFEST_FILE),
596
+ Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, "utf8"),
597
+ );
598
+ this.generation = generation;
599
+ this.collectGarbage(retired.concat(forgotten));
600
+ } finally {
601
+ releaseLock(lockPath, token, io);
602
+ }
603
+ }
604
+
605
+ // Generations this process wrote earlier go at once: nothing else can
606
+ // reference them, and readers hold their rows in memory. A retired foreign
607
+ // generation goes once its retirement is older than the grace period; a
608
+ // never-committed orphan, a stale temporary file or a lock parked by a
609
+ // reclaim or a release once its files are. Best effort: a failure here
610
+ // never fails the write that just completed, and a directory that no
611
+ // longer verifies stops the collection before anything is unlinked.
612
+ collectGarbage(retired) {
613
+ let entries;
614
+ try {
615
+ this.verifyLocation();
616
+ entries = fs.readdirSync(this.dir, { withFileTypes: true });
617
+ } catch {
618
+ return;
619
+ }
620
+ const locked = readLock(path.join(this.dir, LOCK_FILE))?.generation || null;
621
+ const cutoff = Date.now() - GC_GRACE_MS;
622
+ const retiredAt = new Map(
623
+ retired.map((entry) => [entry.generation, Date.parse(entry.retiredAt)]),
624
+ );
625
+ for (const entry of entries) {
626
+ if (entry.isDirectory()) continue;
627
+ const target = path.join(this.dir, entry.name);
628
+ const match = GENERATION_FILE.exec(entry.name);
629
+ let remove = false;
630
+ if (match) {
631
+ const generation = match[2];
632
+ if (generation === this.generation || generation === locked) continue;
633
+ if (this.ownGenerations.has(generation)) remove = true;
634
+ else if (retiredAt.has(generation)) remove = retiredAt.get(generation) < cutoff;
635
+ else remove = mtimeBefore(target, cutoff);
636
+ } else if (
637
+ entry.name.endsWith(".tmp") ||
638
+ LOCK_PARKED_PREFIXES.some((prefix) => entry.name.startsWith(prefix))
639
+ ) {
640
+ remove = mtimeBefore(target, cutoff);
641
+ }
642
+ if (!remove) continue;
643
+ try {
644
+ this.guardedRemove(target);
645
+ } catch (error) {
646
+ if (error instanceof VectorCacheError) return;
647
+ // Another process may have removed it first; the next write retries.
648
+ }
649
+ }
650
+ }
651
+ }
652
+
653
+ // options: embedderId, dimension (required); root (containment is checked at
654
+ // open and before every write), allowInsideRoot, lockWaitMs, lockStaleMs.
655
+ function openVectorCache(dir, options) {
656
+ const config = options || {};
657
+ const resolvedDir = realCacheDir(assertSafeCachePath(dir, path));
658
+ const { embedderId, dimension } = config;
659
+ if (
660
+ typeof embedderId !== "string" ||
661
+ !embedderId.trim() ||
662
+ embedderId.length > 256 ||
663
+ /[\s\p{Cc}]/u.test(embedderId)
664
+ ) {
665
+ throw new VectorCacheError(
666
+ "Vector cache requires the embedder id as a single token.",
667
+ "invalid_cache_options",
668
+ { field: "embedderId" },
669
+ );
670
+ }
671
+ if (!Number.isSafeInteger(dimension) || dimension <= 0 || dimension > MAX_DIMENSION) {
672
+ throw new VectorCacheError(
673
+ "Vector cache requires a positive integer dimension.",
674
+ "invalid_cache_options",
675
+ { field: "dimension" },
676
+ );
677
+ }
678
+ const realRoot =
679
+ typeof config.root === "string" && config.root.trim() ? realpathOrResolve(config.root) : null;
680
+ if (realRoot && config.allowInsideRoot !== true && isInside(realRoot, resolvedDir, path)) {
681
+ throw new VectorCacheError(
682
+ "Cache directory resolves inside the bundle root.",
683
+ "cache_inside_root",
684
+ );
685
+ }
686
+ const cache = new VectorCache(resolvedDir, embedderId, dimension, {
687
+ realRoot,
688
+ allowInsideRoot: config.allowInsideRoot,
689
+ lockWaitMs: config.lockWaitMs,
690
+ lockStaleMs: config.lockStaleMs,
691
+ });
692
+ cache.load();
693
+ return cache;
694
+ }
695
+
696
+ module.exports = {
697
+ VECTOR_CACHE_FORMAT,
698
+ VECTOR_CACHE_FORMAT_VERSION,
699
+ VectorCache,
700
+ VectorCacheError,
701
+ openVectorCache,
702
+ resolveCacheDir,
703
+ };