@mfdaves/okf-mcp 0.9.0 → 0.12.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +176 -19
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/policies/authoring-safety.md +11 -1
- package/package.json +17 -5
- package/server.json +2 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +391 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +776 -0
- package/src/authoring/live.js +789 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +137 -0
- package/src/authoring/receipts.js +329 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli.js +184 -151
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/registry.js +328 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +472 -0
- package/src/mcp/tools/catalog.js +244 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +213 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +195 -77
- package/src/producers.js +265 -101
- package/src/project.js +277 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
|
@@ -0,0 +1,703 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Vector cache outside the bundle (docs/plan-1.0.md R-9, O-3, S-1, S-2).
|
|
4
|
+
// Vectors are derived data: the Markdown files stay the source of truth, so
|
|
5
|
+
// the cache can be deleted at any moment and is rebuilt from the content
|
|
6
|
+
// hashes of the units. Where it may live is decided in ./paths.js, the lock
|
|
7
|
+
// protocol in ./lock.js, the file primitives in ./files.js.
|
|
8
|
+
//
|
|
9
|
+
// Layout of a cache directory:
|
|
10
|
+
//
|
|
11
|
+
// manifest.json { format, formatVersion, embedderId, dimension, count,
|
|
12
|
+
// generation, files: { units, vectors },
|
|
13
|
+
// digests: { units, vectors },
|
|
14
|
+
// retired: [{ generation, retiredAt }], updatedAt }
|
|
15
|
+
// units.<generation>.json [{ id, hash }] in row order; hash = sha256 of the unit text
|
|
16
|
+
// vectors.<generation>.f32 little-endian float32, row-major, count x dimension
|
|
17
|
+
// lock { token, pid, createdAt, generation } during a write
|
|
18
|
+
//
|
|
19
|
+
// Rows are keyed by hash: a second unit with identical text shares the row.
|
|
20
|
+
// Every write holds the lock, produces a new generation of the two data files
|
|
21
|
+
// and then replaces the manifest atomically, so a reader never sees rows of
|
|
22
|
+
// one generation paired with vectors of another; the manifest's SHA-256
|
|
23
|
+
// digests and the finiteness of every stored value are verified on open. A
|
|
24
|
+
// reader loads its generation into memory once and never goes back to the
|
|
25
|
+
// files by name. The manifest records when each superseded generation was
|
|
26
|
+
// retired; garbage collection after a committed write removes at once the
|
|
27
|
+
// generations this process wrote earlier, retired foreign generations once
|
|
28
|
+
// their retirement is older than the grace period, and never-committed
|
|
29
|
+
// orphans once their files are. A different embedder, dimension or format, a
|
|
30
|
+
// corrupt or tampered file, or a missing directory opens an empty cache and
|
|
31
|
+
// reports why.
|
|
32
|
+
//
|
|
33
|
+
// The directory is re-verified (real path unchanged since open, not a link,
|
|
34
|
+
// outside the root) by every mutation: its creation (ancestors checked), each
|
|
35
|
+
// O_EXCL attempt at the lock, each write (again right before the temporary
|
|
36
|
+
// file is renamed into place and before it is cleaned up), rename, link and
|
|
37
|
+
// unlink. Node has no openat, so this narrows the window between the check
|
|
38
|
+
// and the system call on every platform and cannot close it entirely.
|
|
39
|
+
// Writes of one store instance are serialised; across processes the lock
|
|
40
|
+
// serialises them, and a crashed writer blocks others until lockStaleMs.
|
|
41
|
+
|
|
42
|
+
const crypto = require("node:crypto");
|
|
43
|
+
const fs = require("node:fs");
|
|
44
|
+
const os = require("node:os");
|
|
45
|
+
const path = require("node:path");
|
|
46
|
+
|
|
47
|
+
const { createExclusive, mtimeBefore, readRegularFile, writeAtomic } = require("./files");
|
|
48
|
+
const {
|
|
49
|
+
GENERATION_PATTERN,
|
|
50
|
+
LOCK_FILE,
|
|
51
|
+
LOCK_PARKED_PREFIXES,
|
|
52
|
+
acquireLock,
|
|
53
|
+
newLockToken,
|
|
54
|
+
readLock,
|
|
55
|
+
releaseLock,
|
|
56
|
+
} = require("./lock");
|
|
57
|
+
const {
|
|
58
|
+
VectorCacheError,
|
|
59
|
+
assertNotSymlink,
|
|
60
|
+
assertSafeCachePath,
|
|
61
|
+
isInside,
|
|
62
|
+
realCacheDir,
|
|
63
|
+
realpathOfExistingAncestors,
|
|
64
|
+
realpathOrResolve,
|
|
65
|
+
resolveCacheDir,
|
|
66
|
+
} = require("./paths");
|
|
67
|
+
|
|
68
|
+
const VECTOR_CACHE_FORMAT = "okf-vector-cache";
|
|
69
|
+
const VECTOR_CACHE_FORMAT_VERSION = 1;
|
|
70
|
+
const MANIFEST_FILE = "manifest.json";
|
|
71
|
+
const GENERATION_FILE = /^(units|vectors)\.([0-9a-z]{1,16}-[0-9a-f]{8})\.(json|f32)$/;
|
|
72
|
+
const LOCK_STALE_MS = 10 * 60 * 1000;
|
|
73
|
+
const LOCK_WAIT_MS = 2000;
|
|
74
|
+
const GC_GRACE_MS = 15 * 60 * 1000;
|
|
75
|
+
// The retired list keeps every entry younger than the grace period; the cap
|
|
76
|
+
// applies only to entries past it, which are collectable anyway. The parse
|
|
77
|
+
// bound guards against a pathological manifest, not against young entries.
|
|
78
|
+
const MAX_RETIRED = 64;
|
|
79
|
+
const MAX_RETIRED_PARSE = 4096;
|
|
80
|
+
const HASH_PATTERN = /^[a-f0-9]{64}$/;
|
|
81
|
+
const MAX_DIMENSION = 65536;
|
|
82
|
+
const MIN_CAPACITY_ROWS = 64;
|
|
83
|
+
|
|
84
|
+
function movedError() {
|
|
85
|
+
return new VectorCacheError(
|
|
86
|
+
"Cache directory no longer resolves to the location it was opened at.",
|
|
87
|
+
"invalid_cache_dir",
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function littleEndian(buffer) {
|
|
92
|
+
if (os.endianness() === "LE") return buffer;
|
|
93
|
+
const swapped = Buffer.from(buffer);
|
|
94
|
+
swapped.swap32();
|
|
95
|
+
return swapped;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function sha256Hex(buffer) {
|
|
99
|
+
return crypto.createHash("sha256").update(buffer).digest("hex");
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function newGeneration() {
|
|
103
|
+
return `${Date.now().toString(36)}-${crypto.randomBytes(4).toString("hex")}`;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function parseRetired(value) {
|
|
107
|
+
if (!Array.isArray(value)) return [];
|
|
108
|
+
return value
|
|
109
|
+
.filter(
|
|
110
|
+
(entry) =>
|
|
111
|
+
entry &&
|
|
112
|
+
typeof entry.generation === "string" &&
|
|
113
|
+
GENERATION_PATTERN.test(entry.generation) &&
|
|
114
|
+
typeof entry.retiredAt === "string" &&
|
|
115
|
+
Number.isFinite(Date.parse(entry.retiredAt)),
|
|
116
|
+
)
|
|
117
|
+
.map((entry) => ({ generation: entry.generation, retiredAt: entry.retiredAt }))
|
|
118
|
+
.slice(-MAX_RETIRED_PARSE);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function parseManifest(buffer, embedderId, dimension) {
|
|
122
|
+
if (!buffer) return { reason: "manifest_missing" };
|
|
123
|
+
let manifest;
|
|
124
|
+
try {
|
|
125
|
+
manifest = JSON.parse(buffer.toString("utf8"));
|
|
126
|
+
} catch {
|
|
127
|
+
return { reason: "manifest_corrupt" };
|
|
128
|
+
}
|
|
129
|
+
if (!manifest || typeof manifest !== "object" || !Number.isSafeInteger(manifest.count)) {
|
|
130
|
+
return { reason: "manifest_corrupt" };
|
|
131
|
+
}
|
|
132
|
+
if (
|
|
133
|
+
manifest.format !== VECTOR_CACHE_FORMAT ||
|
|
134
|
+
manifest.formatVersion !== VECTOR_CACHE_FORMAT_VERSION
|
|
135
|
+
) {
|
|
136
|
+
return { reason: "format_mismatch" };
|
|
137
|
+
}
|
|
138
|
+
if (manifest.embedderId !== embedderId) return { reason: "embedder_mismatch" };
|
|
139
|
+
if (manifest.dimension !== dimension) return { reason: "dimension_mismatch" };
|
|
140
|
+
const { generation, files, digests } = manifest;
|
|
141
|
+
if (
|
|
142
|
+
manifest.count < 0 ||
|
|
143
|
+
typeof generation !== "string" ||
|
|
144
|
+
!GENERATION_PATTERN.test(generation) ||
|
|
145
|
+
!files ||
|
|
146
|
+
files.units !== `units.${generation}.json` ||
|
|
147
|
+
files.vectors !== `vectors.${generation}.f32` ||
|
|
148
|
+
!digests ||
|
|
149
|
+
!HASH_PATTERN.test(String(digests.units)) ||
|
|
150
|
+
!HASH_PATTERN.test(String(digests.vectors))
|
|
151
|
+
) {
|
|
152
|
+
return { reason: "manifest_corrupt" };
|
|
153
|
+
}
|
|
154
|
+
return { manifest: { ...manifest, retired: parseRetired(manifest.retired) } };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function parseUnits(buffer, count) {
|
|
158
|
+
let rows;
|
|
159
|
+
try {
|
|
160
|
+
rows = JSON.parse(buffer.toString("utf8"));
|
|
161
|
+
} catch {
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
if (!Array.isArray(rows) || rows.length !== count) return null;
|
|
165
|
+
const seen = new Set();
|
|
166
|
+
const parsed = [];
|
|
167
|
+
for (const row of rows) {
|
|
168
|
+
if (
|
|
169
|
+
!row ||
|
|
170
|
+
typeof row.id !== "string" ||
|
|
171
|
+
typeof row.hash !== "string" ||
|
|
172
|
+
!HASH_PATTERN.test(row.hash) ||
|
|
173
|
+
seen.has(row.hash)
|
|
174
|
+
) {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
seen.add(row.hash);
|
|
178
|
+
parsed.push({ id: row.id, hash: row.hash });
|
|
179
|
+
}
|
|
180
|
+
return parsed;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// A vector must have the store's dimension and every value must still be a
|
|
184
|
+
// finite number once stored as float32 (1e100 becomes Infinity).
|
|
185
|
+
function toVector(value, dimension) {
|
|
186
|
+
const source = value instanceof Float32Array ? value : Array.isArray(value) ? value : null;
|
|
187
|
+
if (!source) return { reason: "not_a_vector" };
|
|
188
|
+
if (source.length !== dimension) return { reason: "wrong_length" };
|
|
189
|
+
const vector = new Float32Array(dimension);
|
|
190
|
+
for (let index = 0; index < dimension; index += 1) {
|
|
191
|
+
const entry = source[index];
|
|
192
|
+
if (typeof entry !== "number" || !Number.isFinite(entry)) return { reason: "not_finite" };
|
|
193
|
+
vector[index] = entry;
|
|
194
|
+
if (!Number.isFinite(vector[index])) return { reason: "not_finite_as_float32" };
|
|
195
|
+
}
|
|
196
|
+
return { vector };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function allFinite(data, length) {
|
|
200
|
+
for (let index = 0; index < length; index += 1) {
|
|
201
|
+
if (!Number.isFinite(data[index])) return false;
|
|
202
|
+
}
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
class VectorCache {
|
|
207
|
+
constructor(dir, embedderId, dimension, options) {
|
|
208
|
+
const config = options || {};
|
|
209
|
+
this.dir = dir;
|
|
210
|
+
this.embedderId = embedderId;
|
|
211
|
+
this.dimension = dimension;
|
|
212
|
+
this.realRoot = config.realRoot || null;
|
|
213
|
+
this.allowInsideRoot = config.allowInsideRoot === true;
|
|
214
|
+
this.lockWaitMs = Number.isSafeInteger(config.lockWaitMs) ? config.lockWaitMs : LOCK_WAIT_MS;
|
|
215
|
+
this.lockStaleMs = Number.isSafeInteger(config.lockStaleMs)
|
|
216
|
+
? config.lockStaleMs
|
|
217
|
+
: LOCK_STALE_MS;
|
|
218
|
+
this.generation = null;
|
|
219
|
+
this.ownGenerations = new Set();
|
|
220
|
+
this.rows = [];
|
|
221
|
+
this.byHash = new Map();
|
|
222
|
+
this.data = new Float32Array(0);
|
|
223
|
+
this.opened = null;
|
|
224
|
+
this.writeQueue = Promise.resolve();
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
get size() {
|
|
228
|
+
return this.rows.length;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
load() {
|
|
232
|
+
let directory;
|
|
233
|
+
try {
|
|
234
|
+
directory = fs.lstatSync(this.dir);
|
|
235
|
+
} catch {
|
|
236
|
+
this.opened = { rebuilt: true, reason: "missing", rows: 0 };
|
|
237
|
+
return this.opened;
|
|
238
|
+
}
|
|
239
|
+
if (directory.isSymbolicLink()) {
|
|
240
|
+
throw new VectorCacheError(
|
|
241
|
+
"Cache directory must not be a symbolic link.",
|
|
242
|
+
"invalid_cache_dir",
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
if (!directory.isDirectory()) {
|
|
246
|
+
throw new VectorCacheError("Cache directory path is not a directory.", "invalid_cache_dir");
|
|
247
|
+
}
|
|
248
|
+
let outcome;
|
|
249
|
+
try {
|
|
250
|
+
outcome = this.readFiles();
|
|
251
|
+
} catch {
|
|
252
|
+
outcome = { reason: "unreadable" };
|
|
253
|
+
}
|
|
254
|
+
if (outcome.reason) {
|
|
255
|
+
this.opened = { rebuilt: true, reason: outcome.reason, rows: 0 };
|
|
256
|
+
return this.opened;
|
|
257
|
+
}
|
|
258
|
+
this.rows = outcome.rows;
|
|
259
|
+
this.data = outcome.data;
|
|
260
|
+
this.generation = outcome.generation;
|
|
261
|
+
this.byHash = new Map(this.rows.map((row, index) => [row.hash, index]));
|
|
262
|
+
this.opened = { rebuilt: false, reason: null, rows: this.rows.length };
|
|
263
|
+
return this.opened;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Everything a reader needs is in memory after this; the files are never
|
|
267
|
+
// consulted again by name, so a later commit by another process cannot
|
|
268
|
+
// pull a generation out from under an open store.
|
|
269
|
+
readFiles() {
|
|
270
|
+
const parsed = parseManifest(
|
|
271
|
+
readRegularFile(path.join(this.dir, MANIFEST_FILE)),
|
|
272
|
+
this.embedderId,
|
|
273
|
+
this.dimension,
|
|
274
|
+
);
|
|
275
|
+
if (parsed.reason) return parsed;
|
|
276
|
+
const { manifest } = parsed;
|
|
277
|
+
const count = manifest.count;
|
|
278
|
+
const unitBytes = readRegularFile(path.join(this.dir, manifest.files.units));
|
|
279
|
+
if (!unitBytes) return { reason: "units_corrupt" };
|
|
280
|
+
if (sha256Hex(unitBytes) !== manifest.digests.units) return { reason: "integrity_mismatch" };
|
|
281
|
+
const rows = parseUnits(unitBytes, count);
|
|
282
|
+
if (!rows) return { reason: "units_corrupt" };
|
|
283
|
+
const bytes = readRegularFile(path.join(this.dir, manifest.files.vectors));
|
|
284
|
+
if (!bytes) return { reason: "vectors_corrupt" };
|
|
285
|
+
if (sha256Hex(bytes) !== manifest.digests.vectors) return { reason: "integrity_mismatch" };
|
|
286
|
+
if (bytes.byteLength !== count * this.dimension * 4) return { reason: "vectors_corrupt" };
|
|
287
|
+
const data = new Float32Array(Math.max(count, MIN_CAPACITY_ROWS) * this.dimension);
|
|
288
|
+
new Uint8Array(data.buffer, 0, bytes.byteLength).set(littleEndian(bytes));
|
|
289
|
+
if (!allFinite(data, count * this.dimension)) return { reason: "vectors_corrupt" };
|
|
290
|
+
return { rows, data, generation: manifest.generation };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
has(hash) {
|
|
294
|
+
return this.byHash.has(hash);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// A read-only view into the store, valid as a snapshot until the same hash
|
|
298
|
+
// is written again.
|
|
299
|
+
get(hash) {
|
|
300
|
+
const row = this.byHash.get(hash);
|
|
301
|
+
if (row === undefined) return null;
|
|
302
|
+
return this.data.subarray(row * this.dimension, (row + 1) * this.dimension);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
all() {
|
|
306
|
+
return this.rows.map((row, index) => ({
|
|
307
|
+
id: row.id,
|
|
308
|
+
hash: row.hash,
|
|
309
|
+
vector: this.data.subarray(index * this.dimension, (index + 1) * this.dimension),
|
|
310
|
+
}));
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
ensureCapacity(additionalRows) {
|
|
314
|
+
const needed = (this.rows.length + additionalRows) * this.dimension;
|
|
315
|
+
if (needed <= this.data.length) return;
|
|
316
|
+
const capacity = Math.max(needed, this.data.length * 2, MIN_CAPACITY_ROWS * this.dimension);
|
|
317
|
+
const grown = new Float32Array(capacity);
|
|
318
|
+
grown.set(this.data.subarray(0, this.rows.length * this.dimension));
|
|
319
|
+
this.data = grown;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Validates every entry before touching memory, then upserts by hash and
|
|
323
|
+
// persists. Resolves to how many rows were added and how many replaced.
|
|
324
|
+
async putMany(entries) {
|
|
325
|
+
if (!Array.isArray(entries)) {
|
|
326
|
+
throw new VectorCacheError("putMany expects an array of entries.", "invalid_cache_entry");
|
|
327
|
+
}
|
|
328
|
+
const prepared = entries.map((entry, position) => {
|
|
329
|
+
if (
|
|
330
|
+
!entry ||
|
|
331
|
+
typeof entry.id !== "string" ||
|
|
332
|
+
!entry.id ||
|
|
333
|
+
typeof entry.hash !== "string" ||
|
|
334
|
+
!HASH_PATTERN.test(entry.hash)
|
|
335
|
+
) {
|
|
336
|
+
throw new VectorCacheError(
|
|
337
|
+
`Cache entry ${position} must carry an id and a sha256 hash.`,
|
|
338
|
+
"invalid_cache_entry",
|
|
339
|
+
{ position },
|
|
340
|
+
);
|
|
341
|
+
}
|
|
342
|
+
const converted = toVector(entry.vector, this.dimension);
|
|
343
|
+
if (!converted.vector) {
|
|
344
|
+
throw new VectorCacheError(
|
|
345
|
+
`Cache entry ${position} must carry a vector of ${this.dimension} finite float32 values.`,
|
|
346
|
+
"invalid_vector",
|
|
347
|
+
{ position, reason: converted.reason },
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
return { id: entry.id, hash: entry.hash, vector: converted.vector };
|
|
351
|
+
});
|
|
352
|
+
const result = { added: 0, replaced: 0 };
|
|
353
|
+
if (prepared.length === 0) return result;
|
|
354
|
+
this.ensureCapacity(prepared.length);
|
|
355
|
+
for (const entry of prepared) {
|
|
356
|
+
let row = this.byHash.get(entry.hash);
|
|
357
|
+
if (row === undefined) {
|
|
358
|
+
row = this.rows.length;
|
|
359
|
+
this.rows.push({ id: entry.id, hash: entry.hash });
|
|
360
|
+
this.byHash.set(entry.hash, row);
|
|
361
|
+
result.added += 1;
|
|
362
|
+
} else {
|
|
363
|
+
this.rows[row] = { id: entry.id, hash: entry.hash };
|
|
364
|
+
result.replaced += 1;
|
|
365
|
+
}
|
|
366
|
+
this.data.set(entry.vector, row * this.dimension);
|
|
367
|
+
}
|
|
368
|
+
await this.flush();
|
|
369
|
+
return result;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Drops every row whose hash is not in keepHashes; resolves to how many went.
|
|
373
|
+
async prune(keepHashes) {
|
|
374
|
+
const keep = keepHashes instanceof Set ? keepHashes : new Set(keepHashes || []);
|
|
375
|
+
const survivors = [];
|
|
376
|
+
for (let index = 0; index < this.rows.length; index += 1) {
|
|
377
|
+
if (keep.has(this.rows[index].hash)) survivors.push(index);
|
|
378
|
+
}
|
|
379
|
+
const removed = this.rows.length - survivors.length;
|
|
380
|
+
if (removed === 0) return 0;
|
|
381
|
+
const compacted = new Float32Array(
|
|
382
|
+
Math.max(survivors.length, MIN_CAPACITY_ROWS) * this.dimension,
|
|
383
|
+
);
|
|
384
|
+
const rows = [];
|
|
385
|
+
survivors.forEach((source, target) => {
|
|
386
|
+
compacted.set(
|
|
387
|
+
this.data.subarray(source * this.dimension, (source + 1) * this.dimension),
|
|
388
|
+
target * this.dimension,
|
|
389
|
+
);
|
|
390
|
+
rows.push(this.rows[source]);
|
|
391
|
+
});
|
|
392
|
+
this.rows = rows;
|
|
393
|
+
this.data = compacted;
|
|
394
|
+
this.byHash = new Map(rows.map((row, index) => [row.hash, index]));
|
|
395
|
+
await this.flush();
|
|
396
|
+
return removed;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// The directory must still be the one opened: real path unchanged, not a
|
|
400
|
+
// link, outside the root. Cheap enough to run before every operation group.
|
|
401
|
+
verifyLocation() {
|
|
402
|
+
assertNotSymlink(this.dir);
|
|
403
|
+
let actual;
|
|
404
|
+
try {
|
|
405
|
+
actual = fs.realpathSync(this.dir);
|
|
406
|
+
} catch {
|
|
407
|
+
throw movedError();
|
|
408
|
+
}
|
|
409
|
+
this.assertOpenedLocation(actual);
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// Before the directory exists: its existing ancestors must still resolve
|
|
413
|
+
// to where they did at open, which also keeps it outside the root.
|
|
414
|
+
verifyAncestors() {
|
|
415
|
+
assertNotSymlink(this.dir);
|
|
416
|
+
this.assertOpenedLocation(realpathOfExistingAncestors(this.dir));
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
assertOpenedLocation(actual) {
|
|
420
|
+
if (actual !== this.dir) throw movedError();
|
|
421
|
+
if (this.realRoot && !this.allowInsideRoot && isInside(this.realRoot, actual, path)) {
|
|
422
|
+
throw new VectorCacheError(
|
|
423
|
+
"Cache directory resolves inside the bundle root.",
|
|
424
|
+
"cache_inside_root",
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
locationIntact() {
|
|
430
|
+
try {
|
|
431
|
+
this.verifyLocation();
|
|
432
|
+
return true;
|
|
433
|
+
} catch {
|
|
434
|
+
return false;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// Recreates the directory when it was deleted underneath a running process,
|
|
439
|
+
// after checking that its ancestors still lead where they did at open.
|
|
440
|
+
ensureWritableDirectory() {
|
|
441
|
+
this.guardedMkdir();
|
|
442
|
+
this.verifyLocation();
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// Seams for tests that inject failures; production never replaces them.
|
|
446
|
+
makeDirectory(dir) {
|
|
447
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
writeFile(target, content) {
|
|
451
|
+
writeAtomic(target, content, { verify: () => this.verifyLocation() });
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
createFile(target, content) {
|
|
455
|
+
return createExclusive(target, content);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
removeFile(target) {
|
|
459
|
+
fs.unlinkSync(target);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
renameFile(from, to) {
|
|
463
|
+
fs.renameSync(from, to);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
linkFile(from, to) {
|
|
467
|
+
fs.linkSync(from, to);
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// Every mutation of the directory goes through these: each verifies the
|
|
471
|
+
// location first, so no call site can skip the check. writeFile verifies
|
|
472
|
+
// again inside writeAtomic, before the rename and before cleanup.
|
|
473
|
+
guardedMkdir() {
|
|
474
|
+
this.verifyAncestors();
|
|
475
|
+
this.makeDirectory(this.dir);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
guardedCreate(target, content) {
|
|
479
|
+
this.verifyLocation();
|
|
480
|
+
return this.createFile(target, content);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
guardedWrite(target, content) {
|
|
484
|
+
this.verifyLocation();
|
|
485
|
+
return this.writeFile(target, content);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
guardedRemove(target) {
|
|
489
|
+
this.verifyLocation();
|
|
490
|
+
this.removeFile(target);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
guardedRename(from, to) {
|
|
494
|
+
this.verifyLocation();
|
|
495
|
+
this.renameFile(from, to);
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
guardedLink(from, to) {
|
|
499
|
+
this.verifyLocation();
|
|
500
|
+
this.linkFile(from, to);
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
hasGenerationFiles(entry) {
|
|
504
|
+
return (
|
|
505
|
+
fs.existsSync(path.join(this.dir, `units.${entry.generation}.json`)) ||
|
|
506
|
+
fs.existsSync(path.join(this.dir, `vectors.${entry.generation}.f32`))
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// The committed manifest as it is on disk right now, read under the lock,
|
|
511
|
+
// so retirement is recorded for whatever generation this write supersedes.
|
|
512
|
+
committedState() {
|
|
513
|
+
const parsed = parseManifest(
|
|
514
|
+
readRegularFile(path.join(this.dir, MANIFEST_FILE)),
|
|
515
|
+
this.embedderId,
|
|
516
|
+
this.dimension,
|
|
517
|
+
);
|
|
518
|
+
if (parsed.reason) return { generation: null, retired: [] };
|
|
519
|
+
return { generation: parsed.manifest.generation, retired: parsed.manifest.retired };
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// Writes of one instance run one after another.
|
|
523
|
+
flush() {
|
|
524
|
+
const run = this.writeQueue.then(() => this.flushNow());
|
|
525
|
+
this.writeQueue = run.catch(() => {});
|
|
526
|
+
return run;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// Writes a new generation of the data files under the lock, replaces the
|
|
530
|
+
// manifest atomically as the commit point, then collects garbage.
|
|
531
|
+
async flushNow() {
|
|
532
|
+
this.ensureWritableDirectory();
|
|
533
|
+
const generation = newGeneration();
|
|
534
|
+
const token = newLockToken();
|
|
535
|
+
const io = {
|
|
536
|
+
create: (target, content) => this.guardedCreate(target, content),
|
|
537
|
+
rename: (from, to) => this.guardedRename(from, to),
|
|
538
|
+
remove: (target) => this.guardedRemove(target),
|
|
539
|
+
link: (from, to) => this.guardedLink(from, to),
|
|
540
|
+
};
|
|
541
|
+
const lockPath = await acquireLock(this.dir, {
|
|
542
|
+
token,
|
|
543
|
+
generation,
|
|
544
|
+
waitMs: this.lockWaitMs,
|
|
545
|
+
staleMs: this.lockStaleMs,
|
|
546
|
+
io,
|
|
547
|
+
});
|
|
548
|
+
try {
|
|
549
|
+
this.verifyLocation();
|
|
550
|
+
this.ownGenerations.add(generation);
|
|
551
|
+
const previous = this.committedState();
|
|
552
|
+
const now = new Date().toISOString();
|
|
553
|
+
const cutoff = Date.now() - GC_GRACE_MS;
|
|
554
|
+
const kept = previous.retired.filter(
|
|
555
|
+
(entry) => entry.generation !== generation && this.hasGenerationFiles(entry),
|
|
556
|
+
);
|
|
557
|
+
// Entries younger than the grace period are never forgotten; the cap
|
|
558
|
+
// applies to entries past it, and a forgotten entry's files are
|
|
559
|
+
// collected in this same step.
|
|
560
|
+
const past = kept.filter((entry) => Date.parse(entry.retiredAt) < cutoff);
|
|
561
|
+
const forgotten = past.slice(0, Math.max(0, past.length - MAX_RETIRED));
|
|
562
|
+
const retired = past
|
|
563
|
+
.slice(forgotten.length)
|
|
564
|
+
.concat(kept.filter((entry) => Date.parse(entry.retiredAt) >= cutoff))
|
|
565
|
+
.concat(
|
|
566
|
+
// An own generation is collected right after this commit, so only
|
|
567
|
+
// a foreign one is worth recording.
|
|
568
|
+
previous.generation &&
|
|
569
|
+
previous.generation !== generation &&
|
|
570
|
+
!this.ownGenerations.has(previous.generation)
|
|
571
|
+
? [{ generation: previous.generation, retiredAt: now }]
|
|
572
|
+
: [],
|
|
573
|
+
);
|
|
574
|
+
const count = this.rows.length;
|
|
575
|
+
const files = { units: `units.${generation}.json`, vectors: `vectors.${generation}.f32` };
|
|
576
|
+
const unitBytes = Buffer.from(`${JSON.stringify(this.rows)}\n`, "utf8");
|
|
577
|
+
const vectorBytes = littleEndian(
|
|
578
|
+
Buffer.from(this.data.buffer, this.data.byteOffset, count * this.dimension * 4),
|
|
579
|
+
);
|
|
580
|
+
await this.guardedWrite(path.join(this.dir, files.units), unitBytes);
|
|
581
|
+
await this.guardedWrite(path.join(this.dir, files.vectors), vectorBytes);
|
|
582
|
+
const manifest = {
|
|
583
|
+
format: VECTOR_CACHE_FORMAT,
|
|
584
|
+
formatVersion: VECTOR_CACHE_FORMAT_VERSION,
|
|
585
|
+
embedderId: this.embedderId,
|
|
586
|
+
dimension: this.dimension,
|
|
587
|
+
count,
|
|
588
|
+
generation,
|
|
589
|
+
files,
|
|
590
|
+
digests: { units: sha256Hex(unitBytes), vectors: sha256Hex(vectorBytes) },
|
|
591
|
+
retired,
|
|
592
|
+
updatedAt: now,
|
|
593
|
+
};
|
|
594
|
+
await this.guardedWrite(
|
|
595
|
+
path.join(this.dir, MANIFEST_FILE),
|
|
596
|
+
Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, "utf8"),
|
|
597
|
+
);
|
|
598
|
+
this.generation = generation;
|
|
599
|
+
this.collectGarbage(retired.concat(forgotten));
|
|
600
|
+
} finally {
|
|
601
|
+
releaseLock(lockPath, token, io);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Generations this process wrote earlier go at once: nothing else can
|
|
606
|
+
// reference them, and readers hold their rows in memory. A retired foreign
|
|
607
|
+
// generation goes once its retirement is older than the grace period; a
|
|
608
|
+
// never-committed orphan, a stale temporary file or a lock parked by a
|
|
609
|
+
// reclaim or a release once its files are. Best effort: a failure here
|
|
610
|
+
// never fails the write that just completed, and a directory that no
|
|
611
|
+
// longer verifies stops the collection before anything is unlinked.
|
|
612
|
+
collectGarbage(retired) {
|
|
613
|
+
let entries;
|
|
614
|
+
try {
|
|
615
|
+
this.verifyLocation();
|
|
616
|
+
entries = fs.readdirSync(this.dir, { withFileTypes: true });
|
|
617
|
+
} catch {
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
620
|
+
const locked = readLock(path.join(this.dir, LOCK_FILE))?.generation || null;
|
|
621
|
+
const cutoff = Date.now() - GC_GRACE_MS;
|
|
622
|
+
const retiredAt = new Map(
|
|
623
|
+
retired.map((entry) => [entry.generation, Date.parse(entry.retiredAt)]),
|
|
624
|
+
);
|
|
625
|
+
for (const entry of entries) {
|
|
626
|
+
if (entry.isDirectory()) continue;
|
|
627
|
+
const target = path.join(this.dir, entry.name);
|
|
628
|
+
const match = GENERATION_FILE.exec(entry.name);
|
|
629
|
+
let remove = false;
|
|
630
|
+
if (match) {
|
|
631
|
+
const generation = match[2];
|
|
632
|
+
if (generation === this.generation || generation === locked) continue;
|
|
633
|
+
if (this.ownGenerations.has(generation)) remove = true;
|
|
634
|
+
else if (retiredAt.has(generation)) remove = retiredAt.get(generation) < cutoff;
|
|
635
|
+
else remove = mtimeBefore(target, cutoff);
|
|
636
|
+
} else if (
|
|
637
|
+
entry.name.endsWith(".tmp") ||
|
|
638
|
+
LOCK_PARKED_PREFIXES.some((prefix) => entry.name.startsWith(prefix))
|
|
639
|
+
) {
|
|
640
|
+
remove = mtimeBefore(target, cutoff);
|
|
641
|
+
}
|
|
642
|
+
if (!remove) continue;
|
|
643
|
+
try {
|
|
644
|
+
this.guardedRemove(target);
|
|
645
|
+
} catch (error) {
|
|
646
|
+
if (error instanceof VectorCacheError) return;
|
|
647
|
+
// Another process may have removed it first; the next write retries.
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
// options: embedderId, dimension (required); root (containment is checked at
|
|
654
|
+
// open and before every write), allowInsideRoot, lockWaitMs, lockStaleMs.
|
|
655
|
+
function openVectorCache(dir, options) {
|
|
656
|
+
const config = options || {};
|
|
657
|
+
const resolvedDir = realCacheDir(assertSafeCachePath(dir, path));
|
|
658
|
+
const { embedderId, dimension } = config;
|
|
659
|
+
if (
|
|
660
|
+
typeof embedderId !== "string" ||
|
|
661
|
+
!embedderId.trim() ||
|
|
662
|
+
embedderId.length > 256 ||
|
|
663
|
+
/[\s\p{Cc}]/u.test(embedderId)
|
|
664
|
+
) {
|
|
665
|
+
throw new VectorCacheError(
|
|
666
|
+
"Vector cache requires the embedder id as a single token.",
|
|
667
|
+
"invalid_cache_options",
|
|
668
|
+
{ field: "embedderId" },
|
|
669
|
+
);
|
|
670
|
+
}
|
|
671
|
+
if (!Number.isSafeInteger(dimension) || dimension <= 0 || dimension > MAX_DIMENSION) {
|
|
672
|
+
throw new VectorCacheError(
|
|
673
|
+
"Vector cache requires a positive integer dimension.",
|
|
674
|
+
"invalid_cache_options",
|
|
675
|
+
{ field: "dimension" },
|
|
676
|
+
);
|
|
677
|
+
}
|
|
678
|
+
const realRoot =
|
|
679
|
+
typeof config.root === "string" && config.root.trim() ? realpathOrResolve(config.root) : null;
|
|
680
|
+
if (realRoot && config.allowInsideRoot !== true && isInside(realRoot, resolvedDir, path)) {
|
|
681
|
+
throw new VectorCacheError(
|
|
682
|
+
"Cache directory resolves inside the bundle root.",
|
|
683
|
+
"cache_inside_root",
|
|
684
|
+
);
|
|
685
|
+
}
|
|
686
|
+
const cache = new VectorCache(resolvedDir, embedderId, dimension, {
|
|
687
|
+
realRoot,
|
|
688
|
+
allowInsideRoot: config.allowInsideRoot,
|
|
689
|
+
lockWaitMs: config.lockWaitMs,
|
|
690
|
+
lockStaleMs: config.lockStaleMs,
|
|
691
|
+
});
|
|
692
|
+
cache.load();
|
|
693
|
+
return cache;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
module.exports = {
|
|
697
|
+
VECTOR_CACHE_FORMAT,
|
|
698
|
+
VECTOR_CACHE_FORMAT_VERSION,
|
|
699
|
+
VectorCache,
|
|
700
|
+
VectorCacheError,
|
|
701
|
+
openVectorCache,
|
|
702
|
+
resolveCacheDir,
|
|
703
|
+
};
|