@mfdaves/okf-mcp 0.9.0 → 0.12.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +176 -19
  3. package/bin/okf-mcp.js +6 -1
  4. package/okf/okf-mcp/policies/authoring-safety.md +11 -1
  5. package/package.json +17 -5
  6. package/server.json +2 -2
  7. package/src/assets.js +165 -73
  8. package/src/authoring/candidates.js +391 -0
  9. package/src/authoring/git-preflight.js +225 -0
  10. package/src/authoring/git-publish.js +776 -0
  11. package/src/authoring/live.js +789 -0
  12. package/src/authoring/paths.js +112 -0
  13. package/src/authoring/policy-hook.js +221 -0
  14. package/src/authoring/proposal-policy.js +137 -0
  15. package/src/authoring/receipts.js +329 -0
  16. package/src/authoring/render.js +38 -0
  17. package/src/authoring/shared.js +75 -0
  18. package/src/authoring/staging.js +216 -0
  19. package/src/authoring.js +386 -287
  20. package/src/cache/files.js +147 -0
  21. package/src/cache/lock.js +201 -0
  22. package/src/cache/paths.js +197 -0
  23. package/src/cache/store.js +703 -0
  24. package/src/catalog-text.js +19 -0
  25. package/src/cli.js +184 -151
  26. package/src/computation.js +99 -61
  27. package/src/context/assemble.js +567 -0
  28. package/src/context/cli-options.js +88 -0
  29. package/src/context/score.js +234 -0
  30. package/src/context/sections.js +274 -0
  31. package/src/embed/interface.js +160 -0
  32. package/src/embed/openai.js +715 -0
  33. package/src/embed/prepare.js +182 -0
  34. package/src/embed/units.js +96 -0
  35. package/src/git-source.js +112 -52
  36. package/src/graph.js +95 -37
  37. package/src/http-server.js +36 -17
  38. package/src/index.js +15 -0
  39. package/src/indexer/assemble.js +566 -0
  40. package/src/indexer/build.js +290 -0
  41. package/src/indexer/cache.js +81 -0
  42. package/src/indexer/edges.js +430 -0
  43. package/src/indexer/index.js +44 -0
  44. package/src/indexer/paths.js +178 -0
  45. package/src/indexer/resolve.js +282 -0
  46. package/src/indexer/update.js +720 -0
  47. package/src/indexer.js +3 -1068
  48. package/src/live-authoring.js +4 -2329
  49. package/src/markdown.js +21 -16
  50. package/src/mcp/registry.js +328 -0
  51. package/src/mcp/resources.js +45 -0
  52. package/src/mcp/server.js +472 -0
  53. package/src/mcp/tools/catalog.js +244 -0
  54. package/src/mcp/tools/computation.js +83 -0
  55. package/src/mcp/tools/context.js +133 -0
  56. package/src/mcp/tools/graph.js +167 -0
  57. package/src/mcp/tools/live-authoring.js +316 -0
  58. package/src/mcp/tools/migration.js +86 -0
  59. package/src/mcp/tools/producers.js +79 -0
  60. package/src/mcp/tools/proposals.js +213 -0
  61. package/src/mcp/tools/provenance.js +87 -0
  62. package/src/mcp/tools/remote.js +79 -0
  63. package/src/mcp/tools/search.js +242 -0
  64. package/src/mcp-server.js +3 -1611
  65. package/src/migration.js +431 -253
  66. package/src/parser.js +245 -130
  67. package/src/plugins.js +71 -48
  68. package/src/policy/write-policy.js +713 -0
  69. package/src/producer-publisher.js +195 -77
  70. package/src/producers.js +265 -101
  71. package/src/project.js +277 -93
  72. package/src/remote.js +96 -61
  73. package/src/search/cli-options.js +107 -0
  74. package/src/search/confidence.js +61 -0
  75. package/src/search/dense.js +304 -0
  76. package/src/search/embedding-cli.js +57 -0
  77. package/src/search/embedding-config.js +217 -0
  78. package/src/search/fusion.js +249 -0
  79. package/src/search/hybrid.js +526 -0
  80. package/src/search/languages.js +200 -0
  81. package/src/search/lexical.js +695 -0
  82. package/src/search/query.js +620 -0
  83. package/src/search/stem/among.js +43 -0
  84. package/src/search/stem/base-stemmer.js +311 -0
  85. package/src/search/stem/en-regions.js +456 -0
  86. package/src/search/stem/en-suffixes.js +664 -0
  87. package/src/search/stem/en-tables.js +176 -0
  88. package/src/search/stem/en.js +238 -0
  89. package/src/search/stem/it-regions.js +459 -0
  90. package/src/search/stem/it-suffixes.js +471 -0
  91. package/src/search/stem/it-tables.js +257 -0
  92. package/src/search/stem/it.js +155 -0
  93. package/src/search-index.js +2 -81
  94. package/src/search.js +2 -252
  95. package/src/store.js +134 -81
  96. package/src/v02.js +301 -105
  97. package/src/validation.js +24 -20
@@ -0,0 +1,526 @@
1
+ "use strict";
2
+
3
+ // The embedding runtime behind hybrid search (docs/plan-1.0.md R-8 to R-10,
4
+ // O-2, O-3): one embedder, one vector cache, the fusion weight and the unit
5
+ // function, plus the lifecycle around them. Opening never throws: a missing
6
+ // key, an unreachable endpoint or an unusable cache directory leaves a
7
+ // degraded runtime whose searches run lexical-only and say why with a fixed
8
+ // code (B-6: no server-supplied text is ever carried). Queries are encoded
9
+ // through a small LRU; documents are embedded in the background (startup and
10
+ // after rebuilds) or right after a write batch (only the changed units).
11
+
12
+ const { performance } = require("node:perf_hooks");
13
+
14
+ const { openVectorCache, resolveCacheDir } = require("../cache/store");
15
+ const { EmbedderError, describeEmbedder, validateEmbedder } = require("../embed/interface");
16
+ const { OPENAI_SERVER_CODES, createOpenAiEmbedder } = require("../embed/openai");
17
+ const { prepareEmbeddings } = require("../embed/prepare");
18
+ const { DEFAULT_EMBED_UNITS, denseCoverage, resolveUnits, unitMemo } = require("./dense");
19
+ const {
20
+ ADAPTIVE_ALPHA,
21
+ adaptiveWeights,
22
+ fusionAlphaSetting,
23
+ fusionNormalization,
24
+ } = require("./fusion");
25
+ const { searchConcepts } = require("./query");
26
+ const { minSimilarityOption } = require("./confidence");
27
+
28
+ const QUERY_CACHE_SIZE = 256;
29
+ // A search must not hang on a slow endpoint: query encoding gets its own
30
+ // short deadline and one retry; after a failure, searches stay lexical-only
31
+ // for a cooldown instead of paying the deadline again on every call.
32
+ const QUERY_TIMEOUT_MS = 10000;
33
+ const QUERY_MAX_RETRIES = 1;
34
+ const PROBE_TIMEOUT_MS = 15000;
35
+ const FAILURE_COOLDOWN_MS = 30000;
36
+ const SYNC_RETRY_MS = 60000;
37
+ // Receipts list this many skipped unit ids; the count is always complete.
38
+ const SKIPPED_IDS_SHOWN = 5;
39
+ const CODE_PATTERN = /^[a-z][a-z0-9_]{0,63}$/;
40
+ // Options a caller of search() may not smuggle in: the runtime supplies them.
41
+ const RUNTIME_OPTIONS = ["queryVector", "vectorCache", "embedUnits"];
42
+
43
+ // Only our own constant codes and the enumerated server code survive.
44
+ function failureOf(error, fallback) {
45
+ const code = typeof error?.code === "string" && CODE_PATTERN.test(error.code) ? error.code : null;
46
+ const serverCode = error?.details?.serverCode;
47
+ return {
48
+ code: code || fallback || "embedder_failed",
49
+ ...(OPENAI_SERVER_CODES.includes(serverCode) ? { serverCode } : {}),
50
+ };
51
+ }
52
+
53
+ function round(value) {
54
+ return Math.round(value * 1000) / 1000;
55
+ }
56
+
57
+ // R-9: remote bundles stay lexical-only in 1.0, so their text is never sent
58
+ // to an embedder. One wrapper per runtime keeps the unit memo stable.
59
+ function localUnits(unitsOf) {
60
+ return (index) => {
61
+ const units = unitsOf(index);
62
+ const remote = new Set(
63
+ (index.bundles || []).filter((bundle) => bundle.remote).map((bundle) => bundle.id),
64
+ );
65
+ if (!remote.size) return units;
66
+ return units.filter((unit) => {
67
+ const anchor = unit.id.indexOf("#");
68
+ const doc =
69
+ index.byUri?.get(unit.id) ||
70
+ (anchor === -1 ? null : index.byUri?.get(unit.id.slice(0, anchor)));
71
+ return !doc || !remote.has(doc.bundle);
72
+ });
73
+ };
74
+ }
75
+
76
+ function withTimeout(promise, ms) {
77
+ let timer = null;
78
+ const timeout = new Promise((_resolve, reject) => {
79
+ timer = setTimeout(
80
+ () => reject(new EmbedderError("Embedder probe timed out.", "embedder_timeout")),
81
+ ms,
82
+ );
83
+ timer.unref?.();
84
+ });
85
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
86
+ }
87
+
88
+ // A query vector must match the cache before it is used or kept: same
89
+ // length, every value finite (a client that discovered another dimension,
90
+ // or a broken server, must not poison the query LRU).
91
+ function checkQueryVector(vector, dimension) {
92
+ const length = vector && typeof vector.length === "number" ? vector.length : null;
93
+ let finite = length !== null;
94
+ for (let index = 0; finite && index < length; index += 1) {
95
+ finite = Number.isFinite(vector[index]);
96
+ }
97
+ if (length !== dimension || !finite) {
98
+ throw new EmbedderError(
99
+ "The query vector does not match the vector cache.",
100
+ "embedder_invalid_vector",
101
+ { expected: dimension, received: length },
102
+ );
103
+ }
104
+ return vector;
105
+ }
106
+
107
+ // { count, ids }: how many units the provider rejected (prepareEmbeddings
108
+ // `skipped`) and the first few of their ids, in id order.
109
+ function skippedSummary(skipped) {
110
+ const list = Array.isArray(skipped) ? skipped : [];
111
+ return { count: list.length, ids: list.slice(0, SKIPPED_IDS_SHOWN).map((entry) => entry.id) };
112
+ }
113
+
114
+ function usageOf(embedder) {
115
+ const usage = typeof embedder?.usage === "function" ? embedder.usage() : null;
116
+ return {
117
+ promptTokens: Number.isFinite(usage?.promptTokens) ? usage.promptTokens : 0,
118
+ requests: Number.isFinite(usage?.requests) ? usage.requests : 0,
119
+ };
120
+ }
121
+
122
+ function usageDelta(before, after) {
123
+ return {
124
+ promptTokens: Math.max(0, after.promptTokens - before.promptTokens),
125
+ requests: Math.max(0, after.requests - before.requests),
126
+ };
127
+ }
128
+
129
+ // Embedders for the three roles. A configuration object builds three OpenAI
130
+ // clients (separate usage counters; queries get the short deadline); an
131
+ // embedder instance passed by a library caller serves all three.
132
+ function embeddersFor(spec, context) {
133
+ if (spec && typeof spec.embed === "function") {
134
+ return { documents: spec, writes: spec, queries: spec };
135
+ }
136
+ const options = Object.assign({}, spec || {});
137
+ delete options.type;
138
+ if (context.env) options.env = context.env;
139
+ if (context.fetch) options.fetch = context.fetch;
140
+ const timeoutMs = options.timeoutMs === undefined ? 60000 : options.timeoutMs;
141
+ const maxRetries = options.maxRetries === undefined ? 5 : options.maxRetries;
142
+ return {
143
+ documents: createOpenAiEmbedder(options),
144
+ writes: createOpenAiEmbedder(options),
145
+ queries: createOpenAiEmbedder(
146
+ Object.assign({}, options, {
147
+ timeoutMs: Math.min(timeoutMs, QUERY_TIMEOUT_MS),
148
+ maxRetries: Math.min(maxRetries, QUERY_MAX_RETRIES),
149
+ }),
150
+ ),
151
+ };
152
+ }
153
+
154
+ class EmbeddingRuntime {
155
+ constructor(settings, context) {
156
+ const config = settings || {};
157
+ this.context = context || {};
158
+ this.alpha = fusionAlphaSetting(config.fusion?.alpha);
159
+ this.adaptive = adaptiveWeights(config.fusion?.adaptive);
160
+ this.minSimilarity = minSimilarityOption(config.confidence?.minSimilarity);
161
+ this.normalization = fusionNormalization(config.fusion?.normalization);
162
+ this.embedUnits =
163
+ typeof config.embedUnits === "string" ? config.embedUnits : DEFAULT_EMBED_UNITS;
164
+ this.units = null;
165
+ this.embedders = null;
166
+ this.cache = null;
167
+ // openFailure is permanent for the process; queryFailure cools down.
168
+ this.openFailure = null;
169
+ this.queryFailure = null;
170
+ this.queryFailedAt = 0;
171
+ this.queryVectors = new Map();
172
+ this.syncedIndex = null;
173
+ this.syncing = null;
174
+ this.syncingIndex = null;
175
+ this.latestIndex = null;
176
+ this.syncQueue = Promise.resolve();
177
+ this.lastSync = null;
178
+ this.syncError = null;
179
+ this.syncFailedAt = 0;
180
+ this.startup = null;
181
+ this.now = this.context.now || Date.now;
182
+ }
183
+
184
+ // Builds the embedders, discovers the dimension when it is not configured,
185
+ // validates the implementation and opens the cache outside the root.
186
+ async open(settings) {
187
+ const config = settings || {};
188
+ try {
189
+ this.units = localUnits(resolveUnits(config.embedUnits));
190
+ this.embedders = embeddersFor(config.embedder, this.context);
191
+ const documents = this.embedders.documents;
192
+ if (!Number.isSafeInteger(documents.dimension)) {
193
+ // The interface's order: probe() discovers the dimension before any
194
+ // embed(). It takes no signal, so the wait is bounded here.
195
+ if (typeof documents.probe === "function") {
196
+ await withTimeout(documents.probe(), PROBE_TIMEOUT_MS);
197
+ } else {
198
+ await documents.embed(["okf-mcp"], { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
199
+ }
200
+ }
201
+ validateEmbedder(documents);
202
+ const dir = resolveCacheDir({
203
+ root: this.context.root,
204
+ cacheDir: config.cacheDir,
205
+ env: this.context.env,
206
+ });
207
+ this.cache = openVectorCache(dir, {
208
+ embedderId: documents.id,
209
+ dimension: documents.dimension,
210
+ root: this.context.root,
211
+ });
212
+ } catch (error) {
213
+ this.openFailure = failureOf(error, "embedder_unavailable");
214
+ }
215
+ return this;
216
+ }
217
+
218
+ get ready() {
219
+ return Boolean(this.cache && this.embedders && !this.openFailure);
220
+ }
221
+
222
+ describe(failure) {
223
+ const documents = this.embedders?.documents;
224
+ const base = documents
225
+ ? describeEmbedder(documents)
226
+ : { id: null, remote: null, dimension: null };
227
+ return Object.assign(base, { degraded: Boolean(failure) }, failure || {});
228
+ }
229
+
230
+ cacheReceipt() {
231
+ const receipt = { rows: this.cache ? this.cache.size : 0 };
232
+ // Local paths are an operator's concern: shown only to a writing server.
233
+ if (this.context.write && this.cache) receipt.dir = this.cache.dir;
234
+ return receipt;
235
+ }
236
+
237
+ usage() {
238
+ const total = { promptTokens: 0, requests: 0 };
239
+ const seen = new Set();
240
+ Object.values(this.embedders || {}).forEach((embedder) => {
241
+ if (seen.has(embedder)) return;
242
+ seen.add(embedder);
243
+ const usage = usageOf(embedder);
244
+ total.promptTokens += usage.promptTokens;
245
+ total.requests += usage.requests;
246
+ });
247
+ return total;
248
+ }
249
+
250
+ // The failure searches currently report, if any: a failed open, or a
251
+ // query-encoding failure still inside its cooldown.
252
+ currentFailure() {
253
+ if (!this.ready) return this.openFailure || { code: "embedder_unavailable" };
254
+ if (this.queryFailure && this.now() - this.queryFailedAt < FAILURE_COOLDOWN_MS) {
255
+ return this.queryFailure;
256
+ }
257
+ return null;
258
+ }
259
+
260
+ async encodeQuery(text) {
261
+ const key = String(text);
262
+ const cached = this.queryVectors.get(key);
263
+ if (cached) {
264
+ this.queryVectors.delete(key);
265
+ this.queryVectors.set(key, cached);
266
+ return { vector: cached, cached: true };
267
+ }
268
+ const [raw] = await this.embedders.queries.embed([key], {
269
+ signal: AbortSignal.timeout(QUERY_TIMEOUT_MS),
270
+ concurrency: 1,
271
+ });
272
+ const vector = checkQueryVector(raw, this.cache.dimension);
273
+ this.queryVectors.set(key, vector);
274
+ if (this.queryVectors.size > QUERY_CACHE_SIZE) {
275
+ this.queryVectors.delete(this.queryVectors.keys().next().value);
276
+ }
277
+ return { vector, cached: false };
278
+ }
279
+
280
+ // searchConcepts with the dense side when it can run, lexical otherwise;
281
+ // either way the response says which ran and why (O-2).
282
+ async search(index, options) {
283
+ const config = Object.assign({}, options || {});
284
+ RUNTIME_OPTIONS.forEach((key) => {
285
+ delete config[key];
286
+ });
287
+ const started = performance.now();
288
+ const query = String(config.query || "").trim();
289
+ let failure = null;
290
+ let encoded = null;
291
+ let encodingMs = 0;
292
+ const usageBefore = this.embedders ? usageOf(this.embedders.queries) : null;
293
+ const wanted = config.hybrid !== false && query !== "";
294
+ if (wanted) {
295
+ const current = this.currentFailure();
296
+ if (current) {
297
+ failure = this.ready ? Object.assign({}, current, { cooldown: true }) : current;
298
+ } else if (!this.cache.size || !denseCoverage(index, this.cache, this.units).present) {
299
+ failure = { code: "cache_empty" };
300
+ this.scheduleSync(index);
301
+ } else {
302
+ this.scheduleSync(index);
303
+ const encodeStarted = performance.now();
304
+ try {
305
+ encoded = await this.encodeQuery(query);
306
+ this.queryFailure = null;
307
+ } catch (error) {
308
+ failure = failureOf(error);
309
+ this.queryFailure = failure;
310
+ this.queryFailedAt = this.now();
311
+ }
312
+ encodingMs = performance.now() - encodeStarted;
313
+ }
314
+ }
315
+ const searchStarted = performance.now();
316
+ const lexical = () => searchConcepts(index, Object.assign({}, config, { hybrid: false }));
317
+ let response;
318
+ if (encoded) {
319
+ const hybridConfig = Object.assign({}, config, {
320
+ hybrid: true,
321
+ queryVector: encoded.vector,
322
+ vectorCache: this.cache,
323
+ embedUnits: this.units,
324
+ alpha: config.alpha === undefined ? this.alpha : config.alpha,
325
+ adaptiveAlpha: config.adaptiveAlpha || this.adaptive,
326
+ minSimilarity: config.minSimilarity ?? this.minSimilarity,
327
+ fusionNormalization: config.fusionNormalization || this.normalization,
328
+ });
329
+ try {
330
+ response = searchConcepts(index, hybridConfig);
331
+ } catch (error) {
332
+ // Any failure of the dense stage degrades to lexical; a problem with
333
+ // the caller's own options throws again from the lexical run.
334
+ failure = failureOf(error, "dense_failed");
335
+ response = lexical();
336
+ }
337
+ } else {
338
+ response = lexical();
339
+ }
340
+ const searchMs = performance.now() - searchStarted;
341
+ const results = response.results;
342
+ delete response.results;
343
+ return Object.assign(response, {
344
+ hybrid: response.hybrid === true,
345
+ embedder: this.describe(failure),
346
+ cache: this.cacheReceipt(),
347
+ usage: usageBefore ? usageDelta(usageBefore, usageOf(this.embedders.queries)) : null,
348
+ timings: {
349
+ queryEncodingMs: round(encodingMs),
350
+ searchMs: round(searchMs),
351
+ totalMs: round(performance.now() - started),
352
+ },
353
+ results,
354
+ });
355
+ }
356
+
357
+ // Embeds every unit of `index` the cache lacks and prunes rows no unit
358
+ // uses (prepareEmbeddings). Runs one at a time; the receipt is kept.
359
+ // Startup and background preparation and the incremental embedding of a
360
+ // write batch share one queue, so a slow preparation never prunes rows a
361
+ // later batch added. A preparation always works on the newest index the
362
+ // runtime has seen when it starts: its prune keeps exactly that unit set.
363
+ enqueue(task) {
364
+ const run = this.syncQueue.then(task);
365
+ this.syncQueue = run.catch(() => null);
366
+ return run;
367
+ }
368
+
369
+ noteIndex(index) {
370
+ if (index) this.latestIndex = index;
371
+ }
372
+
373
+ sync(index) {
374
+ this.noteIndex(index);
375
+ return this.enqueue(async () => {
376
+ if (!this.ready) return null;
377
+ const target = this.latestIndex || index;
378
+ try {
379
+ const receipt = await prepareEmbeddings(target, this.embedders.documents, this.cache, {
380
+ units: (current) => unitMemo(current, this.units).units,
381
+ });
382
+ this.syncedIndex = target;
383
+ this.syncError = null;
384
+ this.syncFailedAt = 0;
385
+ // WP3c-2: which unit set was prepared; after a switch of embedUnits the first
386
+ // preparation embeds the new units once and prunes the old ones (embedded, pruned).
387
+ this.lastSync = Object.assign(this.sanitize(receipt), { embedUnits: this.embedUnits });
388
+ if (!this.startup) this.startup = this.lastSync;
389
+ return this.lastSync;
390
+ } catch (error) {
391
+ this.syncError = failureOf(error, "embedding_sync_failed");
392
+ this.syncFailedAt = this.now();
393
+ if (!this.startup) this.startup = { failed: true, error: this.syncError };
394
+ return null;
395
+ }
396
+ });
397
+ }
398
+
399
+ // Background sync for an index this runtime has not prepared yet, at most
400
+ // once per index object and never while a failed sync is cooling down.
401
+ scheduleSync(index) {
402
+ if (!this.ready || index === this.syncedIndex || index === this.syncingIndex) return null;
403
+ if (this.syncFailedAt > 0 && this.now() - this.syncFailedAt < SYNC_RETRY_MS) return null;
404
+ this.syncingIndex = index;
405
+ this.syncing = this.sync(index).finally(() => {
406
+ if (this.syncingIndex === index) this.syncingIndex = null;
407
+ });
408
+ return this.syncing;
409
+ }
410
+
411
+ // After a write batch: embed only the units whose text is new relative to
412
+ // the index before the batch and absent from the cache; one request per
413
+ // batch when the inputs fit. Superseded rows are pruned by the next sync.
414
+ embedChanged(previousIndex, index) {
415
+ this.noteIndex(index);
416
+ return this.enqueue(() => this.embedChangedNow(previousIndex, index));
417
+ }
418
+
419
+ async embedChangedNow(previousIndex, index) {
420
+ if (!this.ready) {
421
+ return {
422
+ embedder: this.describe(this.currentFailure()),
423
+ cache: this.cacheReceipt(),
424
+ skipped: true,
425
+ };
426
+ }
427
+ const started = performance.now();
428
+ const writes = this.embedders.writes;
429
+ const before = usageOf(writes);
430
+ const previous = previousIndex ? unitMemo(previousIndex, this.units).hashes : new Set();
431
+ const pending = new Map();
432
+ unitMemo(index, this.units).units.forEach((unit) => {
433
+ if (!previous.has(unit.hash) && !this.cache.has(unit.hash) && !pending.has(unit.hash)) {
434
+ pending.set(unit.hash, unit);
435
+ }
436
+ });
437
+ const queue = Array.from(pending.values());
438
+ let embedded = 0;
439
+ let failure = null;
440
+ const skipped = [];
441
+ try {
442
+ for (let offset = 0; offset < queue.length; offset += writes.maxBatch) {
443
+ const batch = queue.slice(offset, offset + writes.maxBatch);
444
+ const vectors = await writes.embed(
445
+ batch.map((unit) => unit.text),
446
+ { concurrency: 1, skipRejected: true },
447
+ );
448
+ const rows = [];
449
+ batch.forEach((unit, position) => {
450
+ if (vectors[position] === null) {
451
+ skipped.push({ id: unit.id, reason: "input_rejected" });
452
+ } else {
453
+ rows.push({ id: unit.id, hash: unit.hash, vector: vectors[position] });
454
+ }
455
+ });
456
+ if (rows.length) await this.cache.putMany(rows);
457
+ embedded += rows.length;
458
+ }
459
+ } catch (error) {
460
+ failure = failureOf(error, "embedding_write_failed");
461
+ }
462
+ return {
463
+ units: queue.length,
464
+ embedded,
465
+ skipped: skippedSummary(skipped),
466
+ embedder: this.describe(failure),
467
+ cache: this.cacheReceipt(),
468
+ usage: usageDelta(before, usageOf(writes)),
469
+ durationMs: round(performance.now() - started),
470
+ };
471
+ }
472
+
473
+ sanitize(receipt) {
474
+ const copy = Object.assign({}, receipt, {
475
+ cache: Object.assign({}, receipt.cache),
476
+ skipped: skippedSummary(receipt.skipped),
477
+ });
478
+ if (!this.context.write) delete copy.cache.dir;
479
+ return copy;
480
+ }
481
+
482
+ status() {
483
+ const failure = this.currentFailure();
484
+ return {
485
+ configured: true,
486
+ available: this.ready && Boolean(this.cache?.size),
487
+ alpha: this.alpha,
488
+ ...(this.alpha === ADAPTIVE_ALPHA ? { adaptive: this.adaptive } : {}),
489
+ normalization: this.normalization,
490
+ embedUnits: this.embedUnits,
491
+ minSimilarity: this.minSimilarity,
492
+ embedder: this.describe(failure),
493
+ cache: this.cacheReceipt(),
494
+ // Units the provider rejected in the last preparation (count, first ids).
495
+ skipped: this.lastSync?.skipped || skippedSummary([]),
496
+ startup: this.startup,
497
+ sync: {
498
+ state: this.syncingIndex ? "running" : this.syncError ? "failed" : "idle",
499
+ ...(this.syncError ? { error: this.syncError } : {}),
500
+ last: this.lastSync,
501
+ },
502
+ usage: this.usage(),
503
+ };
504
+ }
505
+
506
+ // Resolves once no background sync is pending (tests, orderly shutdown).
507
+ async idle() {
508
+ await this.syncQueue;
509
+ }
510
+ }
511
+
512
+ // settings: { embedder, cacheDir, fusion: { alpha }, embedUnits } as parsed
513
+ // from trusted configuration (src/search/embedding-config.js), or with an
514
+ // embedder instance in place of the configuration object. context: { root,
515
+ // write, env, fetch, now }.
516
+ async function openEmbeddingRuntime(settings, context) {
517
+ const runtime = new EmbeddingRuntime(settings, context);
518
+ return runtime.open(settings);
519
+ }
520
+
521
+ module.exports = {
522
+ EmbeddingRuntime,
523
+ FAILURE_COOLDOWN_MS,
524
+ QUERY_CACHE_SIZE,
525
+ openEmbeddingRuntime,
526
+ };