@delali/narsil 0.2.2 → 0.2.3-next.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +12 -9
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-CVqI5NBt.d.ts → adapters-79OIa4xm.d.ts} +1 -1
  8. package/dist/brute-force-DbHYCTc3.d.ts +8 -0
  9. package/dist/chunk-BFIH7OV5.mjs +185 -0
  10. package/dist/chunk-C6VVLMRX.mjs +63 -0
  11. package/dist/{chunk-UNJZ7ZDB.mjs → chunk-EMI63EXP.mjs} +33 -2
  12. package/dist/chunk-MFFAJ2NR.mjs +135 -0
  13. package/dist/chunk-OBXS7OZM.mjs +376 -0
  14. package/dist/{chunk-ASI6AOW2.mjs → chunk-RO4EFWE6.mjs} +7833 -3524
  15. package/dist/{chunk-WLR5BFL3.mjs → chunk-TEAJO7CU.mjs} +3067 -2087
  16. package/dist/{chunk-S7RJJMHI.mjs → chunk-WDL632CN.mjs} +699 -222
  17. package/dist/{chunk-BKAI2WXI.mjs → chunk-WTONLDU3.mjs} +5 -0
  18. package/dist/client-DQHB_swQ.d.ts +793 -0
  19. package/dist/client.d.ts +6 -0
  20. package/dist/client.mjs +843 -0
  21. package/dist/{config-DXypXAfN.d.ts → config-ChZ7lfqc.d.ts} +3 -3
  22. package/dist/direct-executor-YARCDMJF.mjs +8 -0
  23. package/dist/distribution/coordinator/etcd.d.ts +3 -3
  24. package/dist/distribution/coordinator/etcd.mjs +471 -2
  25. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  26. package/dist/distribution/coordinator/in-memory.mjs +286 -1
  27. package/dist/distribution/transport/grpc.d.ts +45 -0
  28. package/dist/distribution/transport/grpc.mjs +586 -0
  29. package/dist/distribution/transport/in-memory.d.ts +118 -0
  30. package/dist/distribution/transport/in-memory.mjs +214 -0
  31. package/dist/distribution/transport/tcp.d.ts +27 -0
  32. package/dist/distribution/transport/tcp.mjs +595 -0
  33. package/dist/distribution.d.ts +241 -192
  34. package/dist/distribution.mjs +6620 -4795
  35. package/dist/embeddings/openai.d.ts +1 -1
  36. package/dist/embeddings/openai.mjs +1 -1
  37. package/dist/{engine-CO9Wnfie.d.ts → engine-DScGlT2Z.d.ts} +30 -3
  38. package/dist/index.browser.mjs +10035 -3914
  39. package/dist/index.d.ts +8 -8
  40. package/dist/index.mjs +8 -5
  41. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  42. package/dist/invalidation/filesystem.d.ts +1 -1
  43. package/dist/invalidation/filesystem.mjs +3 -1
  44. package/dist/invalidation/noop.d.ts +1 -1
  45. package/dist/options-BORJg5XB.d.ts +234 -0
  46. package/dist/persistence/durability/checkpoint-worker.d.ts +4 -2
  47. package/dist/persistence/durability/checkpoint-worker.mjs +8 -5
  48. package/dist/react.d.ts +340 -0
  49. package/dist/react.mjs +687 -0
  50. package/dist/{schema-CE6S_aAr.d.ts → schema-emTQfEiX.d.ts} +21 -7
  51. package/dist/{search-DLBtiPdQ.d.ts → search-CQ-9bKxo.d.ts} +226 -14
  52. package/dist/server.d.ts +42 -356
  53. package/dist/server.mjs +493 -106
  54. package/dist/tasks-CnMLpfk-.d.ts +274 -0
  55. package/dist/types-BQDlIvfS.d.ts +34 -0
  56. package/dist/{types-BV3nAKTL.d.ts → types-Bfhhnh3G.d.ts} +23 -1
  57. package/dist/types-D5OwhtsQ.d.ts +97 -0
  58. package/dist/vector/hnsw-build-worker.d.ts +2 -7
  59. package/dist/vector/hnsw-build-worker.mjs +2 -1
  60. package/dist/vector/search-worker.d.ts +373 -0
  61. package/dist/vector/search-worker.mjs +279 -0
  62. package/dist/workers/entry.mjs +43 -25
  63. package/package.json +42 -2
  64. package/dist/chunk-FKK76OR7.mjs +0 -456
  65. package/dist/chunk-RQLXC2YS.mjs +0 -269
  66. package/dist/direct-executor-3W4QRA4E.mjs +0 -5
@@ -0,0 +1,373 @@
1
+ import { V as VectorMetric } from '../brute-force-DbHYCTc3.js';
2
+
3
+ /**
4
+ * A filter over store ordinals, one bit per ordinal, with the number of set
5
+ * bits alongside so a search can judge selectivity without a scan.
6
+ *
7
+ * A search checks membership by indexing a byte array rather than hashing a
8
+ * document id, and the whole filter posts to a worker as a few kilobytes of
9
+ * bytes rather than a set of strings.
10
+ *
11
+ * @internal
12
+ */
13
+ interface OrdinalFilter {
14
+ /** One bit per ordinal, set where the ordinal passes the filter. */
15
+ bits: Uint8Array;
16
+ /** This many ordinals pass the filter. */
17
+ count: number;
18
+ }
19
+
20
+ /**
21
+ * Every stored vector in the form the engine hands to another thread.
22
+ *
23
+ * @internal
24
+ */
25
+ interface VectorStoreSnapshot {
26
+ /** Each vector carries this many components. */
27
+ dimension: number;
28
+ /** The store spans this many ordinals, deleted ones included. */
29
+ slots: number;
30
+ /** This holds every vector end to end, `slots * dimension` components long. */
31
+ vectors: Float32Array;
32
+ /** Each ordinal's vector length, so a reader need not recompute it. */
33
+ magnitudes: Float64Array;
34
+ /** The document at each ordinal, `null` where the ordinal holds none. */
35
+ docIds: Array<string | null>;
36
+ }
37
+
38
+ /**
39
+ * The two constants a scalar quantizer turns a vector component into a byte
40
+ * with, and turns that byte back into a distance with.
41
+ *
42
+ * A worker holding a copy of a vector field receives these rather than deriving
43
+ * its own, because a copy taken after a delete would otherwise measure a
44
+ * narrower range of values and answer the same query differently from the
45
+ * thread that built the index.
46
+ *
47
+ * @internal
48
+ */
49
+ interface ScalarQuantizerCalibration {
50
+ /** Each step of the byte scale spans this much of the component range. */
51
+ alpha: number;
52
+ /** The byte scale starts at this component value. */
53
+ offset: number;
54
+ }
55
+
56
+ /**
57
+ * Every node's neighbours in the flat form the engine hands to another thread.
58
+ *
59
+ * @internal
60
+ */
61
+ interface AdjacencySnapshot {
62
+ /** Each node keeps this many neighbours per layer above the base. */
63
+ m: number;
64
+ /** Each node keeps this many neighbours on the base layer. */
65
+ mMax0: number;
66
+ /** The arrays span this many ordinals. */
67
+ capacity: number;
68
+ /** No node sits at an ordinal at or above this. */
69
+ slots: number;
70
+ /** The upper-layer arena holds this many entries in use. */
71
+ upperUsed: number;
72
+ /** Each ordinal's top layer, or -1 where no node sits there. */
73
+ nodeLevels: Int8Array;
74
+ /** The base layer's neighbours, a count followed by neighbours per ordinal. */
75
+ level0: Int32Array;
76
+ /** Where each ordinal's upper layers start, or -1 where it has none. */
77
+ upperBase: Int32Array;
78
+ /** Every upper layer's neighbours, a count followed by neighbours per layer. */
79
+ upper: Int32Array;
80
+ }
81
+
82
+ /**
83
+ * A built graph in the form the engine hands to another thread.
84
+ *
85
+ * @internal
86
+ */
87
+ interface HNSWSnapshot {
88
+ /** Each vector carries this many components. */
89
+ dimension: number;
90
+ /** Each node keeps this many neighbours per layer. */
91
+ m: number;
92
+ /** The builder explored this many candidates while placing each node. */
93
+ efConstruction: number;
94
+ /** The graph ranks by this metric. */
95
+ metric: VectorMetric;
96
+ /** This holds every node's neighbours, laid out flat. */
97
+ adjacency: AdjacencySnapshot;
98
+ /** A byte per ordinal, set to 1 where the document has been deleted. */
99
+ tombstones: Uint8Array;
100
+ /** This many ordinals are tombstoned. */
101
+ tombstoneCount: number;
102
+ /** The graph holds this many nodes, tombstoned ones included. */
103
+ nodeCount: number;
104
+ /** The graph's own arrays span this many ordinals. */
105
+ capacity: number;
106
+ /** Every search starts at this ordinal, and it is -1 while the graph is empty. */
107
+ entryPointOrd: number;
108
+ /** The graph reaches this many layers. */
109
+ topLayer: number;
110
+ }
111
+
112
+ /**
113
+ * Where each region of a frozen generation sits inside its shared memory.
114
+ *
115
+ * The memory opens with one float32 query scratch slot and one code query
116
+ * scratch slot per worker, so concurrent searches never write over each
117
+ * other, followed by the float32 vector arena and the quantised code arena.
118
+ *
119
+ * @internal
120
+ */
121
+ interface SharedGenerationLayout {
122
+ /** Each vector carries this many components. */
123
+ dimension: number;
124
+ /** The arenas span this many ordinals. */
125
+ slots: number;
126
+ /** The memory holds this many per-worker scratch slot pairs. */
127
+ workerSlots: number;
128
+ /** Each worker's float32 query scratch spans this many bytes. */
129
+ float32ScratchStride: number;
130
+ /** Each worker's code query scratch spans this many bytes. */
131
+ codeScratchStride: number;
132
+ /** The code query scratch slots start at this byte offset. */
133
+ codeScratchOffset: number;
134
+ /** The float32 vector arena starts at this byte offset. */
135
+ vectorsOffset: number;
136
+ /** The quantised code arena starts at this byte offset. */
137
+ codesOffset: number;
138
+ /** The memory spans this many bytes, a whole number of pages. */
139
+ totalBytes: number;
140
+ }
141
+ /**
142
+ * One frozen copy of a vector field, shared across every search thread.
143
+ *
144
+ * The typed arrays are views over shared buffers, so posting this to a worker
145
+ * hands over the same bytes rather than a copy, and nothing in it is written
146
+ * after the freeze apart from the per-worker scratch slots inside `memory`.
147
+ *
148
+ * @internal
149
+ */
150
+ interface SharedGenerationSnapshot {
151
+ /** Each vector carries this many components. */
152
+ dimension: number;
153
+ /** The workers compute quantised distances when this reads `sq8`. */
154
+ quantization: 'sq8' | 'none';
155
+ /** The constants every code was derived with, present when quantised. */
156
+ calibration: ScalarQuantizerCalibration | null;
157
+ /** The shared memory holding the scratch slots and both arenas. */
158
+ memory: WebAssembly.Memory;
159
+ /** Where each region sits inside the shared memory. */
160
+ layout: SharedGenerationLayout;
161
+ /** Each ordinal's vector magnitude. */
162
+ magnitudes: Float64Array;
163
+ /** Each ordinal's code sum, present when quantised. */
164
+ codeSums: Float64Array;
165
+ /** Each ordinal's reconstructed code magnitude, present when quantised. */
166
+ codeMagnitudes: Float64Array;
167
+ /** A byte per ordinal, 1 where codes exist, present when quantised. */
168
+ codePresent: Uint8Array;
169
+ /** The built graph, its arrays backed by shared buffers. */
170
+ graph: HNSWSnapshot;
171
+ /**
172
+ * Each ordinal's document id rank in code point order, or
173
+ * the absent-document rank where the ordinal holds no document.
174
+ */
175
+ rankByOrdinal: Uint32Array;
176
+ }
177
+
178
+ /**
179
+ * One vector field's searchable state in the form the engine clones to a
180
+ * worker, used where the runtime cannot share memory.
181
+ *
182
+ * @internal
183
+ */
184
+ interface WorkerCopySnapshot {
185
+ /** Each vector carries this many components. */
186
+ dimension: number;
187
+ /** The worker rebuilds a quantizer when this reads `sq8`. */
188
+ quantization: 'sq8' | 'none';
189
+ /**
190
+ * The constants the calling thread quantizes with, carried so that the worker
191
+ * derives the same codes rather than recalibrating over a set that a delete
192
+ * has already narrowed.
193
+ */
194
+ calibration: ScalarQuantizerCalibration | null;
195
+ /** This holds every vector and the document id at each ordinal. */
196
+ store: VectorStoreSnapshot;
197
+ /** This holds the built graph. */
198
+ graph: HNSWSnapshot;
199
+ /** These documents have been deleted and must not be returned. */
200
+ tombstones: string[];
201
+ }
202
+
203
+ /**
204
+ * Asks the search worker to hold a cloned copy of one vector field under a
205
+ * handle.
206
+ *
207
+ * These messages travel between the engine and its search worker entry point,
208
+ * and nothing here is part of the engine's supported surface.
209
+ *
210
+ * @internal
211
+ */
212
+ interface VectorLoadRequest {
213
+ /** This marks the message as a cloned copy load. */
214
+ type: 'load';
215
+ /** The reply carries this back so the engine can match it to its caller. */
216
+ requestId: string;
217
+ /** Later messages name the copy by this handle. */
218
+ handle: string;
219
+ /** The worker rebuilds its searchable copy from this. */
220
+ snapshot: WorkerCopySnapshot;
221
+ }
222
+ /**
223
+ * Asks the search worker to open a frozen shared copy under a handle.
224
+ *
225
+ * @internal
226
+ */
227
+ interface SharedCopyLoadRequest {
228
+ /** This marks the message as a shared copy load. */
229
+ type: 'loadShared';
230
+ /** The reply carries this back so the engine can match it to its caller. */
231
+ requestId: string;
232
+ /** Later messages name the copy by this handle. */
233
+ handle: string;
234
+ /** The worker writes query scratch into this reserved slot alone. */
235
+ scratchSlot: number;
236
+ /** The frozen copy, its buffers shared rather than cloned. */
237
+ snapshot: SharedGenerationSnapshot;
238
+ }
239
+ /**
240
+ * Asks the search worker to release a copy it holds.
241
+ *
242
+ * @internal
243
+ */
244
+ interface VectorDropRequest {
245
+ /** This marks the message as a copy release. */
246
+ type: 'drop';
247
+ /** The reply carries this back so the engine can match it to its caller. */
248
+ requestId: string;
249
+ /** The worker releases the copy held under this handle. */
250
+ handle: string;
251
+ }
252
+ /**
253
+ * Asks the search worker to answer one nearest-neighbour query from a cloned
254
+ * copy.
255
+ *
256
+ * @internal
257
+ */
258
+ interface VectorSearchRequest {
259
+ /** This marks the message as a search over a cloned copy. */
260
+ type: 'search';
261
+ /** The reply carries this back so the engine can match it to its caller. */
262
+ requestId: string;
263
+ /** The worker searches the copy held under this handle. */
264
+ handle: string;
265
+ /** The worker ranks against this query vector. */
266
+ query: Float32Array;
267
+ /** The worker returns at most this many results. */
268
+ k: number;
269
+ /** The worker ranks by this metric. */
270
+ metric: VectorMetric;
271
+ /** The worker drops any result scoring below this. */
272
+ minSimilarity: number;
273
+ /** The worker returns only ordinals this filter holds, when present. */
274
+ filter?: OrdinalFilter;
275
+ /** The worker explores this many candidates, or its own default when absent. */
276
+ efSearch?: number;
277
+ }
278
+ /**
279
+ * Asks the search worker to answer one nearest-neighbour query from a shared
280
+ * copy, returning ordinals for the engine to map back to document ids.
281
+ *
282
+ * @internal
283
+ */
284
+ interface VectorOrdinalSearchRequest {
285
+ /** This marks the message as a search over a shared copy. */
286
+ type: 'searchOrdinals';
287
+ /** The reply carries this back so the engine can match it to its caller. */
288
+ requestId: string;
289
+ /** The worker searches the copy held under this handle. */
290
+ handle: string;
291
+ /** The worker ranks against this query vector. */
292
+ query: Float32Array;
293
+ /** The worker returns at most this many results. */
294
+ k: number;
295
+ /** The worker ranks by this metric. */
296
+ metric: VectorMetric;
297
+ /** The worker drops any result scoring below this. */
298
+ minSimilarity: number;
299
+ /** The worker returns only ordinals this filter holds, when present. */
300
+ filter?: OrdinalFilter;
301
+ /** The worker explores this many candidates, or its own default when absent. */
302
+ efSearch?: number;
303
+ }
304
+ /**
305
+ * Every message the engine posts to the search worker.
306
+ *
307
+ * @internal
308
+ */
309
+ type VectorWorkerRequest = VectorLoadRequest | SharedCopyLoadRequest | VectorDropRequest | VectorSearchRequest | VectorOrdinalSearchRequest;
310
+ /**
311
+ * What the search worker returns once a copy is loaded or released.
312
+ *
313
+ * @internal
314
+ */
315
+ interface VectorAckResponse {
316
+ /** This marks the message as a completed load or release. */
317
+ type: 'ack';
318
+ /** This matches the request the engine sent. */
319
+ requestId: string;
320
+ /** This names the copy the acknowledgement belongs to. */
321
+ handle: string;
322
+ }
323
+ /**
324
+ * What the search worker returns for a completed search over a cloned copy.
325
+ *
326
+ * @internal
327
+ */
328
+ interface VectorSearchResponse {
329
+ /** This marks the message as a completed search. */
330
+ type: 'result';
331
+ /** This matches the request the engine sent. */
332
+ requestId: string;
333
+ /** These document ids run best first. */
334
+ docIds: string[];
335
+ /** These scores line up with the document ids. */
336
+ scores: Float64Array;
337
+ }
338
+ /**
339
+ * What the search worker returns for a completed search over a shared copy.
340
+ *
341
+ * @internal
342
+ */
343
+ interface VectorOrdinalSearchResponse {
344
+ /** This marks the message as a completed ordinal search. */
345
+ type: 'ordinalResult';
346
+ /** This matches the request the engine sent. */
347
+ requestId: string;
348
+ /** These store ordinals run best first. */
349
+ ordinals: Uint32Array;
350
+ /** These scores line up with the ordinals. */
351
+ scores: Float64Array;
352
+ }
353
+ /**
354
+ * What the search worker returns when a request fails.
355
+ *
356
+ * @internal
357
+ */
358
+ interface VectorWorkerError {
359
+ /** This marks the message as a failure. */
360
+ type: 'error';
361
+ /** This matches the request the engine sent, and is absent for a load or release. */
362
+ requestId?: string;
363
+ /** This says why the request failed. */
364
+ message: string;
365
+ }
366
+ /**
367
+ * Every message the search worker posts back.
368
+ *
369
+ * @internal
370
+ */
371
+ type VectorWorkerMessage = VectorAckResponse | VectorSearchResponse | VectorOrdinalSearchResponse | VectorWorkerError;
372
+
373
+ export { type AdjacencySnapshot, type HNSWSnapshot, type OrdinalFilter, type ScalarQuantizerCalibration, type SharedCopyLoadRequest, type SharedGenerationLayout, type SharedGenerationSnapshot, type VectorAckResponse, type VectorDropRequest, type VectorLoadRequest, VectorMetric, type VectorOrdinalSearchRequest, type VectorOrdinalSearchResponse, type VectorSearchRequest, type VectorSearchResponse, type VectorStoreSnapshot, type VectorWorkerError, type VectorWorkerMessage, type VectorWorkerRequest, type WorkerCopySnapshot };
@@ -0,0 +1,279 @@
1
+ import { createScalarQuantizer, deriveSq8Constants, arenaQuantizedDistance, quantizeVectorInto, quantizedMagnitude, scalarQuantizedDistance } from '../chunk-OBXS7OZM.mjs';
2
+ import { searchOrdinals, createVectorStore, createHNSWIndex, createSharedArenaSimd, importAdjacency, arenaFloat32Distance, ABSENT_DOCUMENT_RANK } from '../chunk-WDL632CN.mjs';
3
+ import '../chunk-BFIH7OV5.mjs';
4
+
5
+ // src/vector/shared-generation/worker-view.ts
6
+ function createVectorReader(snapshot, simd, scratchSlot) {
7
+ const { layout, magnitudes, rankByOrdinal } = snapshot;
8
+ const { dimension, slots } = layout;
9
+ const vectors = new Float32Array(snapshot.memory.buffer, layout.vectorsOffset, slots * dimension);
10
+ const scratchByteOffset = scratchSlot * layout.float32ScratchStride;
11
+ const scratch = simd ? new Float32Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
12
+ const holdsDocument = (ordinal) => ordinal >= 0 && ordinal < slots && rankByOrdinal[ordinal] !== ABSENT_DOCUMENT_RANK;
13
+ return {
14
+ entryForOrdinal(ordinal) {
15
+ if (!holdsDocument(ordinal)) return void 0;
16
+ const base = ordinal * dimension;
17
+ return { vector: vectors.subarray(base, base + dimension), magnitude: magnitudes[ordinal] };
18
+ },
19
+ prepareQueryArena(query) {
20
+ if (!simd || !scratch || query.length !== dimension) return null;
21
+ scratch.set(query, 0);
22
+ return { magnitude: simd.magnitude(scratchByteOffset, dimension) };
23
+ },
24
+ distanceFromArena(prepared, ordinal, metric) {
25
+ if (!simd || !holdsDocument(ordinal)) return Number.POSITIVE_INFINITY;
26
+ const byteOffset = layout.vectorsOffset + ordinal * dimension * 4;
27
+ return arenaFloat32Distance(
28
+ simd,
29
+ scratchByteOffset,
30
+ byteOffset,
31
+ dimension,
32
+ metric,
33
+ prepared.magnitude,
34
+ magnitudes[ordinal]
35
+ );
36
+ }
37
+ };
38
+ }
39
+ function createQuantizerReader(snapshot, simd, scratchSlot) {
40
+ if (snapshot.quantization !== "sq8" || snapshot.calibration === null) return void 0;
41
+ const { layout, codeSums, codeMagnitudes, codePresent } = snapshot;
42
+ const { dimension, slots } = layout;
43
+ const constants = deriveSq8Constants(snapshot.calibration.alpha, snapshot.calibration.offset, dimension);
44
+ const codes = new Uint8Array(snapshot.memory.buffer, layout.codesOffset, slots * dimension);
45
+ const scratchByteOffset = layout.codeScratchOffset + scratchSlot * layout.codeScratchStride;
46
+ const scratch = simd ? new Uint8Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
47
+ let liveCodes = 0;
48
+ for (let ordinal = 0; ordinal < codePresent.length; ordinal++) {
49
+ if (codePresent[ordinal] === 1) liveCodes++;
50
+ }
51
+ const holdsCodes = (ordinal) => ordinal >= 0 && ordinal < slots && codePresent[ordinal] === 1;
52
+ return {
53
+ get size() {
54
+ return liveCodes;
55
+ },
56
+ isCalibrated: () => true,
57
+ prepareQuery(query) {
58
+ const quantized = new Uint8Array(dimension);
59
+ const { sum, sumSq } = quantizeVectorInto(quantized, query, dimension, constants);
60
+ return { quantized, sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
61
+ },
62
+ distanceFromPreparedByOrdinal(prepared, ordinal, metric) {
63
+ if (!holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
64
+ return scalarQuantizedDistance(
65
+ codes,
66
+ codeSums,
67
+ codeMagnitudes,
68
+ dimension,
69
+ constants,
70
+ prepared.quantized,
71
+ prepared.sum,
72
+ prepared.magnitude,
73
+ ordinal,
74
+ metric
75
+ );
76
+ },
77
+ prepareQueryArena(query) {
78
+ if (!simd || !scratch) return null;
79
+ const { sum, sumSq } = quantizeVectorInto(scratch, query, dimension, constants);
80
+ return { sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
81
+ },
82
+ distanceFromArena(prepared, ordinal, metric) {
83
+ if (!simd || !holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
84
+ return arenaQuantizedDistance(
85
+ simd,
86
+ scratchByteOffset,
87
+ layout.codesOffset + ordinal * dimension,
88
+ dimension,
89
+ constants,
90
+ prepared.sum,
91
+ prepared.magnitude,
92
+ codeSums[ordinal],
93
+ codeMagnitudes[ordinal],
94
+ metric
95
+ );
96
+ }
97
+ };
98
+ }
99
+ function openSharedWorkerCopy(snapshot, scratchSlot) {
100
+ if (scratchSlot < 0 || scratchSlot >= snapshot.layout.workerSlots) {
101
+ throw new Error(`Scratch slot ${scratchSlot} is outside the ${snapshot.layout.workerSlots} reserved slots`);
102
+ }
103
+ const simd = createSharedArenaSimd(snapshot.memory);
104
+ const adjacency = importAdjacency(snapshot.graph.adjacency);
105
+ const capacity = Math.max(snapshot.graph.capacity, adjacency.capacity);
106
+ const searchState = {
107
+ dimension: snapshot.dimension,
108
+ store: createVectorReader(snapshot, simd, scratchSlot),
109
+ quantizer: createQuantizerReader(snapshot, simd, scratchSlot),
110
+ adjacency,
111
+ tombstones: snapshot.graph.tombstones,
112
+ tombstoneCount: snapshot.graph.tombstoneCount,
113
+ nodeCount: snapshot.graph.nodeCount,
114
+ capacity,
115
+ visited: new Uint32Array(capacity),
116
+ visitStamp: 0,
117
+ entryPointOrd: snapshot.graph.entryPointOrd,
118
+ topLayer: snapshot.graph.topLayer
119
+ };
120
+ return { searchState, rankByOrdinal: snapshot.rankByOrdinal };
121
+ }
122
+
123
+ // src/vector/worker-copy.ts
124
+ function restoreWorkerCopy(snapshot) {
125
+ const store = createVectorStore();
126
+ store.restoreSnapshot(snapshot.store);
127
+ const tombstones = new Set(snapshot.tombstones);
128
+ let quantizer = null;
129
+ if (snapshot.quantization === "sq8") {
130
+ const sq8 = createScalarQuantizer(snapshot.dimension, store);
131
+ if (snapshot.calibration !== null) {
132
+ sq8.restoreCalibration(snapshot.calibration.alpha, snapshot.calibration.offset);
133
+ } else {
134
+ const live = [];
135
+ for (const [docId, entry] of store.entries()) {
136
+ if (tombstones.has(docId)) continue;
137
+ live.push(entry.vector);
138
+ }
139
+ sq8.calibrate(live);
140
+ }
141
+ for (const [docId, entry] of store.entries()) {
142
+ if (tombstones.has(docId)) continue;
143
+ sq8.quantize(docId, entry.vector);
144
+ }
145
+ quantizer = sq8;
146
+ }
147
+ const graph = createHNSWIndex(
148
+ snapshot.dimension,
149
+ store,
150
+ { m: snapshot.graph.m, efConstruction: snapshot.graph.efConstruction, metric: snapshot.graph.metric },
151
+ quantizer ?? void 0
152
+ );
153
+ graph.restoreSnapshot(snapshot.graph);
154
+ return { store, graph, tombstones };
155
+ }
156
+
157
+ // src/vector/search-worker.ts
158
+ var MAX_WORKER_COPIES = 64;
159
+ var copies = /* @__PURE__ */ new Map();
160
+ function ensureRoom(handle) {
161
+ if (!copies.has(handle) && copies.size >= MAX_WORKER_COPIES) {
162
+ throw new Error(`Search worker already holds ${MAX_WORKER_COPIES} copies`);
163
+ }
164
+ }
165
+ function handleLoad(request) {
166
+ ensureRoom(request.handle);
167
+ copies.set(request.handle, { kind: "clone", copy: restoreWorkerCopy(request.snapshot) });
168
+ return { type: "ack", requestId: request.requestId, handle: request.handle };
169
+ }
170
+ function handleLoadShared(request) {
171
+ ensureRoom(request.handle);
172
+ copies.set(request.handle, { kind: "shared", copy: openSharedWorkerCopy(request.snapshot, request.scratchSlot) });
173
+ return { type: "ack", requestId: request.requestId, handle: request.handle };
174
+ }
175
+ function handleDrop(request) {
176
+ copies.delete(request.handle);
177
+ return { type: "ack", requestId: request.requestId, handle: request.handle };
178
+ }
179
+ function handleSearch(request) {
180
+ const entry = copies.get(request.handle);
181
+ if (!entry) {
182
+ throw new Error(`Search worker holds no copy for handle ${request.handle}`);
183
+ }
184
+ if (entry.kind !== "clone") {
185
+ throw new Error(`Handle ${request.handle} holds a shared copy, which answers ordinal searches alone`);
186
+ }
187
+ const hits = entry.copy.graph.search(
188
+ request.query,
189
+ request.k,
190
+ request.metric,
191
+ request.minSimilarity,
192
+ request.filter,
193
+ request.efSearch
194
+ );
195
+ const docIds = [];
196
+ const scores = new Float64Array(hits.length);
197
+ for (let i = 0; i < hits.length; i++) {
198
+ docIds.push(hits[i].docId);
199
+ scores[i] = hits[i].score;
200
+ }
201
+ return { type: "result", requestId: request.requestId, docIds, scores };
202
+ }
203
+ function handleSearchOrdinals(request) {
204
+ const entry = copies.get(request.handle);
205
+ if (!entry) {
206
+ throw new Error(`Search worker holds no copy for handle ${request.handle}`);
207
+ }
208
+ if (entry.kind !== "shared") {
209
+ throw new Error(`Handle ${request.handle} holds a cloned copy, which answers document id searches alone`);
210
+ }
211
+ const hits = searchOrdinals(
212
+ entry.copy.searchState,
213
+ request.query,
214
+ request.k,
215
+ request.metric,
216
+ request.minSimilarity,
217
+ entry.copy.rankByOrdinal,
218
+ request.filter,
219
+ request.efSearch
220
+ );
221
+ const ordinals = new Uint32Array(hits.length);
222
+ const scores = new Float64Array(hits.length);
223
+ for (let i = 0; i < hits.length; i++) {
224
+ ordinals[i] = hits[i].ord;
225
+ scores[i] = hits[i].score;
226
+ }
227
+ return { type: "ordinalResult", requestId: request.requestId, ordinals, scores };
228
+ }
229
+ function handleAnyRequest(raw) {
230
+ const request = raw;
231
+ if (request.type === "load") return handleLoad(request);
232
+ if (request.type === "loadShared") return handleLoadShared(request);
233
+ if (request.type === "drop") return handleDrop(request);
234
+ if (request.type === "search") return handleSearch(request);
235
+ if (request.type === "searchOrdinals") return handleSearchOrdinals(request);
236
+ throw new Error(`Unknown request type: ${request.type}`);
237
+ }
238
+ function errorFor(raw, err) {
239
+ const message = err instanceof Error ? err.message : String(err);
240
+ const requestId = raw.requestId;
241
+ return typeof requestId === "string" ? { type: "error", requestId, message } : { type: "error", message };
242
+ }
243
+ function setupWorker() {
244
+ setupAsync().catch((err) => {
245
+ console.error("Vector search worker setup failed:", err);
246
+ });
247
+ }
248
+ async function setupAsync() {
249
+ let parentPort = null;
250
+ try {
251
+ const workerThreads = await import('worker_threads');
252
+ parentPort = workerThreads.parentPort ?? null;
253
+ } catch {
254
+ parentPort = null;
255
+ }
256
+ if (parentPort) {
257
+ const port = parentPort;
258
+ port.on("message", (raw) => {
259
+ try {
260
+ port.postMessage(handleAnyRequest(raw));
261
+ } catch (err) {
262
+ port.postMessage(errorFor(raw, err));
263
+ }
264
+ });
265
+ return;
266
+ }
267
+ const globalSelf = typeof self !== "undefined" ? self : void 0;
268
+ if (globalSelf && typeof globalSelf.postMessage === "function") {
269
+ const webSelf = globalSelf;
270
+ webSelf.onmessage = (event) => {
271
+ try {
272
+ webSelf.postMessage(handleAnyRequest(event.data));
273
+ } catch (err) {
274
+ webSelf.postMessage(errorFor(event.data, err));
275
+ }
276
+ };
277
+ }
278
+ }
279
+ setupWorker();