@delali/narsil 0.2.4-next.59 → 0.3.1-next.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-DffQlmRq.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -0,0 +1,1011 @@
1
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-LGAMKHA2.mjs';
2
+ import { MAX_VECTOR_ORDINALS } from './chunk-GBEEG5AN.mjs';
3
+ import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
4
+ import { readSync, closeSync, openSync } from 'fs';
5
+
6
+ function createVectorFileReader() {
7
+ const descriptors = /* @__PURE__ */ new Map();
8
+ function descriptorOf(path) {
9
+ const open = descriptors.get(path);
10
+ if (open !== void 0) return open;
11
+ try {
12
+ const descriptor = openSync(path, "r");
13
+ descriptors.set(path, descriptor);
14
+ return descriptor;
15
+ } catch {
16
+ return null;
17
+ }
18
+ }
19
+ function closeDescriptor(path, descriptor) {
20
+ descriptors.delete(path);
21
+ try {
22
+ closeSync(descriptor);
23
+ } catch {
24
+ }
25
+ }
26
+ return {
27
+ readInto(path, offset, target) {
28
+ const descriptor = descriptorOf(path);
29
+ if (descriptor === null) return false;
30
+ let read = 0;
31
+ while (read < target.byteLength) {
32
+ let chunk;
33
+ try {
34
+ chunk = readSync(descriptor, target, read, target.byteLength - read, offset + read);
35
+ } catch {
36
+ closeDescriptor(path, descriptor);
37
+ return false;
38
+ }
39
+ if (chunk <= 0) return false;
40
+ read += chunk;
41
+ }
42
+ return true;
43
+ },
44
+ retainOnly(paths) {
45
+ for (const [path, descriptor] of descriptors) {
46
+ if (!paths.has(path)) closeDescriptor(path, descriptor);
47
+ }
48
+ },
49
+ close() {
50
+ for (const [path, descriptor] of descriptors) closeDescriptor(path, descriptor);
51
+ }
52
+ };
53
+ }
54
+
55
+ // src/vector/vector-store/hot-block-distance.ts
56
+ function jsDistance(a, b, magA, magB, metric) {
57
+ switch (metric) {
58
+ case "cosine":
59
+ return 1 - cosineSimilarityWithMagnitudes(a, b, magA, magB);
60
+ case "dotProduct":
61
+ return -dotProduct(a, b);
62
+ case "euclidean":
63
+ return euclideanDistance(a, b);
64
+ }
65
+ }
66
+ function inMemory(reads, ordinal) {
67
+ return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
68
+ }
69
+ function queryDistanceOf(source, prepared, metric) {
70
+ const generic = (ordinal) => source.distanceFromArena(prepared, ordinal, metric);
71
+ const reads = source.hotBlockReads();
72
+ const simd = reads?.block.simd ?? null;
73
+ if (reads === null || simd === null || !reads.block.hasScratch) return generic;
74
+ const { block, present, magnitudes, slotsOffset, slotStride, dimension } = reads;
75
+ const scratchOffset = block.scratchByteOffset;
76
+ const queryMagnitude = prepared.magnitude;
77
+ return (ordinal) => {
78
+ if (!inMemory(reads, ordinal) || block.stagedOrdinal !== QUERY_STAGED) return generic(ordinal);
79
+ if (present[ordinal] !== 1) return Number.POSITIVE_INFINITY;
80
+ const slotOffset = slotsOffset + ordinal * slotStride;
81
+ return arenaFloat32Distance(simd, scratchOffset, slotOffset, dimension, metric, queryMagnitude, magnitudes[ordinal]);
82
+ };
83
+ }
84
+ function pairDistanceOf(source, metric) {
85
+ const generic = (ordA, ordB) => source.distanceByOrdinal(ordA, ordB, metric);
86
+ const reads = source.hotBlockReads();
87
+ const simd = reads?.block.simd ?? null;
88
+ if (reads === null || simd === null) return generic;
89
+ const { present, magnitudes, slotsOffset, slotStride, dimension } = reads;
90
+ return (ordA, ordB) => {
91
+ if (!inMemory(reads, ordA) || !inMemory(reads, ordB) || present[ordA] !== 1) return generic(ordA, ordB);
92
+ if (present[ordB] !== 1) return Number.POSITIVE_INFINITY;
93
+ const offsetA = slotsOffset + ordA * slotStride;
94
+ const offsetB = slotsOffset + ordB * slotStride;
95
+ return arenaFloat32Distance(simd, offsetA, offsetB, dimension, metric, magnitudes[ordA], magnitudes[ordB]);
96
+ };
97
+ }
98
+ function ordinalDistanceOf(source, from, metric) {
99
+ const pair = pairDistanceOf(source, metric);
100
+ return (ordinal) => pair(from, ordinal);
101
+ }
102
+
103
+ // src/vector/vector-store/view.ts
104
+ var decoder = new TextDecoder();
105
+ var NOT_HELD = 0;
106
+ var HELD_COLD = 1;
107
+ var HELD_HOT = 2;
108
+ function openSharedVectorStore(initial, threadSlot) {
109
+ let handles = initial;
110
+ const { dimension, layout } = initial;
111
+ const capacity = layout.capacity;
112
+ const codeCapacity = initial.codeLayout?.capacity ?? 1;
113
+ const blocks = [];
114
+ const codeBlocks = [];
115
+ let present = fixedView(handles.present, Uint8Array);
116
+ let magnitudes = fixedView(handles.magnitudes, Float64Array);
117
+ let partitions = fixedView(handles.partitions, Int32Array);
118
+ let docIdBytes = fixedView(handles.docIdBytes, Uint8Array);
119
+ let docIdOffsets = fixedView(handles.docIdOffsets, Uint32Array);
120
+ let diskFile = fixedView(handles.diskFile, Int32Array);
121
+ let diskOffset = fixedView(handles.diskOffset, Uint32Array);
122
+ let currentQuery = null;
123
+ let reader = null;
124
+ const coldScratchA = new Float32Array(dimension);
125
+ const coldScratchB = new Float32Array(dimension);
126
+ function rebind() {
127
+ present = fixedView(handles.present, Uint8Array);
128
+ magnitudes = fixedView(handles.magnitudes, Float64Array);
129
+ partitions = fixedView(handles.partitions, Int32Array);
130
+ docIdBytes = fixedView(handles.docIdBytes, Uint8Array);
131
+ docIdOffsets = fixedView(handles.docIdOffsets, Uint32Array);
132
+ diskFile = fixedView(handles.diskFile, Int32Array);
133
+ diskOffset = fixedView(handles.diskOffset, Uint32Array);
134
+ }
135
+ function blockAt(index) {
136
+ let block = blocks[index];
137
+ if (block === void 0) {
138
+ const handle = handles.blocks[index];
139
+ if (handle === null || handle === void 0) {
140
+ throw new Error(`Vector block ${index} holds no vectors on this thread`);
141
+ }
142
+ block = openVectorBlock(handle, threadSlot);
143
+ blocks[index] = block;
144
+ }
145
+ return block;
146
+ }
147
+ function codeBlockAt(index) {
148
+ let block = codeBlocks[index];
149
+ if (block === void 0) {
150
+ block = openVectorBlock(handles.codeBlocks[index], threadSlot);
151
+ codeBlocks[index] = block;
152
+ }
153
+ return block;
154
+ }
155
+ function fileOf(ordinal) {
156
+ if (ordinal >= diskFile.length) {
157
+ if (ordinal * 4 >= handles.diskFile.byteLength) return IN_MEMORY;
158
+ rebind();
159
+ }
160
+ return diskFile[ordinal];
161
+ }
162
+ function isCold(ordinal) {
163
+ return fileOf(ordinal) !== IN_MEMORY;
164
+ }
165
+ function holdState(ordinal) {
166
+ if (ordinal < 0) return NOT_HELD;
167
+ if (ordinal >= present.length) {
168
+ if (ordinal >= handles.present.byteLength) return NOT_HELD;
169
+ rebind();
170
+ }
171
+ if (present[ordinal] !== 1) return NOT_HELD;
172
+ if (isCold(ordinal)) return HELD_COLD;
173
+ const index = Math.floor(ordinal / capacity);
174
+ return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
175
+ }
176
+ function holds(ordinal) {
177
+ return holdState(ordinal) !== NOT_HELD;
178
+ }
179
+ function readFromFile(file, offset, target) {
180
+ const path = handles.vectorFiles[file];
181
+ if (path === void 0 || path.length === 0) return false;
182
+ if (reader === null) reader = createVectorFileReader();
183
+ const bytes = new Uint8Array(target.buffer, target.byteOffset, target.byteLength);
184
+ return reader.readInto(path, offset, bytes);
185
+ }
186
+ function readColdInto(ordinal, target) {
187
+ const file = fileOf(ordinal);
188
+ if (file === IN_MEMORY) return false;
189
+ if (ordinal >= diskOffset.length) rebind();
190
+ return readFromFile(file, diskOffset[ordinal], target);
191
+ }
192
+ function vectorIn(block, local) {
193
+ const base = slotByteOffset(layout, local) / 4;
194
+ return block.float32(base + dimension).subarray(base, base + dimension);
195
+ }
196
+ function hotVector(ordinal) {
197
+ return vectorIn(blockAt(ordinal / capacity | 0), ordinal % capacity);
198
+ }
199
+ function vectorFor(ordinal, scratch) {
200
+ if (!isCold(ordinal)) return hotVector(ordinal);
201
+ return readColdInto(ordinal, scratch) ? scratch : null;
202
+ }
203
+ function stageVector(block, ordinal) {
204
+ if (block.stagedOrdinal === ordinal) return true;
205
+ const source = vectorFor(ordinal, coldScratchA);
206
+ if (source === null) return false;
207
+ block.float32(block.scratchByteOffset / 4 + dimension).set(source, block.scratchByteOffset / 4);
208
+ block.stagedOrdinal = ordinal;
209
+ return true;
210
+ }
211
+ function hotBlockReads() {
212
+ const handle = handles.blocks[0];
213
+ if (handle === null || handle === void 0) return null;
214
+ const { slotsOffset, slotStride } = layout;
215
+ const limit = Math.min(present.length, diskFile.length, magnitudes.length, capacity);
216
+ return { block: blockAt(0), present, diskFile, magnitudes, limit, slotsOffset, slotStride, dimension };
217
+ }
218
+ function stageQuery(block) {
219
+ if (currentQuery === null) return false;
220
+ if (block.stagedOrdinal !== QUERY_STAGED) {
221
+ block.float32(block.scratchByteOffset / 4 + dimension).set(currentQuery, block.scratchByteOffset / 4);
222
+ block.stagedOrdinal = QUERY_STAGED;
223
+ }
224
+ return true;
225
+ }
226
+ const view = {
227
+ dimension,
228
+ threadSlot,
229
+ get handles() {
230
+ return handles;
231
+ },
232
+ get slots() {
233
+ return Atomics.load(handles.header, STORE_SLOTS);
234
+ },
235
+ get simdAvailable() {
236
+ const first = handles.blocks.find((block2) => block2 !== null);
237
+ if (first === void 0) return false;
238
+ const block = blockAt(handles.blocks.indexOf(first));
239
+ return block.simd !== null && block.hasScratch;
240
+ },
241
+ get holdsEveryBlock() {
242
+ return Atomics.load(handles.header, STORE_BLOCK_COUNT) <= handles.blocks.length && Atomics.load(handles.header, STORE_CODE_BLOCK_COUNT) <= handles.codeBlocks.length;
243
+ },
244
+ adoptHandles(next) {
245
+ detachNativeStore(handles);
246
+ handles = next;
247
+ rebind();
248
+ for (let index = 0; index < next.blocks.length; index++) {
249
+ if (next.blocks[index] === null) blocks[index] = void 0;
250
+ else if (blocks[index] === void 0) blockAt(index);
251
+ }
252
+ for (let index = codeBlocks.length; index < next.codeBlocks.length; index++) codeBlockAt(index);
253
+ reader?.retainOnly(new Set(next.vectorFiles.filter((path) => path.length > 0)));
254
+ },
255
+ rebind,
256
+ holdsOrdinal: holds,
257
+ isCold,
258
+ readColdInto,
259
+ readFromFile,
260
+ close() {
261
+ detachNativeStore(handles);
262
+ reader?.close();
263
+ reader = null;
264
+ },
265
+ blockOf(ordinal) {
266
+ return blockAt(ordinal / capacity | 0);
267
+ },
268
+ localOrdinal(ordinal) {
269
+ return ordinal % capacity;
270
+ },
271
+ codeBlockOf(ordinal) {
272
+ return codeBlockAt(ordinal / codeCapacity | 0);
273
+ },
274
+ codeLocalOrdinal(ordinal) {
275
+ return ordinal % codeCapacity;
276
+ },
277
+ openCodeBlocks() {
278
+ return codeBlocks;
279
+ },
280
+ vectorAt(ordinal) {
281
+ if (!isCold(ordinal)) return hotVector(ordinal);
282
+ const copy = new Float32Array(dimension);
283
+ if (!readColdInto(ordinal, copy)) {
284
+ throw new NarsilError(
285
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
286
+ `The vector at ordinal ${ordinal} is missing from its checkpoint file`,
287
+ { ordinal, path: handles.vectorFiles[fileOf(ordinal)] }
288
+ );
289
+ }
290
+ return copy;
291
+ },
292
+ magnitudeAt(ordinal) {
293
+ if (ordinal >= magnitudes.length) rebind();
294
+ return magnitudes[ordinal];
295
+ },
296
+ entryForOrdinal(ordinal) {
297
+ if (!holds(ordinal)) return void 0;
298
+ return { vector: view.vectorAt(ordinal), magnitude: magnitudes[ordinal] };
299
+ },
300
+ docIdAt(ordinal) {
301
+ if (ordinal < 0 || ordinal >= view.slots) return void 0;
302
+ if (ordinal + 1 >= docIdOffsets.length) rebind();
303
+ const start = docIdOffsets[ordinal];
304
+ const end = docIdOffsets[ordinal + 1];
305
+ if (end <= start) return void 0;
306
+ if (end > docIdBytes.length) rebind();
307
+ return decoder.decode(docIdBytes.subarray(start, end));
308
+ },
309
+ partitionAt(ordinal) {
310
+ if (ordinal < 0) return void 0;
311
+ if (ordinal >= partitions.length) {
312
+ if (ordinal * 4 >= handles.partitions.byteLength) return void 0;
313
+ rebind();
314
+ }
315
+ const partition = partitions[ordinal];
316
+ return partition < 0 ? void 0 : partition;
317
+ },
318
+ distanceByOrdinal(ordA, ordB, metric) {
319
+ if (!holds(ordA) || !holds(ordB)) return Number.POSITIVE_INFINITY;
320
+ if (!isCold(ordB)) {
321
+ const indexA = ordA / capacity | 0;
322
+ const indexB = ordB / capacity | 0;
323
+ const blockB = blockAt(indexB);
324
+ const localB = ordB % capacity;
325
+ if (blockB.simd !== null) {
326
+ if (indexA === indexB && !isCold(ordA)) {
327
+ return arenaFloat32Distance(
328
+ blockB.simd,
329
+ slotByteOffset(layout, ordA % capacity),
330
+ slotByteOffset(layout, localB),
331
+ dimension,
332
+ metric,
333
+ magnitudes[ordA],
334
+ magnitudes[ordB]
335
+ );
336
+ }
337
+ if (blockB.hasScratch && stageVector(blockB, ordA)) {
338
+ return arenaFloat32Distance(
339
+ blockB.simd,
340
+ blockB.scratchByteOffset,
341
+ slotByteOffset(layout, localB),
342
+ dimension,
343
+ metric,
344
+ magnitudes[ordA],
345
+ magnitudes[ordB]
346
+ );
347
+ }
348
+ }
349
+ }
350
+ const a = vectorFor(ordA, coldScratchA);
351
+ const b = vectorFor(ordB, coldScratchB);
352
+ if (a === null || b === null) return Number.POSITIVE_INFINITY;
353
+ return jsDistance(a, b, magnitudes[ordA], magnitudes[ordB], metric);
354
+ },
355
+ prepareQueryArena(query) {
356
+ if (query.length !== dimension || !view.simdAvailable) return null;
357
+ currentQuery = query;
358
+ for (const block of blocks) {
359
+ if (block !== void 0) block.stagedOrdinal = NOTHING_STAGED;
360
+ }
361
+ return { magnitude: magnitude(query) };
362
+ },
363
+ distanceFromArena(prepared, ordinal, metric) {
364
+ const state = holdState(ordinal);
365
+ if (state === NOT_HELD) return Number.POSITIVE_INFINITY;
366
+ if (currentQuery === null) return Number.POSITIVE_INFINITY;
367
+ if (state === HELD_COLD) {
368
+ if (!readColdInto(ordinal, coldScratchA)) return Number.POSITIVE_INFINITY;
369
+ return jsDistance(currentQuery, coldScratchA, prepared.magnitude, magnitudes[ordinal], metric);
370
+ }
371
+ const block = blockAt(ordinal / capacity | 0);
372
+ const local = ordinal % capacity;
373
+ if (block.simd !== null && block.hasScratch && stageQuery(block)) {
374
+ return arenaFloat32Distance(
375
+ block.simd,
376
+ block.scratchByteOffset,
377
+ slotByteOffset(layout, local),
378
+ dimension,
379
+ metric,
380
+ prepared.magnitude,
381
+ magnitudes[ordinal]
382
+ );
383
+ }
384
+ return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
385
+ },
386
+ hotBlockReads,
387
+ queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
388
+ ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
389
+ pairDistance: (metric) => pairDistanceOf(view, metric)
390
+ };
391
+ for (let index = 0; index < handles.blocks.length; index++) {
392
+ if (handles.blocks[index] !== null) blockAt(index);
393
+ }
394
+ for (let index = 0; index < handles.codeBlocks.length; index++) codeBlockAt(index);
395
+ return view;
396
+ }
397
+
398
+ // src/vector/shared-field/view.ts
399
+ function openSharedVectorField(initial, threadSlot) {
400
+ let handles = initial;
401
+ const store = openSharedVectorStore(initial.store, threadSlot);
402
+ const codeLayout = initial.store.codeLayout;
403
+ const quantizer = initial.quantization !== "none" && codeLayout !== null ? openSharedQuantizer(store, codeLayout, initial.metric) : void 0;
404
+ let graph = null;
405
+ let ordinals = null;
406
+ let scanned = 0;
407
+ let scannedLayoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
408
+ let placed = 0;
409
+ function openGraph(next) {
410
+ graph = next.graph === null ? null : openGraphState(next.graph, next.dimension, store, quantizer, threadSlot);
411
+ }
412
+ openGraph(initial);
413
+ function docIdOf(ordinal) {
414
+ if (!store.holdsOrdinal(ordinal)) return void 0;
415
+ return store.docIdAt(ordinal);
416
+ }
417
+ function ordinalOf(docId) {
418
+ const layoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
419
+ if (layoutGeneration !== scannedLayoutGeneration) {
420
+ ordinals = null;
421
+ scanned = 0;
422
+ scannedLayoutGeneration = layoutGeneration;
423
+ }
424
+ const slots = store.slots;
425
+ if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
426
+ for (; scanned < slots; scanned++) {
427
+ const id = store.docIdAt(scanned);
428
+ if (id !== void 0) ordinals.set(id, scanned);
429
+ }
430
+ const ordinal = ordinals.get(docId);
431
+ return ordinal !== void 0 && store.holdsOrdinal(ordinal) ? ordinal : void 0;
432
+ }
433
+ return {
434
+ get handles() {
435
+ return handles;
436
+ },
437
+ store,
438
+ quantizer,
439
+ get graph() {
440
+ return graph;
441
+ },
442
+ get liveSize() {
443
+ return graph === null ? 0 : nodeCountOf(graph) - tombstoneCountOf(graph);
444
+ },
445
+ get readsEveryVector() {
446
+ return store.holdsEveryBlock;
447
+ },
448
+ get graphRetired() {
449
+ return graph !== null && Atomics.load(graph.header, GRAPH_RETIRED) === 1;
450
+ },
451
+ docIdOf,
452
+ ordinalOf,
453
+ vectorOf(docId) {
454
+ const ordinal = ordinalOf(docId);
455
+ return ordinal === void 0 ? void 0 : store.vectorAt(ordinal);
456
+ },
457
+ adopt(next) {
458
+ store.adoptHandles(next.store);
459
+ if (next.graph?.header !== handles.graph?.header) openGraph(next);
460
+ handles = next;
461
+ },
462
+ close() {
463
+ store.close();
464
+ },
465
+ insertOrdinal(ordinal) {
466
+ if (graph === null) return false;
467
+ const fresh = insertNode(graph, ordinal);
468
+ if (fresh) placed += 1;
469
+ return fresh;
470
+ },
471
+ takeOutcome() {
472
+ const outcome = { placed };
473
+ placed = 0;
474
+ return outcome;
475
+ },
476
+ searchOrdinals(query, k, metric, minSimilarity, options) {
477
+ if (graph === null) return [];
478
+ return searchOrdinals(graph, docIdOf, query, k, metric, minSimilarity, options);
479
+ }
480
+ };
481
+ }
482
+
483
+ // src/vector/vector-store/doc-ids.ts
484
+ var encoder = new TextEncoder();
485
+ function appendDocId(handles, ordinal, docId) {
486
+ const bytes = encoder.encode(docId);
487
+ const start = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
488
+ const end = start + bytes.length;
489
+ growBufferTo(handles.docIdBytes, end);
490
+ growBufferTo(handles.docIdOffsets, (ordinal + 2) * 4);
491
+ new Uint8Array(handles.docIdBytes).set(bytes, start);
492
+ const offsets = new Uint32Array(handles.docIdOffsets);
493
+ offsets[ordinal] = start;
494
+ offsets[ordinal + 1] = end;
495
+ Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
496
+ }
497
+ function resetDocIds(handles) {
498
+ new Uint32Array(handles.docIdOffsets).fill(0);
499
+ Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
500
+ }
501
+
502
+ // src/vector/vector-store/ordinal-reads.ts
503
+ function entryAt(store, ordinal) {
504
+ return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
505
+ }
506
+ function createOrdinalReads(openStore, readable) {
507
+ function pairDistance(metric) {
508
+ const open = openStore();
509
+ if (open === null) return () => Number.POSITIVE_INFINITY;
510
+ const inner = open.view.pairDistance(metric);
511
+ return (ordA, ordB) => readable(ordA) && readable(ordB) ? inner(ordA, ordB) : Number.POSITIVE_INFINITY;
512
+ }
513
+ return {
514
+ entryForOrdinal(ordinal) {
515
+ const open = openStore();
516
+ if (open === null || !readable(ordinal)) return void 0;
517
+ return entryAt(open, ordinal);
518
+ },
519
+ distanceByOrdinal(ordA, ordB, metric) {
520
+ const open = openStore();
521
+ if (open === null || !readable(ordA) || !readable(ordB)) return Number.POSITIVE_INFINITY;
522
+ return open.view.distanceByOrdinal(ordA, ordB, metric);
523
+ },
524
+ distanceFromArena(prepared, ordinal, metric) {
525
+ const open = openStore();
526
+ if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
527
+ return open.view.distanceFromArena(prepared, ordinal, metric);
528
+ },
529
+ queryDistance(prepared, metric) {
530
+ const open = openStore();
531
+ if (open === null) return () => Number.POSITIVE_INFINITY;
532
+ const inner = open.view.queryDistance(prepared, metric);
533
+ return (ordinal) => readable(ordinal) ? inner(ordinal) : Number.POSITIVE_INFINITY;
534
+ },
535
+ ordinalDistance(from, metric) {
536
+ const pair = pairDistance(metric);
537
+ return (ordinal) => pair(from, ordinal);
538
+ },
539
+ pairDistance
540
+ };
541
+ }
542
+
543
+ // src/vector/vector-store/listing.ts
544
+ function* entriesOf(open, documents) {
545
+ if (open === null) return;
546
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
547
+ const docId = documents[ordinal];
548
+ if (docId === void 0) continue;
549
+ yield [docId, entryAt(open, ordinal)];
550
+ }
551
+ }
552
+ function* docIdsOf(open, documents) {
553
+ if (open === null) return;
554
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
555
+ const docId = documents[ordinal];
556
+ if (docId !== void 0) yield docId;
557
+ }
558
+ }
559
+ function snapshotOf(open, documents, replacedOrdinals) {
560
+ const slots = documents.length;
561
+ const dimension = open === null ? 0 : open.handles.dimension;
562
+ const vectors = new Float32Array(slots * dimension);
563
+ const magnitudes = new Float64Array(slots);
564
+ const docIds = new Array(slots);
565
+ for (let ordinal = 0; ordinal < slots; ordinal++) {
566
+ docIds[ordinal] = documents[ordinal] ?? null;
567
+ if (open === null || documents[ordinal] === void 0 && !replacedOrdinals.has(ordinal)) continue;
568
+ vectors.set(open.view.vectorAt(ordinal), ordinal * dimension);
569
+ magnitudes[ordinal] = open.magnitudes[ordinal];
570
+ }
571
+ return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
572
+ }
573
+
574
+ // src/vector/vector-store/open-store.ts
575
+ function openHandles(dimension, codeBits, blockBytes) {
576
+ const handles = createSharedVectorStoreHandles(dimension, codeBits, blockBytes);
577
+ return {
578
+ handles,
579
+ view: openSharedVectorStore(handles, 0),
580
+ magnitudes: new Float64Array(handles.magnitudes),
581
+ present: new Uint8Array(handles.present),
582
+ partitions: new Int32Array(handles.partitions),
583
+ diskFile: new Int32Array(handles.diskFile),
584
+ diskOffset: new Uint32Array(handles.diskOffset)
585
+ };
586
+ }
587
+ function relayout(store) {
588
+ store.handles.layoutRevision += 1;
589
+ store.view.adoptHandles(store.handles);
590
+ }
591
+ function claimSlot(store, docId, cold) {
592
+ const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
593
+ ensureOrdinal(store, ordinal, cold);
594
+ store.diskFile[ordinal] = IN_MEMORY;
595
+ appendDocId(store.handles, ordinal, docId);
596
+ return ordinal;
597
+ }
598
+ function publishOrdinal(store, ordinal) {
599
+ Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
600
+ Atomics.store(store.present, ordinal, 1);
601
+ }
602
+ function publishOnDisk(store, ordinal, vectorMagnitude, location) {
603
+ store.magnitudes[ordinal] = vectorMagnitude;
604
+ store.diskOffset[ordinal] = location.offset;
605
+ store.diskFile[ordinal] = location.fileIndex;
606
+ publishOrdinal(store, ordinal);
607
+ }
608
+ function relocateFiles(store, rewrite) {
609
+ const files = store.handles.vectorFiles;
610
+ if (files.length === 0) return;
611
+ for (let file = 0; file < files.length; file++) {
612
+ if (files[file].length > 0) files[file] = rewrite(files[file]);
613
+ }
614
+ relayout(store);
615
+ }
616
+ function ensureCodeSlot(store, ordinal) {
617
+ const { handles } = store;
618
+ if (handles.codeLayout === null) return;
619
+ const blockIndex = Math.floor(ordinal / handles.codeLayout.capacity);
620
+ while (handles.codeBlocks.length <= blockIndex) {
621
+ handles.codeBlocks.push(createVectorBlock(handles.codeLayout));
622
+ Atomics.store(handles.header, STORE_CODE_BLOCK_COUNT, handles.codeBlocks.length);
623
+ relayout(store);
624
+ }
625
+ ensureBlockSlots(handles.codeBlocks[blockIndex], ordinal % handles.codeLayout.capacity + 1);
626
+ }
627
+ function ensureFloatSlot(store, ordinal) {
628
+ const { handles } = store;
629
+ const blockIndex = Math.floor(ordinal / handles.layout.capacity);
630
+ let changed = false;
631
+ while (handles.blocks.length <= blockIndex) {
632
+ handles.blocks.push(null);
633
+ changed = true;
634
+ }
635
+ if (handles.blocks[blockIndex] === null) {
636
+ handles.blocks[blockIndex] = createVectorBlock(handles.layout);
637
+ changed = true;
638
+ }
639
+ if (changed) {
640
+ Atomics.store(handles.header, STORE_BLOCK_COUNT, handles.blocks.length);
641
+ relayout(store);
642
+ }
643
+ const block = handles.blocks[blockIndex];
644
+ if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
645
+ }
646
+ function ensureOrdinal(store, ordinal, cold) {
647
+ if (ordinal >= MAX_VECTOR_ORDINALS) {
648
+ throw new NarsilError(
649
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
650
+ `A vector field holds at most ${MAX_VECTOR_ORDINALS} vectors, deleted ones included, until it is rebuilt`,
651
+ { ordinal, maxOrdinals: MAX_VECTOR_ORDINALS }
652
+ );
653
+ }
654
+ const { handles } = store;
655
+ growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
656
+ growBufferTo(handles.present, ordinal + 1);
657
+ growBufferTo(handles.unplaced, ordinal + 1);
658
+ growBufferTo(handles.unplacedChunks, unplacedChunkCount(ordinal) * 4);
659
+ growBufferTo(handles.removed, ordinal + 1);
660
+ growBufferTo(handles.partitions, (ordinal + 1) * 4);
661
+ growBufferTo(handles.codePresent, ordinal + 1);
662
+ const diskFileBytes = handles.diskFile.byteLength;
663
+ growBufferTo(handles.diskFile, (ordinal + 1) * 4);
664
+ if (handles.diskFile.byteLength > diskFileBytes) {
665
+ new Int32Array(handles.diskFile).fill(IN_MEMORY, diskFileBytes / 4);
666
+ }
667
+ growBufferTo(handles.diskOffset, (ordinal + 1) * 4);
668
+ if (!cold) ensureFloatSlot(store, ordinal);
669
+ ensureCodeSlot(store, ordinal);
670
+ }
671
+ function writeVector(store, ordinal, vector) {
672
+ const block = store.view.blockOf(ordinal);
673
+ const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
674
+ block.float32(base + vector.length).set(vector, base);
675
+ store.magnitudes[ordinal] = magnitude(vector);
676
+ }
677
+ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
678
+ const capacity = store.handles.layout.capacity;
679
+ const slots = Atomics.load(store.handles.header, STORE_SLOTS);
680
+ const end = Math.min(slots, (blockIndex + 1) * capacity);
681
+ if (end < (blockIndex + 1) * capacity) return false;
682
+ for (let ordinal = blockIndex * capacity; ordinal < end; ordinal++) {
683
+ const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
684
+ if (held && store.diskFile[ordinal] === IN_MEMORY) return false;
685
+ }
686
+ return true;
687
+ }
688
+ function fileHoldsSameBytes(store, ordinal, location) {
689
+ const fromFile = new Float32Array(store.handles.dimension);
690
+ if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
691
+ const held = store.view.vectorAt(ordinal);
692
+ const a = new Uint8Array(fromFile.buffer);
693
+ const b = new Uint8Array(held.buffer, held.byteOffset, held.byteLength);
694
+ for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
695
+ return true;
696
+ }
697
+ function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
698
+ const referenced = /* @__PURE__ */ new Set();
699
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
700
+ const held = documents[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
701
+ if (held && store.diskFile[ordinal] !== IN_MEMORY) {
702
+ referenced.add(store.diskFile[ordinal]);
703
+ }
704
+ }
705
+ let forgotten = 0;
706
+ for (let file = 0; file < store.handles.vectorFiles.length; file++) {
707
+ if (referenced.has(file) || store.handles.vectorFiles[file] === "") continue;
708
+ store.handles.vectorFiles[file] = "";
709
+ forgotten += 1;
710
+ }
711
+ return forgotten;
712
+ }
713
+ function partitionFilterOf(store, documents, partitionIds) {
714
+ const filter = createOrdinalFilter(documents.length);
715
+ if (store === null) return filter;
716
+ let highest = -1;
717
+ for (const partitionId of partitionIds) {
718
+ if (partitionId > highest) highest = partitionId;
719
+ }
720
+ if (highest < 0) return filter;
721
+ const wanted = new Uint8Array(highest + 1);
722
+ for (const partitionId of partitionIds) {
723
+ if (partitionId >= 0) wanted[partitionId] = 1;
724
+ }
725
+ for (let ordinal = 0; ordinal < documents.length; ordinal++) {
726
+ if (documents[ordinal] === void 0) continue;
727
+ const partitionId = store.partitions[ordinal];
728
+ if (partitionId < 0 || partitionId > highest || wanted[partitionId] === 0) continue;
729
+ addToOrdinalFilter(filter, ordinal);
730
+ }
731
+ return filter;
732
+ }
733
+
734
+ // src/vector/vector-store/index.ts
735
+ var UNKNOWN_PARTITION = -1;
736
+ var NO_PARTITION = -2;
737
+ function createVectorStore(options) {
738
+ const codeBits = options?.codeBits ?? null;
739
+ const blockBytes = options?.blockBytes ?? BLOCK_MAX_BYTES;
740
+ const docToOrd = /* @__PURE__ */ new Map();
741
+ const ordToDoc = [];
742
+ const replacedOrdinals = /* @__PURE__ */ new Set();
743
+ let unknownPartitions = 0;
744
+ let liveCount = 0;
745
+ let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
746
+ function forgetDocuments() {
747
+ docToOrd.clear();
748
+ ordToDoc.length = 0;
749
+ replacedOrdinals.clear();
750
+ unknownPartitions = 0;
751
+ liveCount = 0;
752
+ }
753
+ function readable(ordinal) {
754
+ return ordToDoc[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
755
+ }
756
+ function requireOpen() {
757
+ if (open === null) {
758
+ throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
759
+ }
760
+ return open;
761
+ }
762
+ function recordPartition(store, ordinal, partitionId) {
763
+ const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
764
+ store.partitions[ordinal] = given;
765
+ if (given === UNKNOWN_PARTITION) unknownPartitions += 1;
766
+ }
767
+ function retireOrdinal(store, ordinal) {
768
+ ordToDoc[ordinal] = void 0;
769
+ Atomics.store(store.present, ordinal, 0);
770
+ if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
771
+ }
772
+ function keepReplacedOrdinal(store, ordinal) {
773
+ ordToDoc[ordinal] = void 0;
774
+ if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
775
+ store.partitions[ordinal] = NO_PARTITION;
776
+ replacedOrdinals.add(ordinal);
777
+ }
778
+ function restoreReplacedVector(store, ordinal, vector) {
779
+ store.diskFile[ordinal] = IN_MEMORY;
780
+ writeVector(store, ordinal, vector);
781
+ Atomics.store(store.present, ordinal, 1);
782
+ replacedOrdinals.add(ordinal);
783
+ }
784
+ function claimOrdinal(store, docId, cold, partitionId) {
785
+ const ordinal = claimSlot(store, docId, cold);
786
+ recordPartition(store, ordinal, partitionId);
787
+ ordToDoc[ordinal] = docId;
788
+ docToOrd.set(docId, ordinal);
789
+ return ordinal;
790
+ }
791
+ function appendOrdinal(store, docId, vector, partitionId) {
792
+ const ordinal = claimOrdinal(store, docId, false, partitionId);
793
+ writeVector(store, ordinal, vector);
794
+ publishOrdinal(store, ordinal);
795
+ return ordinal;
796
+ }
797
+ function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
798
+ if (previous !== void 0) {
799
+ if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
800
+ else retireOrdinal(store, previous);
801
+ return ordinal;
802
+ }
803
+ liveCount++;
804
+ Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
805
+ return ordinal;
806
+ }
807
+ return {
808
+ ...createOrdinalReads(() => open, readable),
809
+ get size() {
810
+ return liveCount;
811
+ },
812
+ get slots() {
813
+ return ordToDoc.length;
814
+ },
815
+ get dimension() {
816
+ return open === null ? 0 : open.handles.dimension;
817
+ },
818
+ get codeBits() {
819
+ return codeBits;
820
+ },
821
+ get handles() {
822
+ return requireOpen().handles;
823
+ },
824
+ get view() {
825
+ return requireOpen().view;
826
+ },
827
+ get replacedVectorCount() {
828
+ return replacedOrdinals.size;
829
+ },
830
+ insert(docId, vector, partitionId, keepsReplacedVector = false) {
831
+ if (open === null) open = openHandles(vector.length, codeBits, blockBytes);
832
+ const store = open;
833
+ if (vector.length !== store.handles.dimension) {
834
+ throw new NarsilError(
835
+ ErrorCodes.VECTOR_DIMENSION_MISMATCH,
836
+ `Vector dimension mismatch: expected ${store.handles.dimension}, got ${vector.length}`,
837
+ { expected: store.handles.dimension, received: vector.length }
838
+ );
839
+ }
840
+ const previous = docToOrd.get(docId);
841
+ const ordinal = appendOrdinal(store, docId, vector, partitionId);
842
+ return settleReplacement(store, ordinal, previous, keepsReplacedVector);
843
+ },
844
+ releaseReplacedVectors() {
845
+ const released = [...replacedOrdinals];
846
+ replacedOrdinals.clear();
847
+ if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
848
+ return released;
849
+ },
850
+ insertCold(docId, vectorMagnitude, location, partitionId) {
851
+ const store = requireOpen();
852
+ const previous = docToOrd.get(docId);
853
+ Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
854
+ const ordinal = claimOrdinal(store, docId, true, partitionId);
855
+ publishOnDisk(store, ordinal, vectorMagnitude, location);
856
+ return settleReplacement(store, ordinal, previous, false);
857
+ },
858
+ insertReplaced(docId, vector, location) {
859
+ if (open === null) open = openHandles(vector.length, codeBits, blockBytes);
860
+ const store = open;
861
+ if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
862
+ const ordinal = claimSlot(store, docId, location !== void 0);
863
+ store.partitions[ordinal] = NO_PARTITION;
864
+ ordToDoc[ordinal] = void 0;
865
+ replacedOrdinals.add(ordinal);
866
+ if (location !== void 0) {
867
+ publishOnDisk(store, ordinal, magnitude(vector), location);
868
+ return ordinal;
869
+ }
870
+ writeVector(store, ordinal, vector);
871
+ publishOrdinal(store, ordinal);
872
+ return ordinal;
873
+ },
874
+ addVectorFile(path) {
875
+ const store = requireOpen();
876
+ const existing = store.handles.vectorFiles.indexOf(path);
877
+ if (existing !== -1) return existing;
878
+ store.handles.vectorFiles.push(path);
879
+ relayout(store);
880
+ return store.handles.vectorFiles.length - 1;
881
+ },
882
+ isCold(ordinal) {
883
+ if (open === null) return false;
884
+ return open.view.isCold(ordinal);
885
+ },
886
+ releaseToFile(ordinal, location) {
887
+ const store = requireOpen();
888
+ if (ordToDoc[ordinal] === void 0) return false;
889
+ if (store.diskFile[ordinal] === IN_MEMORY && !fileHoldsSameBytes(store, ordinal, location)) return false;
890
+ Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
891
+ store.diskOffset[ordinal] = location.offset;
892
+ store.diskFile[ordinal] = location.fileIndex;
893
+ return true;
894
+ },
895
+ releaseColdBlocks() {
896
+ const store = requireOpen();
897
+ let released = 0;
898
+ for (let index = 0; index < store.handles.blocks.length; index++) {
899
+ if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, replacedOrdinals)) continue;
900
+ store.handles.blocks[index] = null;
901
+ released += 1;
902
+ }
903
+ if (released + forgetUnreferencedFiles(store, ordToDoc, replacedOrdinals) > 0) relayout(store);
904
+ return released;
905
+ },
906
+ setPartition(docId, partitionId) {
907
+ const ordinal = docToOrd.get(docId);
908
+ if (ordinal === void 0 || open === null || partitionId < 0) return;
909
+ if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
910
+ open.partitions[ordinal] = partitionId;
911
+ },
912
+ forgetPartition(docId) {
913
+ const ordinal = docToOrd.get(docId);
914
+ if (ordinal === void 0 || open === null) return;
915
+ if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
916
+ open.partitions[ordinal] = NO_PARTITION;
917
+ },
918
+ partitionOfOrdinal(ordinal) {
919
+ if (open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
920
+ const partitionId = open.partitions[ordinal];
921
+ return partitionId < 0 ? void 0 : partitionId;
922
+ },
923
+ get partitionsKnown() {
924
+ return unknownPartitions === 0;
925
+ },
926
+ partitionFilter(partitionIds) {
927
+ return partitionFilterOf(open, ordToDoc, partitionIds);
928
+ },
929
+ remove(docId) {
930
+ const ordinal = docToOrd.get(docId);
931
+ if (ordinal === void 0 || open === null) return;
932
+ docToOrd.delete(docId);
933
+ retireOrdinal(open, ordinal);
934
+ liveCount--;
935
+ Atomics.sub(open.handles.header, STORE_LIVE_COUNT, 1);
936
+ },
937
+ get(docId) {
938
+ const ordinal = docToOrd.get(docId);
939
+ return ordinal === void 0 || open === null ? void 0 : entryAt(open, ordinal);
940
+ },
941
+ has(docId) {
942
+ return docToOrd.has(docId);
943
+ },
944
+ holdsOrdinal(ordinal) {
945
+ return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
946
+ },
947
+ entries: () => entriesOf(open, ordToDoc),
948
+ docIds: () => docIdsOf(open, ordToDoc),
949
+ clear() {
950
+ forgetDocuments();
951
+ if (open === null) return;
952
+ open.present.fill(0);
953
+ new Uint8Array(open.handles.codePresent).fill(0);
954
+ new Int32Array(open.handles.diskFile).fill(IN_MEMORY);
955
+ open.handles.centroid.fill(0);
956
+ if (open.handles.vectorFiles.length > 0) {
957
+ open.handles.vectorFiles.length = 0;
958
+ relayout(open);
959
+ }
960
+ resetDocIds(open.handles);
961
+ resetStoreHeader(open.handles.header);
962
+ },
963
+ release() {
964
+ forgetDocuments();
965
+ if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
966
+ open?.view.close();
967
+ open = null;
968
+ },
969
+ closeFiles: () => open?.view.close(),
970
+ relocateVectorFiles(rewrite) {
971
+ if (open !== null) relocateFiles(open, rewrite);
972
+ },
973
+ getOrdinal(docId) {
974
+ return docToOrd.get(docId);
975
+ },
976
+ docIdForOrdinal(ordinal) {
977
+ return ordToDoc[ordinal];
978
+ },
979
+ prepareQueryArena(query) {
980
+ return open === null ? null : open.view.prepareQueryArena(query);
981
+ },
982
+ exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
983
+ restoreSnapshot(snapshot) {
984
+ this.clear();
985
+ if (snapshot.dimension === 0 || snapshot.slots === 0) return;
986
+ if (open === null) open = openHandles(snapshot.dimension, codeBits, blockBytes);
987
+ const store = open;
988
+ const replaced = new Set(snapshot.replacedOrdinals);
989
+ for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
990
+ const docId = snapshot.docIds[ordinal];
991
+ const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
992
+ if (docId === null || docId === void 0) {
993
+ ensureOrdinal(store, ordinal, false);
994
+ ordToDoc[ordinal] = void 0;
995
+ store.partitions[ordinal] = NO_PARTITION;
996
+ if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
997
+ Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
998
+ continue;
999
+ }
1000
+ appendOrdinal(store, docId, vector);
1001
+ liveCount++;
1002
+ }
1003
+ Atomics.store(store.handles.header, STORE_LIVE_COUNT, liveCount);
1004
+ },
1005
+ memoryBytes() {
1006
+ return open === null ? 0 : sharedVectorStoreBytes(open.handles);
1007
+ }
1008
+ };
1009
+ }
1010
+
1011
+ export { createVectorStore, openSharedVectorField };