@cmdoss/memwal-sdk 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +547 -547
- package/BENCHMARKS.md +238 -238
- package/README.md +310 -181
- package/dist/ai-sdk/tools.d.ts +2 -2
- package/dist/ai-sdk/tools.js +2 -2
- package/dist/client/ClientMemoryManager.js +2 -2
- package/dist/client/ClientMemoryManager.js.map +1 -1
- package/dist/client/PersonalDataWallet.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.d.ts +29 -1
- package/dist/client/SimplePDWClient.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.js +45 -13
- package/dist/client/SimplePDWClient.js.map +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.d.ts +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.js +1 -1
- package/dist/client/namespaces/MemoryNamespace.d.ts +31 -0
- package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/MemoryNamespace.js +272 -39
- package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/AINamespace.d.ts +2 -2
- package/dist/client/namespaces/consolidated/AINamespace.js +2 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts +12 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js +62 -4
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts +67 -2
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.js +549 -16
- package/dist/client/namespaces/consolidated/StorageNamespace.js.map +1 -1
- package/dist/config/ConfigurationHelper.js +61 -61
- package/dist/config/defaults.js +2 -2
- package/dist/config/defaults.js.map +1 -1
- package/dist/graph/GraphService.js +21 -21
- package/dist/graph/GraphService.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/seal/EncryptionService.d.ts +9 -5
- package/dist/infrastructure/seal/EncryptionService.d.ts.map +1 -1
- package/dist/infrastructure/seal/EncryptionService.js +37 -15
- package/dist/infrastructure/seal/EncryptionService.js.map +1 -1
- package/dist/infrastructure/seal/SealService.d.ts +13 -5
- package/dist/infrastructure/seal/SealService.d.ts.map +1 -1
- package/dist/infrastructure/seal/SealService.js +36 -34
- package/dist/infrastructure/seal/SealService.js.map +1 -1
- package/dist/langchain/createPDWRAG.js +30 -30
- package/dist/retrieval/MemoryDecryptionPipeline.d.ts.map +1 -1
- package/dist/retrieval/MemoryDecryptionPipeline.js +2 -1
- package/dist/retrieval/MemoryDecryptionPipeline.js.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.d.ts +31 -0
- package/dist/retrieval/MemoryRetrievalService.d.ts.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.js +44 -4
- package/dist/retrieval/MemoryRetrievalService.js.map +1 -1
- package/dist/services/CapabilityService.d.ts.map +1 -1
- package/dist/services/CapabilityService.js +30 -14
- package/dist/services/CapabilityService.js.map +1 -1
- package/dist/services/CrossContextPermissionService.d.ts.map +1 -1
- package/dist/services/CrossContextPermissionService.js +9 -7
- package/dist/services/CrossContextPermissionService.js.map +1 -1
- package/dist/services/EmbeddingService.d.ts +28 -1
- package/dist/services/EmbeddingService.d.ts.map +1 -1
- package/dist/services/EmbeddingService.js +54 -0
- package/dist/services/EmbeddingService.js.map +1 -1
- package/dist/services/EncryptionService.d.ts.map +1 -1
- package/dist/services/EncryptionService.js +6 -5
- package/dist/services/EncryptionService.js.map +1 -1
- package/dist/services/GeminiAIService.js +309 -309
- package/dist/services/IndexManager.d.ts +5 -1
- package/dist/services/IndexManager.d.ts.map +1 -1
- package/dist/services/IndexManager.js +17 -40
- package/dist/services/IndexManager.js.map +1 -1
- package/dist/services/QueryService.js +1 -1
- package/dist/services/QueryService.js.map +1 -1
- package/dist/services/StorageService.d.ts +11 -0
- package/dist/services/StorageService.d.ts.map +1 -1
- package/dist/services/StorageService.js +73 -10
- package/dist/services/StorageService.js.map +1 -1
- package/dist/services/TransactionService.d.ts +20 -0
- package/dist/services/TransactionService.d.ts.map +1 -1
- package/dist/services/TransactionService.js +43 -0
- package/dist/services/TransactionService.js.map +1 -1
- package/dist/services/ViewService.js +2 -2
- package/dist/services/ViewService.js.map +1 -1
- package/dist/services/storage/QuiltBatchManager.d.ts +101 -1
- package/dist/services/storage/QuiltBatchManager.d.ts.map +1 -1
- package/dist/services/storage/QuiltBatchManager.js +410 -20
- package/dist/services/storage/QuiltBatchManager.js.map +1 -1
- package/dist/services/storage/index.d.ts +1 -1
- package/dist/services/storage/index.d.ts.map +1 -1
- package/dist/services/storage/index.js.map +1 -1
- package/dist/utils/LRUCache.d.ts +106 -0
- package/dist/utils/LRUCache.d.ts.map +1 -0
- package/dist/utils/LRUCache.js +281 -0
- package/dist/utils/LRUCache.js.map +1 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/memoryIndexOnChain.d.ts +212 -0
- package/dist/utils/memoryIndexOnChain.d.ts.map +1 -0
- package/dist/utils/memoryIndexOnChain.js +312 -0
- package/dist/utils/memoryIndexOnChain.js.map +1 -0
- package/dist/utils/rebuildIndexNode.d.ts +29 -0
- package/dist/utils/rebuildIndexNode.d.ts.map +1 -1
- package/dist/utils/rebuildIndexNode.js +366 -98
- package/dist/utils/rebuildIndexNode.js.map +1 -1
- package/dist/vector/HnswWasmService.d.ts +20 -5
- package/dist/vector/HnswWasmService.d.ts.map +1 -1
- package/dist/vector/HnswWasmService.js +73 -40
- package/dist/vector/HnswWasmService.js.map +1 -1
- package/dist/vector/IHnswService.d.ts +10 -1
- package/dist/vector/IHnswService.d.ts.map +1 -1
- package/dist/vector/IHnswService.js.map +1 -1
- package/dist/vector/NodeHnswService.d.ts +16 -0
- package/dist/vector/NodeHnswService.d.ts.map +1 -1
- package/dist/vector/NodeHnswService.js +84 -5
- package/dist/vector/NodeHnswService.js.map +1 -1
- package/dist/vector/createHnswService.d.ts +1 -1
- package/dist/vector/createHnswService.js +1 -1
- package/dist/vector/index.d.ts +1 -1
- package/dist/vector/index.js +1 -1
- package/package.json +157 -157
- package/src/access/PermissionService.ts +635 -635
- package/src/aggregation/AggregationService.ts +389 -389
- package/src/ai-sdk/PDWVectorStore.ts +715 -715
- package/src/ai-sdk/index.ts +65 -65
- package/src/ai-sdk/tools.ts +460 -460
- package/src/ai-sdk/types.ts +404 -404
- package/src/batch/BatchManager.ts +597 -597
- package/src/batch/BatchingService.ts +429 -429
- package/src/batch/MemoryProcessingCache.ts +492 -492
- package/src/batch/index.ts +30 -30
- package/src/browser.ts +200 -200
- package/src/client/ClientMemoryManager.ts +987 -987
- package/src/client/PersonalDataWallet.ts +345 -345
- package/src/client/SimplePDWClient.ts +1289 -1222
- package/src/client/factory.ts +154 -154
- package/src/client/namespaces/AnalyticsNamespace.ts +377 -377
- package/src/client/namespaces/BatchNamespace.ts +356 -356
- package/src/client/namespaces/CacheNamespace.ts +123 -123
- package/src/client/namespaces/CapabilityNamespace.ts +217 -217
- package/src/client/namespaces/ClassifyNamespace.ts +169 -169
- package/src/client/namespaces/ContextNamespace.ts +297 -297
- package/src/client/namespaces/EmbeddingsNamespace.ts +99 -99
- package/src/client/namespaces/EncryptionNamespace.ts +221 -221
- package/src/client/namespaces/GraphNamespace.ts +468 -468
- package/src/client/namespaces/IndexNamespace.ts +361 -361
- package/src/client/namespaces/MemoryNamespace.ts +1422 -1135
- package/src/client/namespaces/PermissionsNamespace.ts +254 -254
- package/src/client/namespaces/PipelineNamespace.ts +220 -220
- package/src/client/namespaces/SearchNamespace.ts +1049 -1049
- package/src/client/namespaces/StorageNamespace.ts +458 -458
- package/src/client/namespaces/TxNamespace.ts +260 -260
- package/src/client/namespaces/WalletNamespace.ts +243 -243
- package/src/client/namespaces/consolidated/AINamespace.ts +449 -449
- package/src/client/namespaces/consolidated/BlockchainNamespace.ts +607 -546
- package/src/client/namespaces/consolidated/SecurityNamespace.ts +648 -648
- package/src/client/namespaces/consolidated/StorageNamespace.ts +1141 -497
- package/src/client/namespaces/consolidated/index.ts +39 -39
- package/src/client/signers/KeypairSigner.ts +108 -108
- package/src/client/signers/UnifiedSigner.ts +110 -110
- package/src/client/signers/WalletAdapterSigner.ts +159 -159
- package/src/client/signers/index.ts +26 -26
- package/src/config/ConfigurationHelper.ts +412 -412
- package/src/config/defaults.ts +51 -51
- package/src/config/index.ts +8 -8
- package/src/config/validation.ts +70 -70
- package/src/core/index.ts +14 -14
- package/src/core/interfaces/IService.ts +307 -307
- package/src/core/interfaces/index.ts +8 -8
- package/src/core/types/capability.ts +297 -297
- package/src/core/types/index.ts +870 -870
- package/src/core/types/wallet.ts +270 -270
- package/src/core/types.ts +9 -9
- package/src/core/wallet.ts +222 -222
- package/src/embedding/index.ts +19 -19
- package/src/embedding/types.ts +357 -357
- package/src/errors/index.ts +602 -602
- package/src/errors/recovery.ts +461 -461
- package/src/errors/validation.ts +567 -567
- package/src/generated/pdw/capability.ts +319 -319
- package/src/graph/GraphService.ts +887 -887
- package/src/graph/KnowledgeGraphManager.ts +728 -728
- package/src/graph/index.ts +25 -25
- package/src/index.ts +498 -474
- package/src/infrastructure/index.ts +22 -22
- package/src/infrastructure/seal/EncryptionService.ts +628 -603
- package/src/infrastructure/seal/SealService.ts +613 -615
- package/src/infrastructure/seal/index.ts +9 -9
- package/src/infrastructure/sui/BlockchainManager.ts +627 -627
- package/src/infrastructure/sui/SuiService.ts +888 -888
- package/src/infrastructure/sui/index.ts +9 -9
- package/src/infrastructure/walrus/StorageManager.ts +604 -604
- package/src/infrastructure/walrus/WalrusStorageService.ts +612 -612
- package/src/infrastructure/walrus/index.ts +9 -9
- package/src/langchain/PDWEmbeddings.ts +145 -145
- package/src/langchain/PDWVectorStore.ts +456 -456
- package/src/langchain/createPDWRAG.ts +303 -303
- package/src/langchain/index.ts +47 -47
- package/src/permissions/ConsentRepository.browser.ts +249 -249
- package/src/permissions/ConsentRepository.ts +364 -364
- package/src/pipeline/MemoryPipeline.ts +862 -862
- package/src/pipeline/PipelineManager.ts +683 -683
- package/src/pipeline/index.ts +26 -26
- package/src/retrieval/AdvancedSearchService.ts +629 -629
- package/src/retrieval/MemoryAnalyticsService.ts +711 -711
- package/src/retrieval/MemoryDecryptionPipeline.ts +825 -824
- package/src/retrieval/MemoryRetrievalService.ts +904 -830
- package/src/retrieval/index.ts +42 -42
- package/src/services/BatchService.ts +352 -352
- package/src/services/CapabilityService.ts +464 -448
- package/src/services/ClassifierService.ts +465 -465
- package/src/services/CrossContextPermissionService.ts +486 -484
- package/src/services/EmbeddingService.ts +771 -706
- package/src/services/EncryptionService.ts +712 -711
- package/src/services/GeminiAIService.ts +753 -753
- package/src/services/IndexManager.ts +977 -1004
- package/src/services/MemoryIndexService.ts +1003 -1003
- package/src/services/MemoryService.ts +369 -369
- package/src/services/QueryService.ts +890 -890
- package/src/services/StorageService.ts +1182 -1111
- package/src/services/TransactionService.ts +838 -790
- package/src/services/VectorService.ts +462 -462
- package/src/services/ViewService.ts +484 -484
- package/src/services/index.ts +25 -25
- package/src/services/storage/BlobAttributesManager.ts +333 -333
- package/src/services/storage/KnowledgeGraphManager.ts +425 -425
- package/src/services/storage/MemorySearchManager.ts +387 -387
- package/src/services/storage/QuiltBatchManager.ts +1130 -660
- package/src/services/storage/WalrusMetadataManager.ts +268 -268
- package/src/services/storage/WalrusStorageManager.ts +287 -287
- package/src/services/storage/index.ts +57 -52
- package/src/types/index.ts +13 -13
- package/src/utils/LRUCache.ts +378 -0
- package/src/utils/index.ts +76 -68
- package/src/utils/memoryIndexOnChain.ts +507 -0
- package/src/utils/rebuildIndex.ts +290 -290
- package/src/utils/rebuildIndexNode.ts +771 -424
- package/src/vector/BrowserHnswIndexService.ts +758 -758
- package/src/vector/HnswWasmService.ts +731 -679
- package/src/vector/IHnswService.ts +233 -224
- package/src/vector/NodeHnswService.ts +833 -735
- package/src/vector/VectorManager.ts +478 -478
- package/src/vector/createHnswService.ts +135 -135
- package/src/vector/index.ts +56 -56
- package/src/wallet/ContextWalletService.ts +656 -656
- package/src/wallet/MainWalletService.ts +317 -317
|
@@ -1,679 +1,731 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HnswWasmService - Browser-Compatible HNSW Vector Indexing
|
|
3
|
-
*
|
|
4
|
-
* Provides browser-compatible Hierarchical Navigable Small World (HNSW) vector indexing
|
|
5
|
-
* using hnswlib-wasm with IndexedDB persistence. Replaces Node.js-only hnswlib-node.
|
|
6
|
-
*
|
|
7
|
-
* Key Features:
|
|
8
|
-
* - ✅ Runs in browsers (WebAssembly)
|
|
9
|
-
* - ✅ IndexedDB persistence (no filesystem needed)
|
|
10
|
-
* - ✅ Intelligent batching and caching
|
|
11
|
-
* - ✅ Walrus storage integration
|
|
12
|
-
* - ✅ Near-native performance via WASM
|
|
13
|
-
* - ✅ Safe for Node.js/SSR (uses dynamic import)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
//
|
|
18
|
-
|
|
19
|
-
type
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
import {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
throw
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
*
|
|
181
|
-
*/
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
//
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
const
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
cacheEntry
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
this.
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
|
|
1
|
+
/**
|
|
2
|
+
* HnswWasmService - Browser-Compatible HNSW Vector Indexing
|
|
3
|
+
*
|
|
4
|
+
* Provides browser-compatible Hierarchical Navigable Small World (HNSW) vector indexing
|
|
5
|
+
* using hnswlib-wasm with IndexedDB persistence. Replaces Node.js-only hnswlib-node.
|
|
6
|
+
*
|
|
7
|
+
* Key Features:
|
|
8
|
+
* - ✅ Runs in browsers (WebAssembly)
|
|
9
|
+
* - ✅ IndexedDB persistence (no filesystem needed)
|
|
10
|
+
* - ✅ Intelligent batching and caching
|
|
11
|
+
* - ✅ Walrus storage integration
|
|
12
|
+
* - ✅ Near-native performance via WASM
|
|
13
|
+
* - ✅ Safe for Node.js/SSR (uses dynamic import)
|
|
14
|
+
* - ✅ LRU cache with memory limits to prevent OOM
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
// Dynamic import for hnswlib-wasm to avoid bundling issues in Node.js
|
|
18
|
+
// Types defined locally to avoid static import issues
|
|
19
|
+
type HierarchicalNSW = any;
|
|
20
|
+
type HnswlibModule = any;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Helper to dynamically load hnswlib-wasm (browser only)
|
|
24
|
+
*/
|
|
25
|
+
async function loadHnswlibDynamic(): Promise<HnswlibModule> {
|
|
26
|
+
const module = await import('hnswlib-wasm/dist/hnswlib.js');
|
|
27
|
+
return module.loadHnswlib();
|
|
28
|
+
}
|
|
29
|
+
import { StorageService, type MemoryMetadata } from '../services/StorageService';
|
|
30
|
+
import {
|
|
31
|
+
HNSWIndexConfig,
|
|
32
|
+
HNSWSearchResult,
|
|
33
|
+
HNSWSearchOptions,
|
|
34
|
+
BatchConfig,
|
|
35
|
+
BatchJob,
|
|
36
|
+
BatchStats,
|
|
37
|
+
VectorError
|
|
38
|
+
} from '../embedding/types';
|
|
39
|
+
import { LRUCache, estimateIndexCacheSize } from '../utils/LRUCache';
|
|
40
|
+
|
|
41
|
+
interface IndexCacheEntry {
|
|
42
|
+
index: HierarchicalNSW;
|
|
43
|
+
lastModified: Date;
|
|
44
|
+
pendingVectors: Map<number, number[]>; // vectorId -> vector
|
|
45
|
+
isDirty: boolean;
|
|
46
|
+
version: number;
|
|
47
|
+
metadata: Map<number, any>; // vectorId -> metadata
|
|
48
|
+
dimensions: number;
|
|
49
|
+
/** Cached vectors for serialization - only store if needed */
|
|
50
|
+
vectors: Map<number, number[]>;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface IndexMetadata {
|
|
54
|
+
dimension: number;
|
|
55
|
+
maxElements: number;
|
|
56
|
+
efConstruction: number;
|
|
57
|
+
m: number;
|
|
58
|
+
spaceType: string;
|
|
59
|
+
version: number;
|
|
60
|
+
createdAt: Date;
|
|
61
|
+
lastUpdated: Date;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Memory management constants
|
|
65
|
+
const DEFAULT_MAX_CACHED_INDEXES = 5; // Max number of user indexes to keep in memory
|
|
66
|
+
const DEFAULT_INDEX_TTL_MS = 10 * 60 * 1000; // 10 minutes TTL for idle indexes
|
|
67
|
+
const DEFAULT_MAX_MEMORY_MB = 512; // 512MB max memory for index cache
|
|
68
|
+
const DEFAULT_CLEANUP_INTERVAL_MS = 60 * 1000; // Check every 1 minute
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Browser-compatible HNSW vector indexing service using WebAssembly
|
|
72
|
+
* Drop-in replacement for HnswIndexService with identical API
|
|
73
|
+
*
|
|
74
|
+
* Memory Management:
|
|
75
|
+
* - LRU cache limits number of indexes in memory (default: 5)
|
|
76
|
+
* - TTL-based expiration for idle indexes (default: 10 minutes)
|
|
77
|
+
* - Optional memory limit (default: 512MB)
|
|
78
|
+
* - Automatic cleanup of expired/evicted indexes
|
|
79
|
+
*/
|
|
80
|
+
export class HnswWasmService {
|
|
81
|
+
private hnswlib: HnswlibModule | null = null;
|
|
82
|
+
private readonly indexCache: LRUCache<IndexCacheEntry>;
|
|
83
|
+
private readonly batchJobs = new Map<string, BatchJob>();
|
|
84
|
+
private readonly config: Required<BatchConfig>;
|
|
85
|
+
private readonly indexConfig: Required<HNSWIndexConfig>;
|
|
86
|
+
private batchProcessor?: ReturnType<typeof setInterval>;
|
|
87
|
+
private initPromise: Promise<void> | null = null;
|
|
88
|
+
|
|
89
|
+
// Memory management settings
|
|
90
|
+
private readonly maxCachedIndexes: number;
|
|
91
|
+
private readonly indexTtlMs: number;
|
|
92
|
+
private readonly maxMemoryBytes: number;
|
|
93
|
+
|
|
94
|
+
constructor(
|
|
95
|
+
private storageService: StorageService,
|
|
96
|
+
indexConfig: Partial<HNSWIndexConfig> = {},
|
|
97
|
+
batchConfig: Partial<BatchConfig> = {},
|
|
98
|
+
memoryConfig?: {
|
|
99
|
+
maxCachedIndexes?: number;
|
|
100
|
+
indexTtlMs?: number;
|
|
101
|
+
maxMemoryMB?: number;
|
|
102
|
+
}
|
|
103
|
+
) {
|
|
104
|
+
// Default HNSW configuration (matching HnswIndexService)
|
|
105
|
+
this.indexConfig = {
|
|
106
|
+
dimension: indexConfig.dimension || 3072,
|
|
107
|
+
maxElements: indexConfig.maxElements || 10000,
|
|
108
|
+
efConstruction: indexConfig.efConstruction || 200,
|
|
109
|
+
m: indexConfig.m || 16,
|
|
110
|
+
randomSeed: indexConfig.randomSeed || 42,
|
|
111
|
+
spaceType: indexConfig.spaceType || 'cosine'
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
// Default batch configuration
|
|
115
|
+
this.config = {
|
|
116
|
+
maxBatchSize: batchConfig.maxBatchSize || 50,
|
|
117
|
+
batchDelayMs: batchConfig.batchDelayMs || 5000,
|
|
118
|
+
maxCacheSize: batchConfig.maxCacheSize || 100,
|
|
119
|
+
cacheTtlMs: batchConfig.cacheTtlMs || 30 * 60 * 1000 // 30 minutes
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
// Memory management configuration
|
|
123
|
+
this.maxCachedIndexes = memoryConfig?.maxCachedIndexes ?? DEFAULT_MAX_CACHED_INDEXES;
|
|
124
|
+
this.indexTtlMs = memoryConfig?.indexTtlMs ?? DEFAULT_INDEX_TTL_MS;
|
|
125
|
+
this.maxMemoryBytes = (memoryConfig?.maxMemoryMB ?? DEFAULT_MAX_MEMORY_MB) * 1024 * 1024;
|
|
126
|
+
|
|
127
|
+
// Initialize LRU cache with memory limits
|
|
128
|
+
this.indexCache = new LRUCache<IndexCacheEntry>({
|
|
129
|
+
maxSize: this.maxCachedIndexes,
|
|
130
|
+
ttlMs: this.indexTtlMs,
|
|
131
|
+
cleanupIntervalMs: DEFAULT_CLEANUP_INTERVAL_MS,
|
|
132
|
+
maxMemoryBytes: this.maxMemoryBytes,
|
|
133
|
+
sizeEstimator: (entry) => estimateIndexCacheSize({
|
|
134
|
+
vectors: entry.vectors || new Map(),
|
|
135
|
+
metadata: entry.metadata,
|
|
136
|
+
pendingVectors: entry.pendingVectors,
|
|
137
|
+
}),
|
|
138
|
+
onEvict: (userAddress, entry, reason) => {
|
|
139
|
+
console.log(`🧹 [HnswWasmService] Evicting index for ${userAddress} (reason: ${reason})`);
|
|
140
|
+
// Dispose WASM resources
|
|
141
|
+
if (entry.index) {
|
|
142
|
+
try {
|
|
143
|
+
if (typeof entry.index.free === 'function') {
|
|
144
|
+
entry.index.free();
|
|
145
|
+
}
|
|
146
|
+
} catch (e) {
|
|
147
|
+
// Ignore cleanup errors
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
// Remove associated batch job
|
|
151
|
+
this.batchJobs.delete(userAddress);
|
|
152
|
+
},
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
console.log(`✅ HnswWasmService initialized with memory limits:`);
|
|
156
|
+
console.log(` Max indexes: ${this.maxCachedIndexes}, TTL: ${this.indexTtlMs / 1000}s, Max memory: ${this.maxMemoryBytes / 1024 / 1024}MB`);
|
|
157
|
+
|
|
158
|
+
// Initialize WASM library asynchronously
|
|
159
|
+
this.initPromise = this.initialize();
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Initialize hnswlib-wasm (must be called before use)
|
|
164
|
+
*/
|
|
165
|
+
private async initialize(): Promise<void> {
|
|
166
|
+
try {
|
|
167
|
+
console.log('🔧 Loading hnswlib-wasm...');
|
|
168
|
+
this.hnswlib = await loadHnswlibDynamic();
|
|
169
|
+
console.log('✅ hnswlib-wasm loaded successfully');
|
|
170
|
+
|
|
171
|
+
// Start batch processor (cache cleanup is handled by LRUCache)
|
|
172
|
+
this.startBatchProcessor();
|
|
173
|
+
} catch (error) {
|
|
174
|
+
console.error('❌ Failed to load hnswlib-wasm:', error);
|
|
175
|
+
throw error;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Ensure WASM library is loaded
|
|
181
|
+
*/
|
|
182
|
+
private async ensureInitialized(): Promise<void> {
|
|
183
|
+
if (this.initPromise) {
|
|
184
|
+
await this.initPromise;
|
|
185
|
+
}
|
|
186
|
+
if (!this.hnswlib) {
|
|
187
|
+
throw new Error('hnswlib-wasm not initialized');
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Create a new HNSW index
|
|
193
|
+
*/
|
|
194
|
+
async createIndex(
|
|
195
|
+
userAddress: string,
|
|
196
|
+
options: Partial<HNSWIndexConfig> = {}
|
|
197
|
+
): Promise<{ index: HierarchicalNSW; serialized: Uint8Array }> {
|
|
198
|
+
await this.ensureInitialized();
|
|
199
|
+
|
|
200
|
+
try {
|
|
201
|
+
const config = { ...this.indexConfig, ...options };
|
|
202
|
+
|
|
203
|
+
console.log(`🔨 Creating new HNSW index for user ${userAddress}`);
|
|
204
|
+
console.log(` Dimensions: ${config.dimension}, M: ${config.m}, efConstruction: ${config.efConstruction}`);
|
|
205
|
+
console.log(` Cache stats: ${this.indexCache.size}/${this.maxCachedIndexes} indexes, ${(this.indexCache.memoryBytes / 1024 / 1024).toFixed(1)}MB`);
|
|
206
|
+
|
|
207
|
+
// Create a new index using WASM (constructor takes: spaceName, numDimensions, autoSaveFilename)
|
|
208
|
+
const index = new this.hnswlib!.HierarchicalNSW(config.spaceType, config.dimension, '');
|
|
209
|
+
index.initIndex(config.maxElements, config.m, config.efConstruction, config.randomSeed);
|
|
210
|
+
|
|
211
|
+
// Create cache entry (LRU cache handles eviction automatically)
|
|
212
|
+
this.indexCache.set(userAddress, {
|
|
213
|
+
index,
|
|
214
|
+
lastModified: new Date(),
|
|
215
|
+
pendingVectors: new Map(),
|
|
216
|
+
isDirty: false,
|
|
217
|
+
version: 1,
|
|
218
|
+
metadata: new Map(),
|
|
219
|
+
dimensions: config.dimension,
|
|
220
|
+
vectors: new Map(),
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
// Serialize the empty index
|
|
224
|
+
const indexName = `index_${userAddress}_${Date.now()}`;
|
|
225
|
+
await index.writeIndex(indexName);
|
|
226
|
+
|
|
227
|
+
// Sync to IndexedDB (persist the index) - syncFS requires a callback
|
|
228
|
+
await this.hnswlib!.EmscriptenFileSystemManager.syncFS(false, () => {});
|
|
229
|
+
|
|
230
|
+
// Read serialized data from filesystem for returning
|
|
231
|
+
const serialized = new Uint8Array(0); // Placeholder - actual data is in IndexedDB
|
|
232
|
+
|
|
233
|
+
console.log(`✅ Index created successfully for ${userAddress}`);
|
|
234
|
+
|
|
235
|
+
return { index, serialized };
|
|
236
|
+
} catch (error) {
|
|
237
|
+
throw this.createVectorError('INDEX_ERROR', `Failed to create index: ${error}`, error);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Add vector to index with batching (main entry point)
|
|
243
|
+
*/
|
|
244
|
+
addVectorToIndexBatched(
|
|
245
|
+
userAddress: string,
|
|
246
|
+
vectorId: number,
|
|
247
|
+
vector: number[],
|
|
248
|
+
metadata?: any
|
|
249
|
+
): void {
|
|
250
|
+
try {
|
|
251
|
+
// Validate input
|
|
252
|
+
this.validateVector(vector);
|
|
253
|
+
|
|
254
|
+
// Get or create cache entry (LRU cache will evict old entries if needed)
|
|
255
|
+
let cacheEntry = this.indexCache.get(userAddress);
|
|
256
|
+
if (!cacheEntry) {
|
|
257
|
+
console.warn(`No cached index found for user ${userAddress}, will create on first flush`);
|
|
258
|
+
// Create placeholder entry - actual index created on first flush
|
|
259
|
+
cacheEntry = {
|
|
260
|
+
index: null as any, // Will be created on flush
|
|
261
|
+
lastModified: new Date(),
|
|
262
|
+
pendingVectors: new Map(),
|
|
263
|
+
isDirty: true,
|
|
264
|
+
version: 1,
|
|
265
|
+
metadata: new Map(),
|
|
266
|
+
dimensions: vector.length,
|
|
267
|
+
vectors: new Map(),
|
|
268
|
+
};
|
|
269
|
+
this.indexCache.set(userAddress, cacheEntry);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Validate vector dimensions
|
|
273
|
+
if (cacheEntry.dimensions && vector.length !== cacheEntry.dimensions) {
|
|
274
|
+
throw new Error(`Vector dimension mismatch: expected ${cacheEntry.dimensions}, got ${vector.length}`);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// Add to pending queue
|
|
278
|
+
cacheEntry.pendingVectors.set(vectorId, vector);
|
|
279
|
+
if (metadata) {
|
|
280
|
+
cacheEntry.metadata.set(vectorId, metadata);
|
|
281
|
+
}
|
|
282
|
+
// Also cache the vector for serialization
|
|
283
|
+
cacheEntry.vectors.set(vectorId, vector);
|
|
284
|
+
cacheEntry.isDirty = true;
|
|
285
|
+
cacheEntry.lastModified = new Date();
|
|
286
|
+
|
|
287
|
+
// Schedule or update batch job
|
|
288
|
+
this.scheduleBatchJob(userAddress, vectorId, vector);
|
|
289
|
+
|
|
290
|
+
console.debug(`📊 Vector ${vectorId} queued for batch processing. Pending: ${cacheEntry.pendingVectors.size}`);
|
|
291
|
+
|
|
292
|
+
// Process immediately if batch size limit reached
|
|
293
|
+
if (cacheEntry.pendingVectors.size >= this.config.maxBatchSize) {
|
|
294
|
+
console.log(`⚡ Batch size limit reached (${this.config.maxBatchSize}), processing immediately`);
|
|
295
|
+
setTimeout(() => this.flushPendingVectors(userAddress), 0);
|
|
296
|
+
}
|
|
297
|
+
} catch (error) {
|
|
298
|
+
throw this.createVectorError('INDEX_ERROR', `Failed to queue vector: ${error}`, error);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* Search vectors in the index (including pending vectors)
|
|
304
|
+
*/
|
|
305
|
+
async searchVectors(
|
|
306
|
+
userAddress: string,
|
|
307
|
+
queryVector: number[],
|
|
308
|
+
options: HNSWSearchOptions = {}
|
|
309
|
+
): Promise<HNSWSearchResult> {
|
|
310
|
+
await this.ensureInitialized();
|
|
311
|
+
|
|
312
|
+
try {
|
|
313
|
+
this.validateVector(queryVector);
|
|
314
|
+
|
|
315
|
+
const cacheEntry = this.indexCache.get(userAddress);
|
|
316
|
+
if (!cacheEntry?.index) {
|
|
317
|
+
throw new Error(`No index found for user ${userAddress}`);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const { k = 10, efSearch = 50, filter } = options;
|
|
321
|
+
|
|
322
|
+
// Set search parameters
|
|
323
|
+
cacheEntry.index.setEfSearch(efSearch);
|
|
324
|
+
|
|
325
|
+
let searchIndex = cacheEntry.index;
|
|
326
|
+
|
|
327
|
+
// If there are pending vectors, flush them first
|
|
328
|
+
if (cacheEntry.pendingVectors.size > 0) {
|
|
329
|
+
console.log(`⏳ Flushing ${cacheEntry.pendingVectors.size} pending vectors before search`);
|
|
330
|
+
await this.flushPendingVectors(userAddress);
|
|
331
|
+
// Get updated index
|
|
332
|
+
const updatedEntry = this.indexCache.get(userAddress);
|
|
333
|
+
if (updatedEntry?.index) {
|
|
334
|
+
searchIndex = updatedEntry.index;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Perform search (convert to Float32Array if needed)
|
|
339
|
+
const queryFloat32 = queryVector instanceof Float32Array
|
|
340
|
+
? queryVector
|
|
341
|
+
: new Float32Array(queryVector);
|
|
342
|
+
const result = searchIndex.searchKnn(
|
|
343
|
+
queryFloat32,
|
|
344
|
+
k,
|
|
345
|
+
filter && typeof filter === 'function' ? (filter as (label: number) => boolean) : undefined
|
|
346
|
+
);
|
|
347
|
+
|
|
348
|
+
// Apply metadata filter if provided (additional filtering)
|
|
349
|
+
let filteredIds = result.neighbors;
|
|
350
|
+
let filteredDistances = result.distances;
|
|
351
|
+
|
|
352
|
+
if (filter && typeof filter === 'function') {
|
|
353
|
+
const filtered = this.applyMetadataFilter(
|
|
354
|
+
result.neighbors,
|
|
355
|
+
result.distances,
|
|
356
|
+
cacheEntry.metadata,
|
|
357
|
+
filter as (metadata: any) => boolean
|
|
358
|
+
);
|
|
359
|
+
filteredIds = filtered.ids;
|
|
360
|
+
filteredDistances = filtered.distances;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// Convert distances to similarities (for cosine distance)
|
|
364
|
+
const similarities = this.indexConfig.spaceType === 'cosine'
|
|
365
|
+
? filteredDistances.map((dist: number) => 1 - dist)
|
|
366
|
+
: filteredDistances.map((dist: number) => 1 / (1 + dist));
|
|
367
|
+
|
|
368
|
+
console.log(`🔍 Search completed: ${filteredIds.length} results found`);
|
|
369
|
+
|
|
370
|
+
return {
|
|
371
|
+
ids: filteredIds,
|
|
372
|
+
distances: filteredDistances,
|
|
373
|
+
similarities
|
|
374
|
+
};
|
|
375
|
+
} catch (error) {
|
|
376
|
+
throw this.createVectorError('SEARCH_ERROR', `Search failed: ${error}`, error);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* Load index from Walrus storage
|
|
382
|
+
*/
|
|
383
|
+
async loadIndex(blobId: string, userAddress: string): Promise<HierarchicalNSW> {
|
|
384
|
+
await this.ensureInitialized();
|
|
385
|
+
|
|
386
|
+
try {
|
|
387
|
+
console.log(`📥 Loading HNSW index from Walrus: ${blobId}`);
|
|
388
|
+
|
|
389
|
+
// Download index from Walrus
|
|
390
|
+
const retrieveResult = await this.storageService.retrieve(blobId);
|
|
391
|
+
const indexBuffer = retrieveResult.content;
|
|
392
|
+
|
|
393
|
+
// Save to Emscripten virtual filesystem
|
|
394
|
+
const indexName = `index_${userAddress}_${Date.now()}`;
|
|
395
|
+
(this.hnswlib!.EmscriptenFileSystemManager as any).writeFile(indexName, indexBuffer);
|
|
396
|
+
|
|
397
|
+
// Sync from IndexedDB (load the data into memory)
|
|
398
|
+
await this.hnswlib!.EmscriptenFileSystemManager.syncFS(true, () => {});
|
|
399
|
+
|
|
400
|
+
// Create and load index (constructor: spaceName, numDimensions, autoSaveFilename)
|
|
401
|
+
const index = new this.hnswlib!.HierarchicalNSW(this.indexConfig.spaceType, this.indexConfig.dimension, '');
|
|
402
|
+
await index.readIndex(indexName, this.indexConfig.maxElements);
|
|
403
|
+
|
|
404
|
+
// Cache the loaded index
|
|
405
|
+
this.indexCache.set(userAddress, {
|
|
406
|
+
index,
|
|
407
|
+
lastModified: new Date(),
|
|
408
|
+
pendingVectors: new Map(),
|
|
409
|
+
isDirty: false,
|
|
410
|
+
version: 1,
|
|
411
|
+
metadata: new Map(),
|
|
412
|
+
dimensions: this.indexConfig.dimension,
|
|
413
|
+
vectors: new Map(),
|
|
414
|
+
});
|
|
415
|
+
|
|
416
|
+
console.log(`✅ Index loaded successfully for ${userAddress}`);
|
|
417
|
+
return index;
|
|
418
|
+
} catch (error) {
|
|
419
|
+
throw this.createVectorError('STORAGE_ERROR', `Failed to load index: ${error}`, error);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Save index to Walrus storage
|
|
425
|
+
*/
|
|
426
|
+
async saveIndex(userAddress: string): Promise<string> {
|
|
427
|
+
await this.ensureInitialized();
|
|
428
|
+
|
|
429
|
+
try {
|
|
430
|
+
const cacheEntry = this.indexCache.get(userAddress);
|
|
431
|
+
if (!cacheEntry?.index) {
|
|
432
|
+
throw new Error(`No index found for user ${userAddress}`);
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
console.log(`💾 Saving index to Walrus for ${userAddress}`);
|
|
436
|
+
|
|
437
|
+
return await this.saveIndexToWalrus(cacheEntry.index, userAddress);
|
|
438
|
+
} catch (error) {
|
|
439
|
+
throw this.createVectorError('STORAGE_ERROR', `Failed to save index: ${error}`, error);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Force flush all pending vectors for a user
|
|
445
|
+
*/
|
|
446
|
+
async forceFlush(userAddress: string): Promise<void> {
|
|
447
|
+
await this.flushPendingVectors(userAddress);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Get cache statistics
|
|
452
|
+
*/
|
|
453
|
+
getCacheStats(): BatchStats & {
|
|
454
|
+
memoryUsageMB: number;
|
|
455
|
+
maxMemoryMB: number;
|
|
456
|
+
maxCachedIndexes: number;
|
|
457
|
+
} {
|
|
458
|
+
let totalPendingVectors = 0;
|
|
459
|
+
|
|
460
|
+
for (const [, entry] of this.indexCache.entries()) {
|
|
461
|
+
totalPendingVectors += entry.pendingVectors.size;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
const lruStats = this.indexCache.getStats();
|
|
465
|
+
|
|
466
|
+
return {
|
|
467
|
+
totalUsers: this.indexCache.size,
|
|
468
|
+
totalPendingVectors,
|
|
469
|
+
activeBatchJobs: this.batchJobs.size,
|
|
470
|
+
cacheHitRate: 0, // TODO: Implement hit rate tracking
|
|
471
|
+
averageBatchSize: totalPendingVectors / Math.max(1, this.indexCache.size),
|
|
472
|
+
averageProcessingTime: 0, // TODO: Implement timing tracking
|
|
473
|
+
memoryUsageMB: lruStats.memoryBytes / 1024 / 1024,
|
|
474
|
+
maxMemoryMB: this.maxMemoryBytes / 1024 / 1024,
|
|
475
|
+
maxCachedIndexes: this.maxCachedIndexes,
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* Remove a vector from the index
|
|
481
|
+
*/
|
|
482
|
+
removeVector(userAddress: string, vectorId: number): void {
|
|
483
|
+
try {
|
|
484
|
+
const cacheEntry = this.indexCache.get(userAddress);
|
|
485
|
+
if (!cacheEntry?.index) {
|
|
486
|
+
throw new Error(`No index found for user ${userAddress}`);
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
// Remove from pending vectors if exists
|
|
490
|
+
cacheEntry.pendingVectors.delete(vectorId);
|
|
491
|
+
cacheEntry.metadata.delete(vectorId);
|
|
492
|
+
|
|
493
|
+
// Note: hnswlib-wasm doesn't support deletion, mark for rebuild
|
|
494
|
+
cacheEntry.isDirty = true;
|
|
495
|
+
cacheEntry.lastModified = new Date();
|
|
496
|
+
|
|
497
|
+
console.log(`🗑️ Vector ${vectorId} removed from index`);
|
|
498
|
+
} catch (error) {
|
|
499
|
+
throw this.createVectorError('INDEX_ERROR', `Failed to remove vector: ${error}`, error);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Clear user index and cache
|
|
505
|
+
*/
|
|
506
|
+
clearUserIndex(userAddress: string): void {
|
|
507
|
+
this.indexCache.delete(userAddress);
|
|
508
|
+
this.batchJobs.delete(userAddress);
|
|
509
|
+
console.log(`🧹 Cleared index cache for user ${userAddress}`);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/**
|
|
513
|
+
* Cleanup resources
|
|
514
|
+
*/
|
|
515
|
+
destroy(): void {
|
|
516
|
+
if (this.batchProcessor) {
|
|
517
|
+
clearInterval(this.batchProcessor);
|
|
518
|
+
}
|
|
519
|
+
// LRU cache cleanup is handled internally, but we should destroy it
|
|
520
|
+
this.indexCache.destroy();
|
|
521
|
+
this.batchJobs.clear();
|
|
522
|
+
console.log('🛑 HnswWasmService destroyed');
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// ==================== PRIVATE METHODS ====================
|
|
526
|
+
|
|
527
|
+
private async createCacheEntry(dimensions: number): Promise<IndexCacheEntry> {
|
|
528
|
+
await this.ensureInitialized();
|
|
529
|
+
|
|
530
|
+
const index = new this.hnswlib!.HierarchicalNSW(this.indexConfig.spaceType, dimensions, '');
|
|
531
|
+
index.initIndex(this.indexConfig.maxElements, this.indexConfig.m, this.indexConfig.efConstruction, this.indexConfig.randomSeed);
|
|
532
|
+
|
|
533
|
+
return {
|
|
534
|
+
index,
|
|
535
|
+
lastModified: new Date(),
|
|
536
|
+
pendingVectors: new Map(),
|
|
537
|
+
isDirty: false,
|
|
538
|
+
version: 1,
|
|
539
|
+
metadata: new Map(),
|
|
540
|
+
dimensions,
|
|
541
|
+
vectors: new Map(),
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
private scheduleBatchJob(userAddress: string, vectorId: number, vector: number[]): void {
|
|
546
|
+
let batchJob = this.batchJobs.get(userAddress);
|
|
547
|
+
if (!batchJob) {
|
|
548
|
+
batchJob = {
|
|
549
|
+
userAddress,
|
|
550
|
+
vectors: new Map(),
|
|
551
|
+
scheduledAt: new Date()
|
|
552
|
+
};
|
|
553
|
+
this.batchJobs.set(userAddress, batchJob);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
batchJob.vectors.set(vectorId, vector);
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
private startBatchProcessor(): void {
|
|
560
|
+
this.batchProcessor = setInterval(async () => {
|
|
561
|
+
await this.processBatchJobs();
|
|
562
|
+
}, this.config.batchDelayMs);
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Note: Cache cleanup is now handled by LRUCache internally
|
|
566
|
+
|
|
567
|
+
private async processBatchJobs(): Promise<void> {
|
|
568
|
+
const now = Date.now();
|
|
569
|
+
const jobsToProcess: string[] = [];
|
|
570
|
+
|
|
571
|
+
for (const [userAddress, job] of this.batchJobs.entries()) {
|
|
572
|
+
const timeSinceScheduled = now - job.scheduledAt.getTime();
|
|
573
|
+
const cacheEntry = this.indexCache.get(userAddress);
|
|
574
|
+
|
|
575
|
+
if (timeSinceScheduled >= this.config.batchDelayMs ||
|
|
576
|
+
(cacheEntry && cacheEntry.pendingVectors.size >= this.config.maxBatchSize)) {
|
|
577
|
+
jobsToProcess.push(userAddress);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
for (const userAddress of jobsToProcess) {
|
|
582
|
+
try {
|
|
583
|
+
await this.flushPendingVectors(userAddress);
|
|
584
|
+
} catch (error) {
|
|
585
|
+
console.error(`❌ Error processing batch job for user ${userAddress}:`, error);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
private async flushPendingVectors(userAddress: string): Promise<void> {
|
|
591
|
+
await this.ensureInitialized();
|
|
592
|
+
|
|
593
|
+
const cacheEntry = this.indexCache.get(userAddress);
|
|
594
|
+
if (!cacheEntry || cacheEntry.pendingVectors.size === 0) {
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
console.log(`⚡ Flushing ${cacheEntry.pendingVectors.size} pending vectors for user ${userAddress}`);
|
|
599
|
+
|
|
600
|
+
try {
|
|
601
|
+
// Create index if it doesn't exist
|
|
602
|
+
if (!cacheEntry.index) {
|
|
603
|
+
const newEntry = await this.createCacheEntry(cacheEntry.dimensions);
|
|
604
|
+
cacheEntry.index = newEntry.index;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
// Prepare vectors array for batch insertion
|
|
608
|
+
const vectors: number[][] = [];
|
|
609
|
+
const labels: number[] = [];
|
|
610
|
+
|
|
611
|
+
for (const [vectorId, vector] of cacheEntry.pendingVectors.entries()) {
|
|
612
|
+
vectors.push(vector);
|
|
613
|
+
labels.push(vectorId);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// Add all pending vectors to the index in batch
|
|
617
|
+
if (vectors.length > 0) {
|
|
618
|
+
// Convert to Float32Array[] as required by hnswlib-wasm
|
|
619
|
+
const float32Vectors = vectors.map(v =>
|
|
620
|
+
v instanceof Float32Array ? v : new Float32Array(v)
|
|
621
|
+
);
|
|
622
|
+
cacheEntry.index.addItems(float32Vectors, true);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// Save to Walrus
|
|
626
|
+
await this.saveIndexToWalrus(cacheEntry.index, userAddress);
|
|
627
|
+
|
|
628
|
+
// Clear pending vectors
|
|
629
|
+
cacheEntry.pendingVectors.clear();
|
|
630
|
+
cacheEntry.isDirty = false;
|
|
631
|
+
cacheEntry.lastModified = new Date();
|
|
632
|
+
cacheEntry.version++;
|
|
633
|
+
|
|
634
|
+
// Remove batch job
|
|
635
|
+
this.batchJobs.delete(userAddress);
|
|
636
|
+
|
|
637
|
+
console.log(`✅ Successfully flushed vectors for user ${userAddress} (version ${cacheEntry.version})`);
|
|
638
|
+
} catch (error) {
|
|
639
|
+
console.error(`❌ Error flushing vectors for user ${userAddress}:`, error);
|
|
640
|
+
throw error;
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
private async saveIndexToWalrus(index: HierarchicalNSW, userAddress: string): Promise<string> {
|
|
645
|
+
await this.ensureInitialized();
|
|
646
|
+
|
|
647
|
+
try {
|
|
648
|
+
// Serialize index to Emscripten filesystem
|
|
649
|
+
const indexName = `index_${userAddress}_${Date.now()}`;
|
|
650
|
+
await index.writeIndex(indexName);
|
|
651
|
+
|
|
652
|
+
// Sync to IndexedDB
|
|
653
|
+
await this.hnswlib!.EmscriptenFileSystemManager.syncFS(false, () => {});
|
|
654
|
+
|
|
655
|
+
// Read serialized data from filesystem
|
|
656
|
+
const serialized = (this.hnswlib!.EmscriptenFileSystemManager as any).readFile(indexName) as Uint8Array;
|
|
657
|
+
|
|
658
|
+
// Upload to Walrus via StorageService
|
|
659
|
+
const metadata: MemoryMetadata = {
|
|
660
|
+
contentType: 'application/hnsw-index-wasm',
|
|
661
|
+
contentSize: serialized.byteLength,
|
|
662
|
+
contentHash: '', // TODO: Calculate hash
|
|
663
|
+
category: 'vector-index',
|
|
664
|
+
topic: 'hnsw-wasm',
|
|
665
|
+
importance: 8,
|
|
666
|
+
embeddingDimension: this.indexConfig.dimension,
|
|
667
|
+
createdTimestamp: Date.now(),
|
|
668
|
+
customMetadata: {
|
|
669
|
+
'user-address': userAddress,
|
|
670
|
+
'version': '1.0',
|
|
671
|
+
'wasm': 'true'
|
|
672
|
+
}
|
|
673
|
+
};
|
|
674
|
+
|
|
675
|
+
const result = await this.storageService.upload(serialized, metadata);
|
|
676
|
+
|
|
677
|
+
console.log(`💾 Index saved to Walrus: ${result.blobId}`);
|
|
678
|
+
return result.blobId;
|
|
679
|
+
} catch (error) {
|
|
680
|
+
console.error('❌ Failed to save index to Walrus:', error);
|
|
681
|
+
throw error;
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
private applyMetadataFilter(
|
|
686
|
+
ids: number[],
|
|
687
|
+
distances: number[],
|
|
688
|
+
metadata: Map<number, any>,
|
|
689
|
+
filter: (metadata: any) => boolean
|
|
690
|
+
): { ids: number[]; distances: number[] } {
|
|
691
|
+
const filteredIds: number[] = [];
|
|
692
|
+
const filteredDistances: number[] = [];
|
|
693
|
+
|
|
694
|
+
for (let i = 0; i < ids.length; i++) {
|
|
695
|
+
const vectorId = ids[i];
|
|
696
|
+
const vectorMetadata = metadata.get(vectorId);
|
|
697
|
+
|
|
698
|
+
if (!vectorMetadata || filter(vectorMetadata)) {
|
|
699
|
+
filteredIds.push(vectorId);
|
|
700
|
+
filteredDistances.push(distances[i]);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
return { ids: filteredIds, distances: filteredDistances };
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
// Note: cleanupCache is now handled by LRUCache internally
|
|
708
|
+
|
|
709
|
+
private validateVector(vector: number[]): void {
|
|
710
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
711
|
+
throw new Error('Vector must be a non-empty array');
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
if (vector.some(v => typeof v !== 'number' || !isFinite(v))) {
|
|
715
|
+
throw new Error('Vector must contain only finite numbers');
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
if (vector.length !== this.indexConfig.dimension) {
|
|
719
|
+
throw new Error(`Vector dimension mismatch: expected ${this.indexConfig.dimension}, got ${vector.length}`);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
private createVectorError(code: VectorError['code'], message: string, details?: any): VectorError {
|
|
724
|
+
const error = new Error(message) as VectorError;
|
|
725
|
+
error.code = code;
|
|
726
|
+
error.details = details;
|
|
727
|
+
return error;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
export default HnswWasmService;
|