@cmdoss/memwal-sdk 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +547 -547
- package/BENCHMARKS.md +238 -238
- package/README.md +310 -181
- package/dist/ai-sdk/tools.d.ts +2 -2
- package/dist/ai-sdk/tools.js +2 -2
- package/dist/client/ClientMemoryManager.js +2 -2
- package/dist/client/ClientMemoryManager.js.map +1 -1
- package/dist/client/PersonalDataWallet.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.d.ts +29 -1
- package/dist/client/SimplePDWClient.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.js +45 -13
- package/dist/client/SimplePDWClient.js.map +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.d.ts +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.js +1 -1
- package/dist/client/namespaces/MemoryNamespace.d.ts +31 -0
- package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/MemoryNamespace.js +272 -39
- package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/AINamespace.d.ts +2 -2
- package/dist/client/namespaces/consolidated/AINamespace.js +2 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts +12 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js +62 -4
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts +67 -2
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.js +549 -16
- package/dist/client/namespaces/consolidated/StorageNamespace.js.map +1 -1
- package/dist/config/ConfigurationHelper.js +61 -61
- package/dist/config/defaults.js +2 -2
- package/dist/config/defaults.js.map +1 -1
- package/dist/graph/GraphService.js +21 -21
- package/dist/graph/GraphService.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/seal/EncryptionService.d.ts +9 -5
- package/dist/infrastructure/seal/EncryptionService.d.ts.map +1 -1
- package/dist/infrastructure/seal/EncryptionService.js +37 -15
- package/dist/infrastructure/seal/EncryptionService.js.map +1 -1
- package/dist/infrastructure/seal/SealService.d.ts +13 -5
- package/dist/infrastructure/seal/SealService.d.ts.map +1 -1
- package/dist/infrastructure/seal/SealService.js +36 -34
- package/dist/infrastructure/seal/SealService.js.map +1 -1
- package/dist/langchain/createPDWRAG.js +30 -30
- package/dist/retrieval/MemoryDecryptionPipeline.d.ts.map +1 -1
- package/dist/retrieval/MemoryDecryptionPipeline.js +2 -1
- package/dist/retrieval/MemoryDecryptionPipeline.js.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.d.ts +31 -0
- package/dist/retrieval/MemoryRetrievalService.d.ts.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.js +44 -4
- package/dist/retrieval/MemoryRetrievalService.js.map +1 -1
- package/dist/services/CapabilityService.d.ts.map +1 -1
- package/dist/services/CapabilityService.js +30 -14
- package/dist/services/CapabilityService.js.map +1 -1
- package/dist/services/CrossContextPermissionService.d.ts.map +1 -1
- package/dist/services/CrossContextPermissionService.js +9 -7
- package/dist/services/CrossContextPermissionService.js.map +1 -1
- package/dist/services/EmbeddingService.d.ts +28 -1
- package/dist/services/EmbeddingService.d.ts.map +1 -1
- package/dist/services/EmbeddingService.js +54 -0
- package/dist/services/EmbeddingService.js.map +1 -1
- package/dist/services/EncryptionService.d.ts.map +1 -1
- package/dist/services/EncryptionService.js +6 -5
- package/dist/services/EncryptionService.js.map +1 -1
- package/dist/services/GeminiAIService.js +309 -309
- package/dist/services/IndexManager.d.ts +5 -1
- package/dist/services/IndexManager.d.ts.map +1 -1
- package/dist/services/IndexManager.js +17 -40
- package/dist/services/IndexManager.js.map +1 -1
- package/dist/services/QueryService.js +1 -1
- package/dist/services/QueryService.js.map +1 -1
- package/dist/services/StorageService.d.ts +11 -0
- package/dist/services/StorageService.d.ts.map +1 -1
- package/dist/services/StorageService.js +73 -10
- package/dist/services/StorageService.js.map +1 -1
- package/dist/services/TransactionService.d.ts +20 -0
- package/dist/services/TransactionService.d.ts.map +1 -1
- package/dist/services/TransactionService.js +43 -0
- package/dist/services/TransactionService.js.map +1 -1
- package/dist/services/ViewService.js +2 -2
- package/dist/services/ViewService.js.map +1 -1
- package/dist/services/storage/QuiltBatchManager.d.ts +101 -1
- package/dist/services/storage/QuiltBatchManager.d.ts.map +1 -1
- package/dist/services/storage/QuiltBatchManager.js +410 -20
- package/dist/services/storage/QuiltBatchManager.js.map +1 -1
- package/dist/services/storage/index.d.ts +1 -1
- package/dist/services/storage/index.d.ts.map +1 -1
- package/dist/services/storage/index.js.map +1 -1
- package/dist/utils/LRUCache.d.ts +106 -0
- package/dist/utils/LRUCache.d.ts.map +1 -0
- package/dist/utils/LRUCache.js +281 -0
- package/dist/utils/LRUCache.js.map +1 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/memoryIndexOnChain.d.ts +212 -0
- package/dist/utils/memoryIndexOnChain.d.ts.map +1 -0
- package/dist/utils/memoryIndexOnChain.js +312 -0
- package/dist/utils/memoryIndexOnChain.js.map +1 -0
- package/dist/utils/rebuildIndexNode.d.ts +29 -0
- package/dist/utils/rebuildIndexNode.d.ts.map +1 -1
- package/dist/utils/rebuildIndexNode.js +366 -98
- package/dist/utils/rebuildIndexNode.js.map +1 -1
- package/dist/vector/HnswWasmService.d.ts +20 -5
- package/dist/vector/HnswWasmService.d.ts.map +1 -1
- package/dist/vector/HnswWasmService.js +73 -40
- package/dist/vector/HnswWasmService.js.map +1 -1
- package/dist/vector/IHnswService.d.ts +10 -1
- package/dist/vector/IHnswService.d.ts.map +1 -1
- package/dist/vector/IHnswService.js.map +1 -1
- package/dist/vector/NodeHnswService.d.ts +16 -0
- package/dist/vector/NodeHnswService.d.ts.map +1 -1
- package/dist/vector/NodeHnswService.js +84 -5
- package/dist/vector/NodeHnswService.js.map +1 -1
- package/dist/vector/createHnswService.d.ts +1 -1
- package/dist/vector/createHnswService.js +1 -1
- package/dist/vector/index.d.ts +1 -1
- package/dist/vector/index.js +1 -1
- package/package.json +157 -157
- package/src/access/PermissionService.ts +635 -635
- package/src/aggregation/AggregationService.ts +389 -389
- package/src/ai-sdk/PDWVectorStore.ts +715 -715
- package/src/ai-sdk/index.ts +65 -65
- package/src/ai-sdk/tools.ts +460 -460
- package/src/ai-sdk/types.ts +404 -404
- package/src/batch/BatchManager.ts +597 -597
- package/src/batch/BatchingService.ts +429 -429
- package/src/batch/MemoryProcessingCache.ts +492 -492
- package/src/batch/index.ts +30 -30
- package/src/browser.ts +200 -200
- package/src/client/ClientMemoryManager.ts +987 -987
- package/src/client/PersonalDataWallet.ts +345 -345
- package/src/client/SimplePDWClient.ts +1289 -1222
- package/src/client/factory.ts +154 -154
- package/src/client/namespaces/AnalyticsNamespace.ts +377 -377
- package/src/client/namespaces/BatchNamespace.ts +356 -356
- package/src/client/namespaces/CacheNamespace.ts +123 -123
- package/src/client/namespaces/CapabilityNamespace.ts +217 -217
- package/src/client/namespaces/ClassifyNamespace.ts +169 -169
- package/src/client/namespaces/ContextNamespace.ts +297 -297
- package/src/client/namespaces/EmbeddingsNamespace.ts +99 -99
- package/src/client/namespaces/EncryptionNamespace.ts +221 -221
- package/src/client/namespaces/GraphNamespace.ts +468 -468
- package/src/client/namespaces/IndexNamespace.ts +361 -361
- package/src/client/namespaces/MemoryNamespace.ts +1422 -1135
- package/src/client/namespaces/PermissionsNamespace.ts +254 -254
- package/src/client/namespaces/PipelineNamespace.ts +220 -220
- package/src/client/namespaces/SearchNamespace.ts +1049 -1049
- package/src/client/namespaces/StorageNamespace.ts +458 -458
- package/src/client/namespaces/TxNamespace.ts +260 -260
- package/src/client/namespaces/WalletNamespace.ts +243 -243
- package/src/client/namespaces/consolidated/AINamespace.ts +449 -449
- package/src/client/namespaces/consolidated/BlockchainNamespace.ts +607 -546
- package/src/client/namespaces/consolidated/SecurityNamespace.ts +648 -648
- package/src/client/namespaces/consolidated/StorageNamespace.ts +1141 -497
- package/src/client/namespaces/consolidated/index.ts +39 -39
- package/src/client/signers/KeypairSigner.ts +108 -108
- package/src/client/signers/UnifiedSigner.ts +110 -110
- package/src/client/signers/WalletAdapterSigner.ts +159 -159
- package/src/client/signers/index.ts +26 -26
- package/src/config/ConfigurationHelper.ts +412 -412
- package/src/config/defaults.ts +51 -51
- package/src/config/index.ts +8 -8
- package/src/config/validation.ts +70 -70
- package/src/core/index.ts +14 -14
- package/src/core/interfaces/IService.ts +307 -307
- package/src/core/interfaces/index.ts +8 -8
- package/src/core/types/capability.ts +297 -297
- package/src/core/types/index.ts +870 -870
- package/src/core/types/wallet.ts +270 -270
- package/src/core/types.ts +9 -9
- package/src/core/wallet.ts +222 -222
- package/src/embedding/index.ts +19 -19
- package/src/embedding/types.ts +357 -357
- package/src/errors/index.ts +602 -602
- package/src/errors/recovery.ts +461 -461
- package/src/errors/validation.ts +567 -567
- package/src/generated/pdw/capability.ts +319 -319
- package/src/graph/GraphService.ts +887 -887
- package/src/graph/KnowledgeGraphManager.ts +728 -728
- package/src/graph/index.ts +25 -25
- package/src/index.ts +498 -474
- package/src/infrastructure/index.ts +22 -22
- package/src/infrastructure/seal/EncryptionService.ts +628 -603
- package/src/infrastructure/seal/SealService.ts +613 -615
- package/src/infrastructure/seal/index.ts +9 -9
- package/src/infrastructure/sui/BlockchainManager.ts +627 -627
- package/src/infrastructure/sui/SuiService.ts +888 -888
- package/src/infrastructure/sui/index.ts +9 -9
- package/src/infrastructure/walrus/StorageManager.ts +604 -604
- package/src/infrastructure/walrus/WalrusStorageService.ts +612 -612
- package/src/infrastructure/walrus/index.ts +9 -9
- package/src/langchain/PDWEmbeddings.ts +145 -145
- package/src/langchain/PDWVectorStore.ts +456 -456
- package/src/langchain/createPDWRAG.ts +303 -303
- package/src/langchain/index.ts +47 -47
- package/src/permissions/ConsentRepository.browser.ts +249 -249
- package/src/permissions/ConsentRepository.ts +364 -364
- package/src/pipeline/MemoryPipeline.ts +862 -862
- package/src/pipeline/PipelineManager.ts +683 -683
- package/src/pipeline/index.ts +26 -26
- package/src/retrieval/AdvancedSearchService.ts +629 -629
- package/src/retrieval/MemoryAnalyticsService.ts +711 -711
- package/src/retrieval/MemoryDecryptionPipeline.ts +825 -824
- package/src/retrieval/MemoryRetrievalService.ts +904 -830
- package/src/retrieval/index.ts +42 -42
- package/src/services/BatchService.ts +352 -352
- package/src/services/CapabilityService.ts +464 -448
- package/src/services/ClassifierService.ts +465 -465
- package/src/services/CrossContextPermissionService.ts +486 -484
- package/src/services/EmbeddingService.ts +771 -706
- package/src/services/EncryptionService.ts +712 -711
- package/src/services/GeminiAIService.ts +753 -753
- package/src/services/IndexManager.ts +977 -1004
- package/src/services/MemoryIndexService.ts +1003 -1003
- package/src/services/MemoryService.ts +369 -369
- package/src/services/QueryService.ts +890 -890
- package/src/services/StorageService.ts +1182 -1111
- package/src/services/TransactionService.ts +838 -790
- package/src/services/VectorService.ts +462 -462
- package/src/services/ViewService.ts +484 -484
- package/src/services/index.ts +25 -25
- package/src/services/storage/BlobAttributesManager.ts +333 -333
- package/src/services/storage/KnowledgeGraphManager.ts +425 -425
- package/src/services/storage/MemorySearchManager.ts +387 -387
- package/src/services/storage/QuiltBatchManager.ts +1130 -660
- package/src/services/storage/WalrusMetadataManager.ts +268 -268
- package/src/services/storage/WalrusStorageManager.ts +287 -287
- package/src/services/storage/index.ts +57 -52
- package/src/types/index.ts +13 -13
- package/src/utils/LRUCache.ts +378 -0
- package/src/utils/index.ts +76 -68
- package/src/utils/memoryIndexOnChain.ts +507 -0
- package/src/utils/rebuildIndex.ts +290 -290
- package/src/utils/rebuildIndexNode.ts +771 -424
- package/src/vector/BrowserHnswIndexService.ts +758 -758
- package/src/vector/HnswWasmService.ts +731 -679
- package/src/vector/IHnswService.ts +233 -224
- package/src/vector/NodeHnswService.ts +833 -735
- package/src/vector/VectorManager.ts +478 -478
- package/src/vector/createHnswService.ts +135 -135
- package/src/vector/index.ts +56 -56
- package/src/wallet/ContextWalletService.ts +656 -656
- package/src/wallet/MainWalletService.ts +317 -317
|
@@ -1,660 +1,1130 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* QuiltBatchManager - Batch Memory Operations via Walrus Quilts
|
|
3
|
-
*
|
|
4
|
-
* Handles batch uploads and queries using Walrus Quilt functionality.
|
|
5
|
-
* Extracted from StorageService for better separation of concerns.
|
|
6
|
-
*
|
|
7
|
-
* Features:
|
|
8
|
-
* - Batch upload with ~90% gas savings (single transaction for multiple files)
|
|
9
|
-
* - Tag-based filtering at the Walrus level
|
|
10
|
-
* - Multi-file retrieval via quiltPatchId
|
|
11
|
-
* - Browser-compatible using writeFilesFlow (2 user signatures)
|
|
12
|
-
*
|
|
13
|
-
* Quilt Structure:
|
|
14
|
-
* - quiltId: ID of the entire batch (blob containing all files)
|
|
15
|
-
* - quiltPatchId: Unique ID for each file within the quilt
|
|
16
|
-
* - identifier: Human-readable name for each file
|
|
17
|
-
* - tags: Metadata for filtering (category, importance, etc.)
|
|
18
|
-
*
|
|
19
|
-
* Upload Flow (writeFilesFlow - works with DappKitSigner):
|
|
20
|
-
* 1. encode() - Encode files into blob format (no signature)
|
|
21
|
-
* 2. register() - Register blob on-chain (USER SIGNS - Transaction 1)
|
|
22
|
-
* 3. upload() - Upload to Walrus storage nodes (no signature)
|
|
23
|
-
* 4. certify() - Certify upload on-chain (USER SIGNS - Transaction 2)
|
|
24
|
-
*
|
|
25
|
-
* @see https://sdk.mystenlabs.com/walrus/index
|
|
26
|
-
*/
|
|
27
|
-
|
|
28
|
-
import { WalrusClient, WalrusFile } from '@mysten/walrus';
|
|
29
|
-
import type { ClientWithExtensions } from '@mysten/sui/experimental';
|
|
30
|
-
import type { SuiClient } from '@mysten/sui/client';
|
|
31
|
-
import type { UnifiedSigner } from '../../client/signers/UnifiedSigner';
|
|
32
|
-
|
|
33
|
-
// ============================================================================
|
|
34
|
-
// Types
|
|
35
|
-
// ============================================================================
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
export interface
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
*
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
);
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
identifier
|
|
324
|
-
tags
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
)
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
const
|
|
444
|
-
|
|
445
|
-
console.log(`✅
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
const
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* QuiltBatchManager - Batch Memory Operations via Walrus Quilts
|
|
3
|
+
*
|
|
4
|
+
* Handles batch uploads and queries using Walrus Quilt functionality.
|
|
5
|
+
* Extracted from StorageService for better separation of concerns.
|
|
6
|
+
*
|
|
7
|
+
* Features:
|
|
8
|
+
* - Batch upload with ~90% gas savings (single transaction for multiple files)
|
|
9
|
+
* - Tag-based filtering at the Walrus level
|
|
10
|
+
* - Multi-file retrieval via quiltPatchId
|
|
11
|
+
* - Browser-compatible using writeFilesFlow (2 user signatures)
|
|
12
|
+
*
|
|
13
|
+
* Quilt Structure:
|
|
14
|
+
* - quiltId: ID of the entire batch (blob containing all files)
|
|
15
|
+
* - quiltPatchId: Unique ID for each file within the quilt
|
|
16
|
+
* - identifier: Human-readable name for each file
|
|
17
|
+
* - tags: Metadata for filtering (category, importance, etc.)
|
|
18
|
+
*
|
|
19
|
+
* Upload Flow (writeFilesFlow - works with DappKitSigner):
|
|
20
|
+
* 1. encode() - Encode files into blob format (no signature)
|
|
21
|
+
* 2. register() - Register blob on-chain (USER SIGNS - Transaction 1)
|
|
22
|
+
* 3. upload() - Upload to Walrus storage nodes (no signature)
|
|
23
|
+
* 4. certify() - Certify upload on-chain (USER SIGNS - Transaction 2)
|
|
24
|
+
*
|
|
25
|
+
* @see https://sdk.mystenlabs.com/walrus/index
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { WalrusClient, WalrusFile } from '@mysten/walrus';
|
|
29
|
+
import type { ClientWithExtensions } from '@mysten/sui/experimental';
|
|
30
|
+
import type { SuiClient } from '@mysten/sui/client';
|
|
31
|
+
import type { UnifiedSigner } from '../../client/signers/UnifiedSigner';
|
|
32
|
+
|
|
33
|
+
// ============================================================================
|
|
34
|
+
// Types
|
|
35
|
+
// ============================================================================
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Input for batch memory upload
|
|
39
|
+
*/
|
|
40
|
+
export interface BatchMemory {
|
|
41
|
+
content: string;
|
|
42
|
+
category: string;
|
|
43
|
+
importance: number;
|
|
44
|
+
topic: string;
|
|
45
|
+
embedding: number[];
|
|
46
|
+
encryptedContent?: Uint8Array; // Optional - only when encryption is enabled
|
|
47
|
+
summary?: string;
|
|
48
|
+
id?: string; // Optional client-side ID for tracking
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Memory package stored in Quilt as JSON
|
|
53
|
+
* This format is consistent with regular memory storage
|
|
54
|
+
*/
|
|
55
|
+
export interface QuiltMemoryPackage {
|
|
56
|
+
content: string; // Plaintext content (empty if encrypted)
|
|
57
|
+
embedding: number[]; // Vector embedding
|
|
58
|
+
metadata: {
|
|
59
|
+
category: string;
|
|
60
|
+
importance: number;
|
|
61
|
+
topic: string;
|
|
62
|
+
[key: string]: unknown;
|
|
63
|
+
};
|
|
64
|
+
timestamp: number;
|
|
65
|
+
version: string; // Package format version
|
|
66
|
+
encrypted?: boolean; // Whether content is encrypted
|
|
67
|
+
encryptedContent?: string; // Base64-encoded encrypted content (if encrypted)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface QuiltUploadOptions {
|
|
71
|
+
signer: UnifiedSigner;
|
|
72
|
+
epochs?: number;
|
|
73
|
+
userAddress: string;
|
|
74
|
+
deletable?: boolean;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface QuiltFileResult {
|
|
78
|
+
identifier: string;
|
|
79
|
+
blobId: string;
|
|
80
|
+
quiltPatchId?: string;
|
|
81
|
+
tags: Record<string, string>;
|
|
82
|
+
size: number;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface QuiltUploadResult {
|
|
86
|
+
quiltId: string;
|
|
87
|
+
blobObjectId?: string;
|
|
88
|
+
files: QuiltFileResult[];
|
|
89
|
+
uploadTimeMs: number;
|
|
90
|
+
totalSize: number;
|
|
91
|
+
gasSaved: string; // Percentage saved vs individual uploads
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface QuiltRetrieveResult {
|
|
95
|
+
identifier: string;
|
|
96
|
+
content: Uint8Array;
|
|
97
|
+
tags: Record<string, string>;
|
|
98
|
+
retrievalTimeMs: number;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Result when retrieving a memory package from Quilt
|
|
103
|
+
*/
|
|
104
|
+
export interface QuiltMemoryRetrieveResult {
|
|
105
|
+
identifier: string;
|
|
106
|
+
memoryPackage: QuiltMemoryPackage;
|
|
107
|
+
tags: Record<string, string>;
|
|
108
|
+
retrievalTimeMs: number;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export interface QuiltListResult {
|
|
112
|
+
identifier: string;
|
|
113
|
+
quiltPatchId: string;
|
|
114
|
+
tags: Record<string, string>;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ============================================================================
|
|
118
|
+
// QuiltBatchManager
|
|
119
|
+
// ============================================================================
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* QuiltBatchManager - Manages batch memory operations via Quilts
|
|
123
|
+
*
|
|
124
|
+
* Quilts provide:
|
|
125
|
+
* - Multi-file uploads in single transaction (~90% gas savings)
|
|
126
|
+
* - Tag-based metadata for filtering
|
|
127
|
+
* - Efficient retrieval via identifier or quiltPatchId
|
|
128
|
+
*
|
|
129
|
+
* @example
|
|
130
|
+
* ```typescript
|
|
131
|
+
* const manager = new QuiltBatchManager(walrus, sui, true, 3);
|
|
132
|
+
*
|
|
133
|
+
* // Upload batch
|
|
134
|
+
* const result = await manager.uploadMemoryBatch(memories, { signer, userAddress });
|
|
135
|
+
*
|
|
136
|
+
* // Retrieve all files
|
|
137
|
+
* const files = await manager.getQuiltFiles(result.quiltId);
|
|
138
|
+
*
|
|
139
|
+
* // Retrieve by identifier
|
|
140
|
+
* const file = await manager.getFileByIdentifier(result.quiltId, 'memory-123.json');
|
|
141
|
+
*
|
|
142
|
+
* // Filter by tags
|
|
143
|
+
* const facts = await manager.getQuiltFilesByTags(result.quiltId, [{ category: 'fact' }]);
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
export class QuiltBatchManager {
|
|
147
|
+
constructor(
|
|
148
|
+
private walrusWithRelay: WalrusClient,
|
|
149
|
+
private walrusWithoutRelay: WalrusClient,
|
|
150
|
+
private suiClient: ClientWithExtensions<{ jsonRpc: SuiClient; walrus: WalrusClient }>,
|
|
151
|
+
private useUploadRelay: boolean,
|
|
152
|
+
private epochs: number
|
|
153
|
+
) {}
|
|
154
|
+
|
|
155
|
+
// ==========================================================================
|
|
156
|
+
// Upload Operations
|
|
157
|
+
// ==========================================================================
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Upload batch of memories as a Quilt using writeFilesFlow
|
|
161
|
+
*
|
|
162
|
+
* Uses the writeFilesFlow pattern which works with DappKitSigner:
|
|
163
|
+
* 1. encode() - Encode files (no signature)
|
|
164
|
+
* 2. register() - Register blob on-chain (USER SIGNS)
|
|
165
|
+
* 3. upload() - Upload to storage nodes (no signature)
|
|
166
|
+
* 4. certify() - Certify upload on-chain (USER SIGNS)
|
|
167
|
+
*
|
|
168
|
+
* Each memory becomes a WalrusFile with:
|
|
169
|
+
* - Identifier: unique file name (memory-{timestamp}-{index}-{random}.json)
|
|
170
|
+
* - Tags: plaintext metadata (searchable)
|
|
171
|
+
* - Content: encrypted data (Uint8Array)
|
|
172
|
+
*
|
|
173
|
+
* @param memories - Array of BatchMemory objects
|
|
174
|
+
* @param options - Upload options including signer and userAddress
|
|
175
|
+
* @returns QuiltUploadResult with quiltId and file details
|
|
176
|
+
*/
|
|
177
|
+
async uploadMemoryBatch(
|
|
178
|
+
memories: BatchMemory[],
|
|
179
|
+
options: QuiltUploadOptions
|
|
180
|
+
): Promise<QuiltUploadResult> {
|
|
181
|
+
const startTime = performance.now();
|
|
182
|
+
let totalSize = 0;
|
|
183
|
+
|
|
184
|
+
console.log(`📦 Uploading batch of ${memories.length} memories as Quilt (writeFilesFlow)...`);
|
|
185
|
+
|
|
186
|
+
try {
|
|
187
|
+
// Create WalrusFile for each memory as JSON package
|
|
188
|
+
// This format is consistent with regular memory storage
|
|
189
|
+
const files = memories.map((memory, index) => {
|
|
190
|
+
const identifier = memory.id
|
|
191
|
+
? `memory-${memory.id}.json`
|
|
192
|
+
: `memory-${Date.now()}-${index}-${Math.random().toString(36).slice(2, 9)}.json`;
|
|
193
|
+
|
|
194
|
+
const isEncrypted = !!memory.encryptedContent && memory.encryptedContent.length > 0;
|
|
195
|
+
const timestamp = Date.now();
|
|
196
|
+
|
|
197
|
+
// Create memory package (JSON format - consistent with regular storage)
|
|
198
|
+
const memoryPackage: QuiltMemoryPackage = {
|
|
199
|
+
// Content: plaintext if not encrypted, empty if encrypted
|
|
200
|
+
content: isEncrypted ? '' : memory.content,
|
|
201
|
+
embedding: memory.embedding,
|
|
202
|
+
metadata: {
|
|
203
|
+
category: memory.category,
|
|
204
|
+
importance: memory.importance,
|
|
205
|
+
topic: memory.topic,
|
|
206
|
+
...(memory.summary ? { summary: memory.summary } : {}),
|
|
207
|
+
...(memory.id ? { memoryId: memory.id } : {})
|
|
208
|
+
},
|
|
209
|
+
timestamp,
|
|
210
|
+
version: '2.0.0', // Quilt JSON package version
|
|
211
|
+
encrypted: isEncrypted,
|
|
212
|
+
// Store encrypted content as base64 for JSON compatibility
|
|
213
|
+
...(isEncrypted && memory.encryptedContent ? {
|
|
214
|
+
encryptedContent: this.uint8ArrayToBase64(memory.encryptedContent)
|
|
215
|
+
} : {})
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
// Serialize to JSON and encode as bytes
|
|
219
|
+
const jsonString = JSON.stringify(memoryPackage);
|
|
220
|
+
const contents = new TextEncoder().encode(jsonString);
|
|
221
|
+
totalSize += contents.length;
|
|
222
|
+
|
|
223
|
+
// Diagnostic logging for debugging Quilt corruption issues
|
|
224
|
+
console.log(` 📝 File ${index}: identifier=${identifier}`);
|
|
225
|
+
console.log(` JSON string length: ${jsonString.length} chars`);
|
|
226
|
+
console.log(` Encoded bytes: ${contents.length} bytes`);
|
|
227
|
+
console.log(` Last 50 chars of JSON: ...${jsonString.slice(-50)}`);
|
|
228
|
+
console.log(` Last 10 bytes (hex): ${Array.from(contents.slice(-10)).map(b => b.toString(16).padStart(2, '0')).join(' ')}`);
|
|
229
|
+
|
|
230
|
+
return WalrusFile.from({
|
|
231
|
+
contents,
|
|
232
|
+
identifier,
|
|
233
|
+
tags: {
|
|
234
|
+
// Core metadata (plaintext for filtering without decryption)
|
|
235
|
+
'content-type': 'application/json',
|
|
236
|
+
'category': memory.category,
|
|
237
|
+
'importance': memory.importance.toString(),
|
|
238
|
+
'topic': memory.topic,
|
|
239
|
+
'timestamp': new Date(timestamp).toISOString(),
|
|
240
|
+
'created_at': new Date(timestamp).toISOString(),
|
|
241
|
+
|
|
242
|
+
// Encryption info
|
|
243
|
+
'encrypted': isEncrypted ? 'true' : 'false',
|
|
244
|
+
...(isEncrypted ? { 'encryption_type': 'seal' } : {}),
|
|
245
|
+
|
|
246
|
+
// Owner
|
|
247
|
+
'owner': options.userAddress,
|
|
248
|
+
|
|
249
|
+
// Content info
|
|
250
|
+
'content_size': contents.length.toString(),
|
|
251
|
+
'embedding_dimensions': memory.embedding.length.toString(),
|
|
252
|
+
'package_version': '2.0.0',
|
|
253
|
+
|
|
254
|
+
// Optional rich metadata
|
|
255
|
+
...(memory.summary ? { 'summary': memory.summary } : {}),
|
|
256
|
+
...(memory.id ? { 'memory_id': memory.id } : {})
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
console.log(` Created ${files.length} WalrusFiles with plaintext tags`);
|
|
262
|
+
console.log(` Total size: ${(totalSize / 1024).toFixed(2)} KB`);
|
|
263
|
+
console.log(` Using upload relay: ${this.useUploadRelay}`);
|
|
264
|
+
|
|
265
|
+
// Use writeFilesFlow pattern (works with DappKitSigner)
|
|
266
|
+
const walrusClient = this.useUploadRelay
|
|
267
|
+
? this.walrusWithRelay
|
|
268
|
+
: this.walrusWithoutRelay;
|
|
269
|
+
|
|
270
|
+
// Step 1: Create flow and encode files (no signature needed)
|
|
271
|
+
console.log(` Step 1/4: Encoding files...`);
|
|
272
|
+
const flow = walrusClient.writeFilesFlow({ files });
|
|
273
|
+
await flow.encode();
|
|
274
|
+
console.log(` ✓ Files encoded`);
|
|
275
|
+
|
|
276
|
+
// Step 2: Register blob on-chain (USER SIGNS - Transaction 1)
|
|
277
|
+
console.log(` Step 2/4: Registering blob (requires signature)...`);
|
|
278
|
+
const registerTx = flow.register({
|
|
279
|
+
epochs: options.epochs || this.epochs,
|
|
280
|
+
owner: options.userAddress,
|
|
281
|
+
deletable: options.deletable ?? true
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
const registerResult = await options.signer.signAndExecuteTransaction(registerTx);
|
|
285
|
+
console.log(` ✓ Blob registered, digest: ${registerResult.digest}`);
|
|
286
|
+
|
|
287
|
+
// Step 3: Upload to Walrus storage nodes (no signature needed)
|
|
288
|
+
console.log(` Step 3/4: Uploading to storage nodes...`);
|
|
289
|
+
await flow.upload({ digest: registerResult.digest });
|
|
290
|
+
console.log(` ✓ Uploaded to storage nodes`);
|
|
291
|
+
|
|
292
|
+
// Step 4: Certify upload on-chain (USER SIGNS - Transaction 2)
|
|
293
|
+
console.log(` Step 4/4: Certifying upload (requires signature)...`);
|
|
294
|
+
const certifyTx = flow.certify();
|
|
295
|
+
|
|
296
|
+
if (certifyTx) {
|
|
297
|
+
const certifyResult = await options.signer.signAndExecuteTransaction(certifyTx);
|
|
298
|
+
console.log(` ✓ Upload certified, digest: ${certifyResult.digest}`);
|
|
299
|
+
} else {
|
|
300
|
+
console.log(` ✓ No certification needed (already certified)`);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// Get uploaded files info from flow
|
|
304
|
+
const uploadedFilesInfo = await flow.listFiles();
|
|
305
|
+
|
|
306
|
+
const uploadTimeMs = performance.now() - startTime;
|
|
307
|
+
const gasSaved = memories.length > 1
|
|
308
|
+
? `~${((1 - 1 / memories.length) * 100).toFixed(0)}%`
|
|
309
|
+
: '0%';
|
|
310
|
+
|
|
311
|
+
console.log(`✅ Quilt upload successful!`);
|
|
312
|
+
console.log(` Files uploaded: ${uploadedFilesInfo.length}`);
|
|
313
|
+
console.log(` Upload time: ${uploadTimeMs.toFixed(1)}ms`);
|
|
314
|
+
console.log(` Gas saved: ${gasSaved} vs individual uploads`);
|
|
315
|
+
|
|
316
|
+
// Build file results using original WalrusFile objects for metadata
|
|
317
|
+
// Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
|
|
318
|
+
// Match files by identifier when reading
|
|
319
|
+
const quiltId = uploadedFilesInfo[0]?.blobId || '';
|
|
320
|
+
|
|
321
|
+
const fileResults: QuiltFileResult[] = await Promise.all(
|
|
322
|
+
files.map(async (originalFile, i) => {
|
|
323
|
+
const identifier = await originalFile.getIdentifier() || `file-${i}`;
|
|
324
|
+
const tags = await originalFile.getTags() || {};
|
|
325
|
+
const fileInfo = uploadedFilesInfo[i];
|
|
326
|
+
|
|
327
|
+
// quiltPatchId is stored for reference but not used for retrieval
|
|
328
|
+
const quiltPatchId = fileInfo?.id || '';
|
|
329
|
+
|
|
330
|
+
console.log(` File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
|
|
331
|
+
|
|
332
|
+
return {
|
|
333
|
+
identifier,
|
|
334
|
+
// Use shared quiltId as blobId - read via getBlob(quiltId).files()
|
|
335
|
+
blobId: quiltId,
|
|
336
|
+
quiltPatchId,
|
|
337
|
+
tags: Object.fromEntries(
|
|
338
|
+
Object.entries(tags).map(([k, v]) => [k, String(v)])
|
|
339
|
+
),
|
|
340
|
+
size: memories[i]?.encryptedContent?.length || memories[i]?.content?.length || 0
|
|
341
|
+
};
|
|
342
|
+
})
|
|
343
|
+
);
|
|
344
|
+
|
|
345
|
+
return {
|
|
346
|
+
quiltId,
|
|
347
|
+
blobObjectId: undefined, // Not available from flow
|
|
348
|
+
files: fileResults,
|
|
349
|
+
uploadTimeMs,
|
|
350
|
+
totalSize,
|
|
351
|
+
gasSaved
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
} catch (error) {
|
|
355
|
+
console.error(`❌ Quilt batch upload failed:`, error);
|
|
356
|
+
throw new Error(`Quilt batch upload failed: ${error}`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Upload raw files as a Quilt using writeFilesFlow
|
|
362
|
+
*
|
|
363
|
+
* Uses the writeFilesFlow pattern which works with DappKitSigner:
|
|
364
|
+
* 1. encode() - Encode files (no signature)
|
|
365
|
+
* 2. register() - Register blob on-chain (USER SIGNS)
|
|
366
|
+
* 3. upload() - Upload to storage nodes (no signature)
|
|
367
|
+
* 4. certify() - Certify upload on-chain (USER SIGNS)
|
|
368
|
+
*
|
|
369
|
+
* @param files - Array of { identifier, data, tags }
|
|
370
|
+
* @param options - Upload options
|
|
371
|
+
* @returns QuiltUploadResult
|
|
372
|
+
*/
|
|
373
|
+
async uploadFilesBatch(
|
|
374
|
+
files: Array<{
|
|
375
|
+
identifier: string;
|
|
376
|
+
data: Uint8Array;
|
|
377
|
+
tags?: Record<string, string>;
|
|
378
|
+
}>,
|
|
379
|
+
options: QuiltUploadOptions
|
|
380
|
+
): Promise<QuiltUploadResult> {
|
|
381
|
+
const startTime = performance.now();
|
|
382
|
+
let totalSize = 0;
|
|
383
|
+
|
|
384
|
+
console.log(`📁 Uploading ${files.length} files as Quilt (writeFilesFlow)...`);
|
|
385
|
+
|
|
386
|
+
try {
|
|
387
|
+
const walrusFiles = files.map(file => {
|
|
388
|
+
totalSize += file.data.length;
|
|
389
|
+
|
|
390
|
+
return WalrusFile.from({
|
|
391
|
+
contents: file.data,
|
|
392
|
+
identifier: file.identifier,
|
|
393
|
+
tags: {
|
|
394
|
+
'timestamp': new Date().toISOString(),
|
|
395
|
+
'owner': options.userAddress,
|
|
396
|
+
'content_size': file.data.length.toString(),
|
|
397
|
+
...(file.tags || {})
|
|
398
|
+
}
|
|
399
|
+
});
|
|
400
|
+
});
|
|
401
|
+
|
|
402
|
+
// Use writeFilesFlow pattern (works with DappKitSigner)
|
|
403
|
+
const walrusClient = this.useUploadRelay
|
|
404
|
+
? this.walrusWithRelay
|
|
405
|
+
: this.walrusWithoutRelay;
|
|
406
|
+
|
|
407
|
+
// Step 1: Create flow and encode files (no signature needed)
|
|
408
|
+
console.log(` Step 1/4: Encoding files...`);
|
|
409
|
+
const flow = walrusClient.writeFilesFlow({ files: walrusFiles });
|
|
410
|
+
await flow.encode();
|
|
411
|
+
console.log(` ✓ Files encoded`);
|
|
412
|
+
|
|
413
|
+
// Step 2: Register blob on-chain (USER SIGNS - Transaction 1)
|
|
414
|
+
console.log(` Step 2/4: Registering blob (requires signature)...`);
|
|
415
|
+
const registerTx = flow.register({
|
|
416
|
+
epochs: options.epochs || this.epochs,
|
|
417
|
+
owner: options.userAddress,
|
|
418
|
+
deletable: options.deletable ?? true
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
const registerResult = await options.signer.signAndExecuteTransaction(registerTx);
|
|
422
|
+
console.log(` ✓ Blob registered, digest: ${registerResult.digest}`);
|
|
423
|
+
|
|
424
|
+
// Step 3: Upload to Walrus storage nodes (no signature needed)
|
|
425
|
+
console.log(` Step 3/4: Uploading to storage nodes...`);
|
|
426
|
+
await flow.upload({ digest: registerResult.digest });
|
|
427
|
+
console.log(` ✓ Uploaded to storage nodes`);
|
|
428
|
+
|
|
429
|
+
// Step 4: Certify upload on-chain (USER SIGNS - Transaction 2)
|
|
430
|
+
console.log(` Step 4/4: Certifying upload (requires signature)...`);
|
|
431
|
+
const certifyTx = flow.certify();
|
|
432
|
+
|
|
433
|
+
if (certifyTx) {
|
|
434
|
+
const certifyResult = await options.signer.signAndExecuteTransaction(certifyTx);
|
|
435
|
+
console.log(` ✓ Upload certified, digest: ${certifyResult.digest}`);
|
|
436
|
+
} else {
|
|
437
|
+
console.log(` ✓ No certification needed (already certified)`);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// Get uploaded files info from flow
|
|
441
|
+
const uploadedFilesInfo = await flow.listFiles();
|
|
442
|
+
|
|
443
|
+
const uploadTimeMs = performance.now() - startTime;
|
|
444
|
+
|
|
445
|
+
console.log(`✅ Files batch upload successful!`);
|
|
446
|
+
console.log(` Files uploaded: ${uploadedFilesInfo.length}`);
|
|
447
|
+
console.log(` Upload time: ${uploadTimeMs.toFixed(1)}ms`);
|
|
448
|
+
|
|
449
|
+
// Build file results using original WalrusFile objects for metadata
|
|
450
|
+
// Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
|
|
451
|
+
// Match files by identifier when reading
|
|
452
|
+
const quiltId = uploadedFilesInfo[0]?.blobId || '';
|
|
453
|
+
|
|
454
|
+
const fileResults: QuiltFileResult[] = await Promise.all(
|
|
455
|
+
walrusFiles.map(async (originalFile, i) => {
|
|
456
|
+
const identifier = await originalFile.getIdentifier() || files[i]?.identifier || `file-${i}`;
|
|
457
|
+
const tags = await originalFile.getTags() || {};
|
|
458
|
+
const fileInfo = uploadedFilesInfo[i];
|
|
459
|
+
|
|
460
|
+
// quiltPatchId is stored for reference but not used for retrieval
|
|
461
|
+
const quiltPatchId = fileInfo?.id || '';
|
|
462
|
+
|
|
463
|
+
console.log(` File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
|
|
464
|
+
|
|
465
|
+
return {
|
|
466
|
+
identifier,
|
|
467
|
+
// Use shared quiltId as blobId - read via getBlob(quiltId).files()
|
|
468
|
+
blobId: quiltId,
|
|
469
|
+
quiltPatchId,
|
|
470
|
+
tags: Object.fromEntries(
|
|
471
|
+
Object.entries(tags).map(([k, v]) => [k, String(v)])
|
|
472
|
+
),
|
|
473
|
+
size: files[i]?.data.length || 0
|
|
474
|
+
};
|
|
475
|
+
})
|
|
476
|
+
);
|
|
477
|
+
|
|
478
|
+
return {
|
|
479
|
+
quiltId,
|
|
480
|
+
blobObjectId: undefined, // Not available from flow
|
|
481
|
+
files: fileResults,
|
|
482
|
+
uploadTimeMs,
|
|
483
|
+
totalSize,
|
|
484
|
+
gasSaved: files.length > 1 ? `~${((1 - 1 / files.length) * 100).toFixed(0)}%` : '0%'
|
|
485
|
+
};
|
|
486
|
+
|
|
487
|
+
} catch (error) {
|
|
488
|
+
console.error(`❌ Files batch upload failed:`, error);
|
|
489
|
+
throw new Error(`Files batch upload failed: ${error}`);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// ==========================================================================
|
|
494
|
+
// Retrieval Operations
|
|
495
|
+
// ==========================================================================
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Retrieve all files from a Quilt
|
|
499
|
+
*
|
|
500
|
+
* Uses getBlob().files() pattern which correctly parses Quilt structure
|
|
501
|
+
* and returns individual files with their identifiers and tags.
|
|
502
|
+
*
|
|
503
|
+
* @param quiltId - The Quilt blob ID (shared blobId)
|
|
504
|
+
* @returns Array of WalrusFile objects
|
|
505
|
+
*/
|
|
506
|
+
async getQuiltFiles(quiltId: string): Promise<Array<WalrusFile>> {
|
|
507
|
+
try {
|
|
508
|
+
console.log(`📂 Retrieving files from Quilt ${quiltId}...`);
|
|
509
|
+
|
|
510
|
+
// Try to parse as Quilt first (getBlob().files() returns ALL files in Quilt)
|
|
511
|
+
// Fall back to getFiles() for regular blobs
|
|
512
|
+
let files: WalrusFile[];
|
|
513
|
+
try {
|
|
514
|
+
const blob = await this.suiClient.walrus.getBlob({ blobId: quiltId });
|
|
515
|
+
files = await blob.files();
|
|
516
|
+
console.log(`✅ Retrieved ${files.length} files from Quilt`);
|
|
517
|
+
} catch (quiltError: any) {
|
|
518
|
+
// Not a Quilt - try as regular blob
|
|
519
|
+
console.log(`📄 Not a Quilt format, fetching as regular blob...`);
|
|
520
|
+
files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
|
|
521
|
+
console.log(`✅ Retrieved ${files.length} file(s) as regular blob`);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
return files;
|
|
525
|
+
|
|
526
|
+
} catch (error) {
|
|
527
|
+
console.error(`❌ Failed to retrieve Quilt files:`, error);
|
|
528
|
+
throw new Error(`Failed to retrieve Quilt ${quiltId}: ${error}`);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/**
|
|
533
|
+
* Retrieve a specific file by identifier from a Quilt
|
|
534
|
+
*
|
|
535
|
+
* Uses getBlob().files() to get all files then matches by identifier.
|
|
536
|
+
*
|
|
537
|
+
* @param quiltId - The Quilt blob ID (shared blobId)
|
|
538
|
+
* @param identifier - The file identifier within the quilt
|
|
539
|
+
* @returns QuiltRetrieveResult with content and metadata
|
|
540
|
+
*/
|
|
541
|
+
async getFileByIdentifier(
|
|
542
|
+
quiltId: string,
|
|
543
|
+
identifier: string
|
|
544
|
+
): Promise<QuiltRetrieveResult> {
|
|
545
|
+
const startTime = performance.now();
|
|
546
|
+
|
|
547
|
+
try {
|
|
548
|
+
console.log(`📄 Retrieving file "${identifier}" from Quilt ${quiltId}...`);
|
|
549
|
+
|
|
550
|
+
// Get all files from the blob (Quilt or regular)
|
|
551
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
552
|
+
|
|
553
|
+
// Find file by identifier
|
|
554
|
+
let matchingFile: WalrusFile | undefined;
|
|
555
|
+
for (const f of files) {
|
|
556
|
+
const fileIdentifier = await f.getIdentifier();
|
|
557
|
+
if (fileIdentifier === identifier) {
|
|
558
|
+
matchingFile = f;
|
|
559
|
+
break;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
if (!matchingFile) {
|
|
564
|
+
throw new Error(`File "${identifier}" not found in Quilt`);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
const content = await matchingFile.bytes();
|
|
568
|
+
const tags = await matchingFile.getTags();
|
|
569
|
+
const retrievalTimeMs = performance.now() - startTime;
|
|
570
|
+
|
|
571
|
+
console.log(`✅ Retrieved file "${identifier}" (${content.length} bytes)`);
|
|
572
|
+
|
|
573
|
+
return {
|
|
574
|
+
identifier,
|
|
575
|
+
content,
|
|
576
|
+
tags,
|
|
577
|
+
retrievalTimeMs
|
|
578
|
+
};
|
|
579
|
+
|
|
580
|
+
} catch (error) {
|
|
581
|
+
console.error(`❌ Failed to retrieve file by identifier:`, error);
|
|
582
|
+
throw new Error(`Failed to retrieve "${identifier}" from Quilt: ${error}`);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* List all patches in a Quilt with their metadata
|
|
588
|
+
*
|
|
589
|
+
* Uses getBlob().files() to correctly parse Quilt structure.
|
|
590
|
+
*
|
|
591
|
+
* @param quiltId - The Quilt blob ID (shared blobId)
|
|
592
|
+
* @returns Array of QuiltListResult with identifiers and tags
|
|
593
|
+
*/
|
|
594
|
+
async listQuiltPatches(quiltId: string): Promise<QuiltListResult[]> {
|
|
595
|
+
try {
|
|
596
|
+
console.log(`📋 Listing patches in Quilt ${quiltId}...`);
|
|
597
|
+
|
|
598
|
+
// Get all files from the blob (Quilt or regular)
|
|
599
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
600
|
+
|
|
601
|
+
const results: QuiltListResult[] = await Promise.all(
|
|
602
|
+
files.map(async (file) => {
|
|
603
|
+
const identifier = await file.getIdentifier() || 'unknown';
|
|
604
|
+
const tags = await file.getTags();
|
|
605
|
+
|
|
606
|
+
return {
|
|
607
|
+
identifier,
|
|
608
|
+
quiltPatchId: '', // Would need API to get this
|
|
609
|
+
tags
|
|
610
|
+
};
|
|
611
|
+
})
|
|
612
|
+
);
|
|
613
|
+
|
|
614
|
+
console.log(`✅ Found ${results.length} patches in Quilt`);
|
|
615
|
+
|
|
616
|
+
return results;
|
|
617
|
+
|
|
618
|
+
} catch (error) {
|
|
619
|
+
console.error(`❌ Failed to list Quilt patches:`, error);
|
|
620
|
+
throw new Error(`Failed to list patches in Quilt ${quiltId}: ${error}`);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// ==========================================================================
|
|
625
|
+
// Query Operations
|
|
626
|
+
// ==========================================================================
|
|
627
|
+
|
|
628
|
+
/**
|
|
629
|
+
* Query Quilt files by tags (client-side filtering)
|
|
630
|
+
*
|
|
631
|
+
* @param quiltId - The Quilt blob ID
|
|
632
|
+
* @param tagFilters - Array of tag key-value pairs to match
|
|
633
|
+
* @returns Array of matching WalrusFile objects
|
|
634
|
+
*/
|
|
635
|
+
async getQuiltFilesByTags(
|
|
636
|
+
quiltId: string,
|
|
637
|
+
tagFilters: Array<Record<string, string>>
|
|
638
|
+
): Promise<Array<WalrusFile>> {
|
|
639
|
+
try {
|
|
640
|
+
console.log(`🔍 Querying Quilt ${quiltId} with tag filters:`, tagFilters);
|
|
641
|
+
|
|
642
|
+
// Fetch all files
|
|
643
|
+
const allFiles = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
|
|
644
|
+
|
|
645
|
+
// Client-side tag filtering
|
|
646
|
+
const matchingFiles: WalrusFile[] = [];
|
|
647
|
+
|
|
648
|
+
for (const file of allFiles) {
|
|
649
|
+
const fileTags = await file.getTags();
|
|
650
|
+
|
|
651
|
+
// Check if file matches any of the tag filters
|
|
652
|
+
const matches = tagFilters.some(filter => {
|
|
653
|
+
return Object.entries(filter).every(([key, value]) => {
|
|
654
|
+
return fileTags[key] === value;
|
|
655
|
+
});
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
if (matches) {
|
|
659
|
+
matchingFiles.push(file);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
console.log(`✅ Found ${matchingFiles.length} matching files out of ${allFiles.length}`);
|
|
664
|
+
|
|
665
|
+
return matchingFiles;
|
|
666
|
+
|
|
667
|
+
} catch (error) {
|
|
668
|
+
console.error(`❌ Quilt query failed:`, error);
|
|
669
|
+
throw new Error(`Failed to query Quilt ${quiltId}: ${error}`);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Query files by category
|
|
675
|
+
*
|
|
676
|
+
* @param quiltId - The Quilt blob ID
|
|
677
|
+
* @param category - Category to filter by
|
|
678
|
+
* @returns Array of matching WalrusFile objects
|
|
679
|
+
*/
|
|
680
|
+
async getFilesByCategory(
|
|
681
|
+
quiltId: string,
|
|
682
|
+
category: string
|
|
683
|
+
): Promise<Array<WalrusFile>> {
|
|
684
|
+
return this.getQuiltFilesByTags(quiltId, [{ category }]);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Query files by importance threshold
|
|
689
|
+
*
|
|
690
|
+
* @param quiltId - The Quilt blob ID
|
|
691
|
+
* @param minImportance - Minimum importance value
|
|
692
|
+
* @returns Array of matching WalrusFile objects
|
|
693
|
+
*/
|
|
694
|
+
async getFilesByImportance(
|
|
695
|
+
quiltId: string,
|
|
696
|
+
minImportance: number
|
|
697
|
+
): Promise<Array<WalrusFile>> {
|
|
698
|
+
const allFiles = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
|
|
699
|
+
const matchingFiles: WalrusFile[] = [];
|
|
700
|
+
|
|
701
|
+
for (const file of allFiles) {
|
|
702
|
+
const tags = await file.getTags();
|
|
703
|
+
const importance = parseInt(tags['importance'] || '0', 10);
|
|
704
|
+
|
|
705
|
+
if (importance >= minImportance) {
|
|
706
|
+
matchingFiles.push(file);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
return matchingFiles;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
// ==========================================================================
|
|
714
|
+
// JSON Memory Package Retrieval
|
|
715
|
+
// ==========================================================================
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* Retrieve a memory package as JSON from a Quilt
|
|
719
|
+
*
|
|
720
|
+
* Uses file.json() for efficient parsing (SDK handles it)
|
|
721
|
+
*
|
|
722
|
+
* @param quiltId - The Quilt blob ID
|
|
723
|
+
* @param identifier - The file identifier within the quilt
|
|
724
|
+
* @returns QuiltMemoryRetrieveResult with parsed memory package
|
|
725
|
+
*/
|
|
726
|
+
async getMemoryPackage(
|
|
727
|
+
quiltId: string,
|
|
728
|
+
identifier: string
|
|
729
|
+
): Promise<QuiltMemoryRetrieveResult> {
|
|
730
|
+
const startTime = performance.now();
|
|
731
|
+
|
|
732
|
+
try {
|
|
733
|
+
console.log(`📄 Retrieving memory package "${identifier}" from Quilt ${quiltId}...`);
|
|
734
|
+
|
|
735
|
+
// Get all files from the blob
|
|
736
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
737
|
+
|
|
738
|
+
// Find file by identifier
|
|
739
|
+
let matchingFile: WalrusFile | undefined;
|
|
740
|
+
for (const f of files) {
|
|
741
|
+
const fileIdentifier = await f.getIdentifier();
|
|
742
|
+
if (fileIdentifier === identifier) {
|
|
743
|
+
matchingFile = f;
|
|
744
|
+
break;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
if (!matchingFile) {
|
|
749
|
+
throw new Error(`File "${identifier}" not found in Quilt`);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
const tags = await matchingFile.getTags();
|
|
753
|
+
let memoryPackage: QuiltMemoryPackage;
|
|
754
|
+
|
|
755
|
+
try {
|
|
756
|
+
// Parse directly as JSON (SDK handles it!)
|
|
757
|
+
memoryPackage = await matchingFile.json() as QuiltMemoryPackage;
|
|
758
|
+
} catch (parseError) {
|
|
759
|
+
// Try partial recovery for truncated JSON
|
|
760
|
+
console.warn(`⚠️ JSON parse failed for "${identifier}", attempting recovery...`);
|
|
761
|
+
const bytes = await matchingFile.bytes();
|
|
762
|
+
const recovered = this.tryRecoverTruncatedPackage(bytes);
|
|
763
|
+
if (recovered) {
|
|
764
|
+
console.log(`🔧 Partially recovered "${identifier}" (encryptedContent may be corrupted)`);
|
|
765
|
+
memoryPackage = recovered;
|
|
766
|
+
} else {
|
|
767
|
+
throw parseError;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
const retrievalTimeMs = performance.now() - startTime;
|
|
772
|
+
|
|
773
|
+
console.log(`✅ Retrieved memory package "${identifier}" (${retrievalTimeMs.toFixed(1)}ms)`);
|
|
774
|
+
|
|
775
|
+
return {
|
|
776
|
+
identifier,
|
|
777
|
+
memoryPackage,
|
|
778
|
+
tags,
|
|
779
|
+
retrievalTimeMs
|
|
780
|
+
};
|
|
781
|
+
|
|
782
|
+
} catch (error) {
|
|
783
|
+
console.error(`❌ Failed to retrieve memory package:`, error);
|
|
784
|
+
throw new Error(`Failed to retrieve memory package "${identifier}": ${error}`);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Retrieve all memory packages from a Quilt as JSON
|
|
790
|
+
*
|
|
791
|
+
* @param quiltId - The Quilt blob ID
|
|
792
|
+
* @returns Array of memory packages with metadata
|
|
793
|
+
*/
|
|
794
|
+
async getAllMemoryPackages(quiltId: string): Promise<QuiltMemoryRetrieveResult[]> {
|
|
795
|
+
const startTime = performance.now();
|
|
796
|
+
|
|
797
|
+
try {
|
|
798
|
+
console.log(`📂 Retrieving all memory packages from Quilt ${quiltId}...`);
|
|
799
|
+
|
|
800
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
801
|
+
const results: QuiltMemoryRetrieveResult[] = [];
|
|
802
|
+
|
|
803
|
+
for (const file of files) {
|
|
804
|
+
const identifier = await file.getIdentifier() || 'unknown';
|
|
805
|
+
const tags = await file.getTags();
|
|
806
|
+
|
|
807
|
+
try {
|
|
808
|
+
// Parse as JSON
|
|
809
|
+
const memoryPackage = await file.json() as QuiltMemoryPackage;
|
|
810
|
+
results.push({
|
|
811
|
+
identifier,
|
|
812
|
+
memoryPackage,
|
|
813
|
+
tags,
|
|
814
|
+
retrievalTimeMs: 0 // Individual timing not tracked in batch
|
|
815
|
+
});
|
|
816
|
+
} catch (parseError) {
|
|
817
|
+
console.warn(`⚠️ Failed to parse "${identifier}" as JSON:`, parseError);
|
|
818
|
+
|
|
819
|
+
// Try partial recovery for truncated JSON
|
|
820
|
+
try {
|
|
821
|
+
const bytes = await file.bytes();
|
|
822
|
+
const recoveredPackage = this.tryRecoverTruncatedPackage(bytes);
|
|
823
|
+
if (recoveredPackage) {
|
|
824
|
+
console.log(`🔧 Partially recovered "${identifier}" (encryptedContent truncated)`);
|
|
825
|
+
results.push({
|
|
826
|
+
identifier,
|
|
827
|
+
memoryPackage: recoveredPackage,
|
|
828
|
+
tags,
|
|
829
|
+
retrievalTimeMs: 0
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
} catch {
|
|
833
|
+
// Skip files that can't be recovered
|
|
834
|
+
console.warn(`❌ Could not recover "${identifier}"`);
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
const totalTimeMs = performance.now() - startTime;
|
|
840
|
+
console.log(`✅ Retrieved ${results.length} memory packages (${totalTimeMs.toFixed(1)}ms)`);
|
|
841
|
+
|
|
842
|
+
return results;
|
|
843
|
+
|
|
844
|
+
} catch (error) {
|
|
845
|
+
console.error(`❌ Failed to retrieve memory packages:`, error);
|
|
846
|
+
throw new Error(`Failed to retrieve memory packages from Quilt ${quiltId}: ${error}`);
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Find a specific memory in a Quilt using multiple matching strategies
|
|
852
|
+
*
|
|
853
|
+
* Strategies (in order of priority):
|
|
854
|
+
* 1. Match by tags['memory_id'] === memoryId
|
|
855
|
+
* 2. Match by identifier === `memory-${memoryId}.json`
|
|
856
|
+
* 3. Match by JSON metadata.memoryId === memoryId
|
|
857
|
+
* 4. Fallback to index-based matching (if fileIndex provided)
|
|
858
|
+
*
|
|
859
|
+
* @param quiltId - The Quilt blob ID
|
|
860
|
+
* @param memoryId - The memory ID (usually vectorId) to find
|
|
861
|
+
* @param fileIndex - Optional fallback index if other strategies fail
|
|
862
|
+
* @returns The matching memory package result, or null if not found
|
|
863
|
+
*/
|
|
864
|
+
async findMemoryInQuilt(
|
|
865
|
+
quiltId: string,
|
|
866
|
+
memoryId: string,
|
|
867
|
+
fileIndex?: number
|
|
868
|
+
): Promise<QuiltMemoryRetrieveResult | null> {
|
|
869
|
+
const startTime = performance.now();
|
|
870
|
+
|
|
871
|
+
try {
|
|
872
|
+
console.log(`🔍 Finding memory "${memoryId}" in Quilt ${quiltId.substring(0, 20)}...`);
|
|
873
|
+
|
|
874
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
875
|
+
let matchedFile: WalrusFile | undefined;
|
|
876
|
+
let matchStrategy: string = '';
|
|
877
|
+
|
|
878
|
+
// Strategy 1: Match by tags['memory_id']
|
|
879
|
+
for (const f of files) {
|
|
880
|
+
const tags = await f.getTags();
|
|
881
|
+
if (tags?.['memory_id'] === memoryId) {
|
|
882
|
+
matchedFile = f;
|
|
883
|
+
matchStrategy = 'memory_id tag';
|
|
884
|
+
break;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
// Strategy 2: Match by identifier pattern "memory-{memoryId}.json"
|
|
889
|
+
if (!matchedFile) {
|
|
890
|
+
for (const f of files) {
|
|
891
|
+
const identifier = await f.getIdentifier();
|
|
892
|
+
if (identifier === `memory-${memoryId}.json`) {
|
|
893
|
+
matchedFile = f;
|
|
894
|
+
matchStrategy = 'identifier pattern';
|
|
895
|
+
break;
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// Strategy 3: Parse JSON to find matching metadata.memoryId
|
|
901
|
+
if (!matchedFile) {
|
|
902
|
+
for (const f of files) {
|
|
903
|
+
try {
|
|
904
|
+
const json = await f.json() as QuiltMemoryPackage;
|
|
905
|
+
if (json?.metadata?.memoryId === memoryId) {
|
|
906
|
+
matchedFile = f;
|
|
907
|
+
matchStrategy = 'JSON metadata.memoryId';
|
|
908
|
+
break;
|
|
909
|
+
}
|
|
910
|
+
} catch {
|
|
911
|
+
// Not valid JSON, continue
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// Strategy 4: Fallback to index-based matching
|
|
917
|
+
if (!matchedFile && fileIndex !== undefined && fileIndex < files.length) {
|
|
918
|
+
matchedFile = files[fileIndex];
|
|
919
|
+
matchStrategy = `index fallback (${fileIndex})`;
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
if (!matchedFile) {
|
|
923
|
+
console.log(`❌ Memory "${memoryId}" not found in Quilt (${files.length} files)`);
|
|
924
|
+
return null;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
const identifier = await matchedFile.getIdentifier() || 'unknown';
|
|
928
|
+
const tags = await matchedFile.getTags();
|
|
929
|
+
|
|
930
|
+
let memoryPackage: QuiltMemoryPackage;
|
|
931
|
+
try {
|
|
932
|
+
memoryPackage = await matchedFile.json() as QuiltMemoryPackage;
|
|
933
|
+
} catch (parseError) {
|
|
934
|
+
// Try recovery for truncated JSON
|
|
935
|
+
const bytes = await matchedFile.bytes();
|
|
936
|
+
const recovered = this.tryRecoverTruncatedPackage(bytes);
|
|
937
|
+
if (recovered) {
|
|
938
|
+
memoryPackage = recovered;
|
|
939
|
+
} else {
|
|
940
|
+
throw parseError;
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
const retrievalTimeMs = performance.now() - startTime;
|
|
945
|
+
console.log(`✅ Found memory "${memoryId}" via ${matchStrategy} (${identifier}) in ${retrievalTimeMs.toFixed(1)}ms`);
|
|
946
|
+
|
|
947
|
+
return {
|
|
948
|
+
identifier,
|
|
949
|
+
memoryPackage,
|
|
950
|
+
tags,
|
|
951
|
+
retrievalTimeMs
|
|
952
|
+
};
|
|
953
|
+
|
|
954
|
+
} catch (error) {
|
|
955
|
+
console.error(`❌ Failed to find memory in Quilt:`, error);
|
|
956
|
+
throw new Error(`Failed to find memory "${memoryId}" in Quilt ${quiltId}: ${error}`);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* Get memory content from a Quilt file
|
|
962
|
+
*
|
|
963
|
+
* Handles both encrypted and unencrypted content:
|
|
964
|
+
* - Unencrypted: Returns content directly from package
|
|
965
|
+
* - Encrypted: Returns decrypted content if sessionKey provided, otherwise throws
|
|
966
|
+
*
|
|
967
|
+
* @param quiltId - The Quilt blob ID
|
|
968
|
+
* @param identifier - The file identifier
|
|
969
|
+
* @param sessionKey - Optional session key for encrypted content
|
|
970
|
+
* @returns Memory content as string
|
|
971
|
+
*/
|
|
972
|
+
async getMemoryContent(
|
|
973
|
+
quiltId: string,
|
|
974
|
+
identifier: string,
|
|
975
|
+
decryptFn?: (encryptedBase64: string) => Promise<string>
|
|
976
|
+
): Promise<string> {
|
|
977
|
+
const result = await this.getMemoryPackage(quiltId, identifier);
|
|
978
|
+
const pkg = result.memoryPackage;
|
|
979
|
+
|
|
980
|
+
if (!pkg.encrypted) {
|
|
981
|
+
// Not encrypted - return content directly
|
|
982
|
+
return pkg.content;
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
if (!pkg.encryptedContent) {
|
|
986
|
+
throw new Error('Memory is marked as encrypted but no encrypted content found');
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
if (!decryptFn) {
|
|
990
|
+
throw new Error('Memory is encrypted. Provide decryptFn to decrypt content.');
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
// Decrypt using provided function
|
|
994
|
+
return await decryptFn(pkg.encryptedContent);
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
// ==========================================================================
|
|
998
|
+
// Utility Methods
|
|
999
|
+
// ==========================================================================
|
|
1000
|
+
|
|
1001
|
+
/**
|
|
1002
|
+
* Try to recover a partially truncated memory package
|
|
1003
|
+
*
|
|
1004
|
+
* Handles cases where JSON was truncated (e.g., in the middle of encryptedContent)
|
|
1005
|
+
* by extracting metadata and marking the encrypted content as corrupted.
|
|
1006
|
+
*
|
|
1007
|
+
* @param bytes - Raw bytes of the file
|
|
1008
|
+
* @returns Recovered QuiltMemoryPackage or null if recovery fails
|
|
1009
|
+
*/
|
|
1010
|
+
private tryRecoverTruncatedPackage(bytes: Uint8Array): QuiltMemoryPackage | null {
|
|
1011
|
+
try {
|
|
1012
|
+
const rawString = new TextDecoder().decode(bytes);
|
|
1013
|
+
|
|
1014
|
+
// Find and trim trailing null bytes
|
|
1015
|
+
let lastValidIndex = rawString.length - 1;
|
|
1016
|
+
while (lastValidIndex >= 0 && rawString.charCodeAt(lastValidIndex) === 0) {
|
|
1017
|
+
lastValidIndex--;
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
const trimmedString = rawString.slice(0, lastValidIndex + 1);
|
|
1021
|
+
|
|
1022
|
+
// First try to parse as-is (maybe nulls were the only issue)
|
|
1023
|
+
try {
|
|
1024
|
+
return JSON.parse(trimmedString) as QuiltMemoryPackage;
|
|
1025
|
+
} catch {
|
|
1026
|
+
// Continue to partial recovery
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
// Look for encryptedContent field - data likely truncated there
|
|
1030
|
+
const encryptedIdx = trimmedString.indexOf('"encryptedContent":"');
|
|
1031
|
+
if (encryptedIdx > 0) {
|
|
1032
|
+
// Extract everything before encryptedContent
|
|
1033
|
+
const beforeEncrypted = trimmedString.slice(0, encryptedIdx);
|
|
1034
|
+
// Remove trailing comma and close the object
|
|
1035
|
+
const cleanedJson = beforeEncrypted.replace(/,\s*$/, '') + '}';
|
|
1036
|
+
|
|
1037
|
+
try {
|
|
1038
|
+
const partialPackage = JSON.parse(cleanedJson);
|
|
1039
|
+
return {
|
|
1040
|
+
...partialPackage,
|
|
1041
|
+
encrypted: true,
|
|
1042
|
+
encryptedContent: '[CORRUPTED - data truncated during storage]'
|
|
1043
|
+
} as QuiltMemoryPackage;
|
|
1044
|
+
} catch {
|
|
1045
|
+
// Partial extraction failed
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
// Try to find the last complete JSON object by looking for closing brace
|
|
1050
|
+
// This handles cases where truncation happened elsewhere
|
|
1051
|
+
for (let i = trimmedString.length - 1; i >= 0; i--) {
|
|
1052
|
+
if (trimmedString[i] === '}') {
|
|
1053
|
+
try {
|
|
1054
|
+
const candidate = trimmedString.slice(0, i + 1);
|
|
1055
|
+
return JSON.parse(candidate) as QuiltMemoryPackage;
|
|
1056
|
+
} catch {
|
|
1057
|
+
// This position doesn't form valid JSON, try earlier
|
|
1058
|
+
continue;
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
return null;
|
|
1064
|
+
} catch {
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
/**
|
|
1070
|
+
* Convert Uint8Array to base64 string
|
|
1071
|
+
*/
|
|
1072
|
+
private uint8ArrayToBase64(bytes: Uint8Array): string {
|
|
1073
|
+
// Use Buffer in Node.js, btoa in browser
|
|
1074
|
+
if (typeof Buffer !== 'undefined') {
|
|
1075
|
+
return Buffer.from(bytes).toString('base64');
|
|
1076
|
+
}
|
|
1077
|
+
// Browser fallback
|
|
1078
|
+
let binary = '';
|
|
1079
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
1080
|
+
binary += String.fromCharCode(bytes[i]);
|
|
1081
|
+
}
|
|
1082
|
+
return btoa(binary);
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/**
|
|
1086
|
+
* Convert base64 string to Uint8Array
|
|
1087
|
+
*/
|
|
1088
|
+
private base64ToUint8Array(base64: string): Uint8Array {
|
|
1089
|
+
// Use Buffer in Node.js, atob in browser
|
|
1090
|
+
if (typeof Buffer !== 'undefined') {
|
|
1091
|
+
return new Uint8Array(Buffer.from(base64, 'base64'));
|
|
1092
|
+
}
|
|
1093
|
+
// Browser fallback
|
|
1094
|
+
const binary = atob(base64);
|
|
1095
|
+
const bytes = new Uint8Array(binary.length);
|
|
1096
|
+
for (let i = 0; i < binary.length; i++) {
|
|
1097
|
+
bytes[i] = binary.charCodeAt(i);
|
|
1098
|
+
}
|
|
1099
|
+
return bytes;
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/**
|
|
1103
|
+
* Get statistics
|
|
1104
|
+
*/
|
|
1105
|
+
getStats() {
|
|
1106
|
+
return {
|
|
1107
|
+
useUploadRelay: this.useUploadRelay,
|
|
1108
|
+
epochs: this.epochs
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
/**
|
|
1113
|
+
* Get Walrus client
|
|
1114
|
+
*/
|
|
1115
|
+
getWalrusClient(useRelay?: boolean): WalrusClient {
|
|
1116
|
+
return (useRelay ?? this.useUploadRelay)
|
|
1117
|
+
? this.walrusWithRelay
|
|
1118
|
+
: this.walrusWithoutRelay;
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
/**
|
|
1122
|
+
* Get base64 converter (for external use)
|
|
1123
|
+
*/
|
|
1124
|
+
getBase64Utils() {
|
|
1125
|
+
return {
|
|
1126
|
+
encode: this.uint8ArrayToBase64.bind(this),
|
|
1127
|
+
decode: this.base64ToUint8Array.bind(this)
|
|
1128
|
+
};
|
|
1129
|
+
}
|
|
1130
|
+
}
|