@cmdoss/memwal-sdk 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-sdk/PDWVectorStore.d.ts.map +1 -1
- package/dist/ai-sdk/PDWVectorStore.js +4 -1
- package/dist/ai-sdk/PDWVectorStore.js.map +1 -1
- package/dist/ai-sdk/tools.d.ts +2 -2
- package/dist/ai-sdk/tools.js +2 -2
- package/dist/browser.d.ts +5 -6
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +7 -6
- package/dist/browser.js.map +1 -1
- package/dist/client/ClientMemoryManager.d.ts +1 -0
- package/dist/client/ClientMemoryManager.d.ts.map +1 -1
- package/dist/client/ClientMemoryManager.js +5 -1
- package/dist/client/ClientMemoryManager.js.map +1 -1
- package/dist/client/SimplePDWClient.d.ts +24 -1
- package/dist/client/SimplePDWClient.d.ts.map +1 -1
- package/dist/client/SimplePDWClient.js +31 -9
- package/dist/client/SimplePDWClient.js.map +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.d.ts +1 -1
- package/dist/client/namespaces/EmbeddingsNamespace.js +1 -1
- package/dist/client/namespaces/IndexNamespace.d.ts +38 -9
- package/dist/client/namespaces/IndexNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/IndexNamespace.js +77 -10
- package/dist/client/namespaces/IndexNamespace.js.map +1 -1
- package/dist/client/namespaces/MemoryNamespace.d.ts +27 -0
- package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/MemoryNamespace.js +104 -0
- package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
- package/dist/client/namespaces/SearchNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/SearchNamespace.js +25 -14
- package/dist/client/namespaces/SearchNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/AINamespace.d.ts +2 -2
- package/dist/client/namespaces/consolidated/AINamespace.js +2 -2
- package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js +69 -1
- package/dist/client/namespaces/consolidated/BlockchainNamespace.js.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts +46 -0
- package/dist/client/namespaces/consolidated/StorageNamespace.d.ts.map +1 -1
- package/dist/client/namespaces/consolidated/StorageNamespace.js +34 -0
- package/dist/client/namespaces/consolidated/StorageNamespace.js.map +1 -1
- package/dist/graph/GraphService.js +2 -2
- package/dist/graph/GraphService.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/permissions/ConsentRepository.browser.d.ts +56 -0
- package/dist/permissions/ConsentRepository.browser.d.ts.map +1 -0
- package/dist/permissions/ConsentRepository.browser.js +198 -0
- package/dist/permissions/ConsentRepository.browser.js.map +1 -0
- package/dist/retrieval/MemoryRetrievalService.d.ts +31 -0
- package/dist/retrieval/MemoryRetrievalService.d.ts.map +1 -1
- package/dist/retrieval/MemoryRetrievalService.js +44 -4
- package/dist/retrieval/MemoryRetrievalService.js.map +1 -1
- package/dist/services/EmbeddingService.d.ts +28 -1
- package/dist/services/EmbeddingService.d.ts.map +1 -1
- package/dist/services/EmbeddingService.js +54 -0
- package/dist/services/EmbeddingService.js.map +1 -1
- package/dist/services/GeminiAIService.d.ts.map +1 -1
- package/dist/services/GeminiAIService.js +283 -27
- package/dist/services/GeminiAIService.js.map +1 -1
- package/dist/services/IndexManager.d.ts +5 -1
- package/dist/services/IndexManager.d.ts.map +1 -1
- package/dist/services/IndexManager.js +17 -40
- package/dist/services/IndexManager.js.map +1 -1
- package/dist/services/MemoryIndexService.d.ts +31 -2
- package/dist/services/MemoryIndexService.d.ts.map +1 -1
- package/dist/services/MemoryIndexService.js +75 -3
- package/dist/services/MemoryIndexService.js.map +1 -1
- package/dist/services/QueryService.js +1 -1
- package/dist/services/QueryService.js.map +1 -1
- package/dist/services/StorageService.d.ts +10 -0
- package/dist/services/StorageService.d.ts.map +1 -1
- package/dist/services/StorageService.js +13 -0
- package/dist/services/StorageService.js.map +1 -1
- package/dist/services/storage/QuiltBatchManager.d.ts +111 -4
- package/dist/services/storage/QuiltBatchManager.d.ts.map +1 -1
- package/dist/services/storage/QuiltBatchManager.js +450 -38
- package/dist/services/storage/QuiltBatchManager.js.map +1 -1
- package/dist/services/storage/index.d.ts +1 -1
- package/dist/services/storage/index.d.ts.map +1 -1
- package/dist/services/storage/index.js.map +1 -1
- package/dist/utils/LRUCache.d.ts +106 -0
- package/dist/utils/LRUCache.d.ts.map +1 -0
- package/dist/utils/LRUCache.js +281 -0
- package/dist/utils/LRUCache.js.map +1 -0
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/memoryIndexOnChain.d.ts +212 -0
- package/dist/utils/memoryIndexOnChain.d.ts.map +1 -0
- package/dist/utils/memoryIndexOnChain.js +312 -0
- package/dist/utils/memoryIndexOnChain.js.map +1 -0
- package/dist/utils/rebuildIndexNode.d.ts +29 -0
- package/dist/utils/rebuildIndexNode.d.ts.map +1 -1
- package/dist/utils/rebuildIndexNode.js +387 -45
- package/dist/utils/rebuildIndexNode.js.map +1 -1
- package/dist/vector/HnswWasmService.d.ts +20 -5
- package/dist/vector/HnswWasmService.d.ts.map +1 -1
- package/dist/vector/HnswWasmService.js +73 -40
- package/dist/vector/HnswWasmService.js.map +1 -1
- package/dist/vector/IHnswService.d.ts +10 -1
- package/dist/vector/IHnswService.d.ts.map +1 -1
- package/dist/vector/IHnswService.js.map +1 -1
- package/dist/vector/NodeHnswService.d.ts +16 -0
- package/dist/vector/NodeHnswService.d.ts.map +1 -1
- package/dist/vector/NodeHnswService.js +108 -10
- package/dist/vector/NodeHnswService.js.map +1 -1
- package/dist/vector/createHnswService.d.ts +1 -1
- package/dist/vector/createHnswService.js +1 -1
- package/dist/vector/index.d.ts +1 -1
- package/dist/vector/index.js +1 -1
- package/package.json +157 -157
- package/src/ai-sdk/PDWVectorStore.ts +4 -1
- package/src/ai-sdk/tools.ts +2 -2
- package/src/browser.ts +15 -10
- package/src/client/ClientMemoryManager.ts +6 -1
- package/src/client/SimplePDWClient.ts +63 -10
- package/src/client/namespaces/EmbeddingsNamespace.ts +1 -1
- package/src/client/namespaces/IndexNamespace.ts +89 -11
- package/src/client/namespaces/MemoryNamespace.ts +137 -0
- package/src/client/namespaces/SearchNamespace.ts +27 -14
- package/src/client/namespaces/consolidated/AINamespace.ts +2 -2
- package/src/client/namespaces/consolidated/BlockchainNamespace.ts +73 -1
- package/src/client/namespaces/consolidated/StorageNamespace.ts +57 -0
- package/src/core/types/index.ts +1 -1
- package/src/generated/pdw/capability.ts +319 -319
- package/src/graph/GraphService.ts +2 -2
- package/src/index.ts +25 -1
- package/src/permissions/ConsentRepository.browser.ts +249 -0
- package/src/retrieval/MemoryRetrievalService.ts +78 -4
- package/src/services/EmbeddingService.ts +66 -1
- package/src/services/GeminiAIService.ts +283 -27
- package/src/services/IndexManager.ts +18 -45
- package/src/services/MemoryIndexService.ts +85 -3
- package/src/services/QueryService.ts +1 -1
- package/src/services/StorageService.ts +15 -0
- package/src/services/storage/QuiltBatchManager.ts +538 -42
- package/src/services/storage/index.ts +6 -1
- package/src/utils/LRUCache.ts +378 -0
- package/src/utils/index.ts +8 -0
- package/src/utils/memoryIndexOnChain.ts +507 -0
- package/src/utils/rebuildIndexNode.ts +482 -52
- package/src/vector/HnswWasmService.ts +95 -43
- package/src/vector/IHnswService.ts +10 -1
- package/src/vector/NodeHnswService.ts +130 -10
- package/src/vector/createHnswService.ts +1 -1
- package/src/vector/index.ts +1 -1
|
@@ -34,17 +34,39 @@ import type { UnifiedSigner } from '../../client/signers/UnifiedSigner';
|
|
|
34
34
|
// Types
|
|
35
35
|
// ============================================================================
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Input for batch memory upload
|
|
39
|
+
*/
|
|
37
40
|
export interface BatchMemory {
|
|
38
41
|
content: string;
|
|
39
42
|
category: string;
|
|
40
43
|
importance: number;
|
|
41
44
|
topic: string;
|
|
42
45
|
embedding: number[];
|
|
43
|
-
encryptedContent
|
|
46
|
+
encryptedContent?: Uint8Array; // Optional - only when encryption is enabled
|
|
44
47
|
summary?: string;
|
|
45
48
|
id?: string; // Optional client-side ID for tracking
|
|
46
49
|
}
|
|
47
50
|
|
|
51
|
+
/**
|
|
52
|
+
* Memory package stored in Quilt as JSON
|
|
53
|
+
* This format is consistent with regular memory storage
|
|
54
|
+
*/
|
|
55
|
+
export interface QuiltMemoryPackage {
|
|
56
|
+
content: string; // Plaintext content (empty if encrypted)
|
|
57
|
+
embedding: number[]; // Vector embedding
|
|
58
|
+
metadata: {
|
|
59
|
+
category: string;
|
|
60
|
+
importance: number;
|
|
61
|
+
topic: string;
|
|
62
|
+
[key: string]: unknown;
|
|
63
|
+
};
|
|
64
|
+
timestamp: number;
|
|
65
|
+
version: string; // Package format version
|
|
66
|
+
encrypted?: boolean; // Whether content is encrypted
|
|
67
|
+
encryptedContent?: string; // Base64-encoded encrypted content (if encrypted)
|
|
68
|
+
}
|
|
69
|
+
|
|
48
70
|
export interface QuiltUploadOptions {
|
|
49
71
|
signer: UnifiedSigner;
|
|
50
72
|
epochs?: number;
|
|
@@ -76,6 +98,16 @@ export interface QuiltRetrieveResult {
|
|
|
76
98
|
retrievalTimeMs: number;
|
|
77
99
|
}
|
|
78
100
|
|
|
101
|
+
/**
|
|
102
|
+
* Result when retrieving a memory package from Quilt
|
|
103
|
+
*/
|
|
104
|
+
export interface QuiltMemoryRetrieveResult {
|
|
105
|
+
identifier: string;
|
|
106
|
+
memoryPackage: QuiltMemoryPackage;
|
|
107
|
+
tags: Record<string, string>;
|
|
108
|
+
retrievalTimeMs: number;
|
|
109
|
+
}
|
|
110
|
+
|
|
79
111
|
export interface QuiltListResult {
|
|
80
112
|
identifier: string;
|
|
81
113
|
quiltPatchId: string;
|
|
@@ -152,35 +184,72 @@ export class QuiltBatchManager {
|
|
|
152
184
|
console.log(`📦 Uploading batch of ${memories.length} memories as Quilt (writeFilesFlow)...`);
|
|
153
185
|
|
|
154
186
|
try {
|
|
155
|
-
// Create WalrusFile for each memory
|
|
187
|
+
// Create WalrusFile for each memory as JSON package
|
|
188
|
+
// This format is consistent with regular memory storage
|
|
156
189
|
const files = memories.map((memory, index) => {
|
|
157
190
|
const identifier = memory.id
|
|
158
191
|
? `memory-${memory.id}.json`
|
|
159
192
|
: `memory-${Date.now()}-${index}-${Math.random().toString(36).slice(2, 9)}.json`;
|
|
160
193
|
|
|
161
|
-
|
|
194
|
+
const isEncrypted = !!memory.encryptedContent && memory.encryptedContent.length > 0;
|
|
195
|
+
const timestamp = Date.now();
|
|
196
|
+
|
|
197
|
+
// Create memory package (JSON format - consistent with regular storage)
|
|
198
|
+
const memoryPackage: QuiltMemoryPackage = {
|
|
199
|
+
// Content: plaintext if not encrypted, empty if encrypted
|
|
200
|
+
content: isEncrypted ? '' : memory.content,
|
|
201
|
+
embedding: memory.embedding,
|
|
202
|
+
metadata: {
|
|
203
|
+
category: memory.category,
|
|
204
|
+
importance: memory.importance,
|
|
205
|
+
topic: memory.topic,
|
|
206
|
+
...(memory.summary ? { summary: memory.summary } : {}),
|
|
207
|
+
...(memory.id ? { memoryId: memory.id } : {})
|
|
208
|
+
},
|
|
209
|
+
timestamp,
|
|
210
|
+
version: '2.0.0', // Quilt JSON package version
|
|
211
|
+
encrypted: isEncrypted,
|
|
212
|
+
// Store encrypted content as base64 for JSON compatibility
|
|
213
|
+
...(isEncrypted && memory.encryptedContent ? {
|
|
214
|
+
encryptedContent: this.uint8ArrayToBase64(memory.encryptedContent)
|
|
215
|
+
} : {})
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
// Serialize to JSON and encode as bytes
|
|
219
|
+
const jsonString = JSON.stringify(memoryPackage);
|
|
220
|
+
const contents = new TextEncoder().encode(jsonString);
|
|
221
|
+
totalSize += contents.length;
|
|
222
|
+
|
|
223
|
+
// Diagnostic logging for debugging Quilt corruption issues
|
|
224
|
+
console.log(` 📝 File ${index}: identifier=${identifier}`);
|
|
225
|
+
console.log(` JSON string length: ${jsonString.length} chars`);
|
|
226
|
+
console.log(` Encoded bytes: ${contents.length} bytes`);
|
|
227
|
+
console.log(` Last 50 chars of JSON: ...${jsonString.slice(-50)}`);
|
|
228
|
+
console.log(` Last 10 bytes (hex): ${Array.from(contents.slice(-10)).map(b => b.toString(16).padStart(2, '0')).join(' ')}`);
|
|
162
229
|
|
|
163
230
|
return WalrusFile.from({
|
|
164
|
-
contents
|
|
231
|
+
contents,
|
|
165
232
|
identifier,
|
|
166
233
|
tags: {
|
|
167
|
-
// Core metadata (plaintext for filtering)
|
|
234
|
+
// Core metadata (plaintext for filtering without decryption)
|
|
235
|
+
'content-type': 'application/json',
|
|
168
236
|
'category': memory.category,
|
|
169
237
|
'importance': memory.importance.toString(),
|
|
170
238
|
'topic': memory.topic,
|
|
171
|
-
'timestamp': new Date().toISOString(),
|
|
172
|
-
'created_at': new Date().toISOString(),
|
|
239
|
+
'timestamp': new Date(timestamp).toISOString(),
|
|
240
|
+
'created_at': new Date(timestamp).toISOString(),
|
|
173
241
|
|
|
174
242
|
// Encryption info
|
|
175
|
-
'encrypted': 'true',
|
|
176
|
-
'encryption_type': 'seal',
|
|
243
|
+
'encrypted': isEncrypted ? 'true' : 'false',
|
|
244
|
+
...(isEncrypted ? { 'encryption_type': 'seal' } : {}),
|
|
177
245
|
|
|
178
246
|
// Owner
|
|
179
247
|
'owner': options.userAddress,
|
|
180
248
|
|
|
181
249
|
// Content info
|
|
182
|
-
'content_size':
|
|
250
|
+
'content_size': contents.length.toString(),
|
|
183
251
|
'embedding_dimensions': memory.embedding.length.toString(),
|
|
252
|
+
'package_version': '2.0.0',
|
|
184
253
|
|
|
185
254
|
// Optional rich metadata
|
|
186
255
|
...(memory.summary ? { 'summary': memory.summary } : {}),
|
|
@@ -245,29 +314,34 @@ export class QuiltBatchManager {
|
|
|
245
314
|
console.log(` Gas saved: ${gasSaved} vs individual uploads`);
|
|
246
315
|
|
|
247
316
|
// Build file results using original WalrusFile objects for metadata
|
|
248
|
-
//
|
|
317
|
+
// Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
|
|
318
|
+
// Match files by identifier when reading
|
|
319
|
+
const quiltId = uploadedFilesInfo[0]?.blobId || '';
|
|
320
|
+
|
|
249
321
|
const fileResults: QuiltFileResult[] = await Promise.all(
|
|
250
322
|
files.map(async (originalFile, i) => {
|
|
251
323
|
const identifier = await originalFile.getIdentifier() || `file-${i}`;
|
|
252
324
|
const tags = await originalFile.getTags() || {};
|
|
253
|
-
|
|
254
|
-
|
|
325
|
+
const fileInfo = uploadedFilesInfo[i];
|
|
326
|
+
|
|
327
|
+
// quiltPatchId is stored for reference but not used for retrieval
|
|
328
|
+
const quiltPatchId = fileInfo?.id || '';
|
|
329
|
+
|
|
330
|
+
console.log(` File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
|
|
255
331
|
|
|
256
332
|
return {
|
|
257
333
|
identifier,
|
|
258
|
-
blobId
|
|
259
|
-
|
|
334
|
+
// Use shared quiltId as blobId - read via getBlob(quiltId).files()
|
|
335
|
+
blobId: quiltId,
|
|
336
|
+
quiltPatchId,
|
|
260
337
|
tags: Object.fromEntries(
|
|
261
338
|
Object.entries(tags).map(([k, v]) => [k, String(v)])
|
|
262
339
|
),
|
|
263
|
-
size: memories[i]?.encryptedContent
|
|
340
|
+
size: memories[i]?.encryptedContent?.length || memories[i]?.content?.length || 0
|
|
264
341
|
};
|
|
265
342
|
})
|
|
266
343
|
);
|
|
267
344
|
|
|
268
|
-
// Get quiltId from first uploaded file
|
|
269
|
-
const quiltId = uploadedFilesInfo[0]?.blobId || '';
|
|
270
|
-
|
|
271
345
|
return {
|
|
272
346
|
quiltId,
|
|
273
347
|
blobObjectId: undefined, // Not available from flow
|
|
@@ -373,18 +447,26 @@ export class QuiltBatchManager {
|
|
|
373
447
|
console.log(` Upload time: ${uploadTimeMs.toFixed(1)}ms`);
|
|
374
448
|
|
|
375
449
|
// Build file results using original WalrusFile objects for metadata
|
|
376
|
-
//
|
|
450
|
+
// Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
|
|
451
|
+
// Match files by identifier when reading
|
|
452
|
+
const quiltId = uploadedFilesInfo[0]?.blobId || '';
|
|
453
|
+
|
|
377
454
|
const fileResults: QuiltFileResult[] = await Promise.all(
|
|
378
455
|
walrusFiles.map(async (originalFile, i) => {
|
|
379
456
|
const identifier = await originalFile.getIdentifier() || files[i]?.identifier || `file-${i}`;
|
|
380
457
|
const tags = await originalFile.getTags() || {};
|
|
381
|
-
|
|
382
|
-
|
|
458
|
+
const fileInfo = uploadedFilesInfo[i];
|
|
459
|
+
|
|
460
|
+
// quiltPatchId is stored for reference but not used for retrieval
|
|
461
|
+
const quiltPatchId = fileInfo?.id || '';
|
|
462
|
+
|
|
463
|
+
console.log(` File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
|
|
383
464
|
|
|
384
465
|
return {
|
|
385
466
|
identifier,
|
|
386
|
-
blobId
|
|
387
|
-
|
|
467
|
+
// Use shared quiltId as blobId - read via getBlob(quiltId).files()
|
|
468
|
+
blobId: quiltId,
|
|
469
|
+
quiltPatchId,
|
|
388
470
|
tags: Object.fromEntries(
|
|
389
471
|
Object.entries(tags).map(([k, v]) => [k, String(v)])
|
|
390
472
|
),
|
|
@@ -393,9 +475,6 @@ export class QuiltBatchManager {
|
|
|
393
475
|
})
|
|
394
476
|
);
|
|
395
477
|
|
|
396
|
-
// Get quiltId from first uploaded file
|
|
397
|
-
const quiltId = uploadedFilesInfo[0]?.blobId || '';
|
|
398
|
-
|
|
399
478
|
return {
|
|
400
479
|
quiltId,
|
|
401
480
|
blobObjectId: undefined, // Not available from flow
|
|
@@ -418,16 +497,29 @@ export class QuiltBatchManager {
|
|
|
418
497
|
/**
|
|
419
498
|
* Retrieve all files from a Quilt
|
|
420
499
|
*
|
|
421
|
-
*
|
|
500
|
+
* Uses getBlob().files() pattern which correctly parses Quilt structure
|
|
501
|
+
* and returns individual files with their identifiers and tags.
|
|
502
|
+
*
|
|
503
|
+
* @param quiltId - The Quilt blob ID (shared blobId)
|
|
422
504
|
* @returns Array of WalrusFile objects
|
|
423
505
|
*/
|
|
424
506
|
async getQuiltFiles(quiltId: string): Promise<Array<WalrusFile>> {
|
|
425
507
|
try {
|
|
426
508
|
console.log(`📂 Retrieving files from Quilt ${quiltId}...`);
|
|
427
509
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
510
|
+
// Try to parse as Quilt first (getBlob().files() returns ALL files in Quilt)
|
|
511
|
+
// Fall back to getFiles() for regular blobs
|
|
512
|
+
let files: WalrusFile[];
|
|
513
|
+
try {
|
|
514
|
+
const blob = await this.suiClient.walrus.getBlob({ blobId: quiltId });
|
|
515
|
+
files = await blob.files();
|
|
516
|
+
console.log(`✅ Retrieved ${files.length} files from Quilt`);
|
|
517
|
+
} catch (quiltError: any) {
|
|
518
|
+
// Not a Quilt - try as regular blob
|
|
519
|
+
console.log(`📄 Not a Quilt format, fetching as regular blob...`);
|
|
520
|
+
files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
|
|
521
|
+
console.log(`✅ Retrieved ${files.length} file(s) as regular blob`);
|
|
522
|
+
}
|
|
431
523
|
|
|
432
524
|
return files;
|
|
433
525
|
|
|
@@ -440,7 +532,9 @@ export class QuiltBatchManager {
|
|
|
440
532
|
/**
|
|
441
533
|
* Retrieve a specific file by identifier from a Quilt
|
|
442
534
|
*
|
|
443
|
-
*
|
|
535
|
+
* Uses getBlob().files() to get all files then matches by identifier.
|
|
536
|
+
*
|
|
537
|
+
* @param quiltId - The Quilt blob ID (shared blobId)
|
|
444
538
|
* @param identifier - The file identifier within the quilt
|
|
445
539
|
* @returns QuiltRetrieveResult with content and metadata
|
|
446
540
|
*/
|
|
@@ -453,21 +547,25 @@ export class QuiltBatchManager {
|
|
|
453
547
|
try {
|
|
454
548
|
console.log(`📄 Retrieving file "${identifier}" from Quilt ${quiltId}...`);
|
|
455
549
|
|
|
456
|
-
// Get all files from
|
|
457
|
-
const files = await this.
|
|
550
|
+
// Get all files from the blob (Quilt or regular)
|
|
551
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
458
552
|
|
|
459
553
|
// Find file by identifier
|
|
460
|
-
|
|
554
|
+
let matchingFile: WalrusFile | undefined;
|
|
555
|
+
for (const f of files) {
|
|
461
556
|
const fileIdentifier = await f.getIdentifier();
|
|
462
|
-
|
|
463
|
-
|
|
557
|
+
if (fileIdentifier === identifier) {
|
|
558
|
+
matchingFile = f;
|
|
559
|
+
break;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
464
562
|
|
|
465
|
-
if (!
|
|
563
|
+
if (!matchingFile) {
|
|
466
564
|
throw new Error(`File "${identifier}" not found in Quilt`);
|
|
467
565
|
}
|
|
468
566
|
|
|
469
|
-
const content = await
|
|
470
|
-
const tags = await
|
|
567
|
+
const content = await matchingFile.bytes();
|
|
568
|
+
const tags = await matchingFile.getTags();
|
|
471
569
|
const retrievalTimeMs = performance.now() - startTime;
|
|
472
570
|
|
|
473
571
|
console.log(`✅ Retrieved file "${identifier}" (${content.length} bytes)`);
|
|
@@ -488,14 +586,17 @@ export class QuiltBatchManager {
|
|
|
488
586
|
/**
|
|
489
587
|
* List all patches in a Quilt with their metadata
|
|
490
588
|
*
|
|
491
|
-
*
|
|
589
|
+
* Uses getBlob().files() to correctly parse Quilt structure.
|
|
590
|
+
*
|
|
591
|
+
* @param quiltId - The Quilt blob ID (shared blobId)
|
|
492
592
|
* @returns Array of QuiltListResult with identifiers and tags
|
|
493
593
|
*/
|
|
494
594
|
async listQuiltPatches(quiltId: string): Promise<QuiltListResult[]> {
|
|
495
595
|
try {
|
|
496
596
|
console.log(`📋 Listing patches in Quilt ${quiltId}...`);
|
|
497
597
|
|
|
498
|
-
|
|
598
|
+
// Get all files from the blob (Quilt or regular)
|
|
599
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
499
600
|
|
|
500
601
|
const results: QuiltListResult[] = await Promise.all(
|
|
501
602
|
files.map(async (file) => {
|
|
@@ -609,10 +710,395 @@ export class QuiltBatchManager {
|
|
|
609
710
|
return matchingFiles;
|
|
610
711
|
}
|
|
611
712
|
|
|
713
|
+
// ==========================================================================
|
|
714
|
+
// JSON Memory Package Retrieval
|
|
715
|
+
// ==========================================================================
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* Retrieve a memory package as JSON from a Quilt
|
|
719
|
+
*
|
|
720
|
+
* Uses file.json() for efficient parsing (SDK handles it)
|
|
721
|
+
*
|
|
722
|
+
* @param quiltId - The Quilt blob ID
|
|
723
|
+
* @param identifier - The file identifier within the quilt
|
|
724
|
+
* @returns QuiltMemoryRetrieveResult with parsed memory package
|
|
725
|
+
*/
|
|
726
|
+
async getMemoryPackage(
|
|
727
|
+
quiltId: string,
|
|
728
|
+
identifier: string
|
|
729
|
+
): Promise<QuiltMemoryRetrieveResult> {
|
|
730
|
+
const startTime = performance.now();
|
|
731
|
+
|
|
732
|
+
try {
|
|
733
|
+
console.log(`📄 Retrieving memory package "${identifier}" from Quilt ${quiltId}...`);
|
|
734
|
+
|
|
735
|
+
// Get all files from the blob
|
|
736
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
737
|
+
|
|
738
|
+
// Find file by identifier
|
|
739
|
+
let matchingFile: WalrusFile | undefined;
|
|
740
|
+
for (const f of files) {
|
|
741
|
+
const fileIdentifier = await f.getIdentifier();
|
|
742
|
+
if (fileIdentifier === identifier) {
|
|
743
|
+
matchingFile = f;
|
|
744
|
+
break;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
if (!matchingFile) {
|
|
749
|
+
throw new Error(`File "${identifier}" not found in Quilt`);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
const tags = await matchingFile.getTags();
|
|
753
|
+
let memoryPackage: QuiltMemoryPackage;
|
|
754
|
+
|
|
755
|
+
try {
|
|
756
|
+
// Parse directly as JSON (SDK handles it!)
|
|
757
|
+
memoryPackage = await matchingFile.json() as QuiltMemoryPackage;
|
|
758
|
+
} catch (parseError) {
|
|
759
|
+
// Try partial recovery for truncated JSON
|
|
760
|
+
console.warn(`⚠️ JSON parse failed for "${identifier}", attempting recovery...`);
|
|
761
|
+
const bytes = await matchingFile.bytes();
|
|
762
|
+
const recovered = this.tryRecoverTruncatedPackage(bytes);
|
|
763
|
+
if (recovered) {
|
|
764
|
+
console.log(`🔧 Partially recovered "${identifier}" (encryptedContent may be corrupted)`);
|
|
765
|
+
memoryPackage = recovered;
|
|
766
|
+
} else {
|
|
767
|
+
throw parseError;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
const retrievalTimeMs = performance.now() - startTime;
|
|
772
|
+
|
|
773
|
+
console.log(`✅ Retrieved memory package "${identifier}" (${retrievalTimeMs.toFixed(1)}ms)`);
|
|
774
|
+
|
|
775
|
+
return {
|
|
776
|
+
identifier,
|
|
777
|
+
memoryPackage,
|
|
778
|
+
tags,
|
|
779
|
+
retrievalTimeMs
|
|
780
|
+
};
|
|
781
|
+
|
|
782
|
+
} catch (error) {
|
|
783
|
+
console.error(`❌ Failed to retrieve memory package:`, error);
|
|
784
|
+
throw new Error(`Failed to retrieve memory package "${identifier}": ${error}`);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Retrieve all memory packages from a Quilt as JSON
|
|
790
|
+
*
|
|
791
|
+
* @param quiltId - The Quilt blob ID
|
|
792
|
+
* @returns Array of memory packages with metadata
|
|
793
|
+
*/
|
|
794
|
+
async getAllMemoryPackages(quiltId: string): Promise<QuiltMemoryRetrieveResult[]> {
|
|
795
|
+
const startTime = performance.now();
|
|
796
|
+
|
|
797
|
+
try {
|
|
798
|
+
console.log(`📂 Retrieving all memory packages from Quilt ${quiltId}...`);
|
|
799
|
+
|
|
800
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
801
|
+
const results: QuiltMemoryRetrieveResult[] = [];
|
|
802
|
+
|
|
803
|
+
for (const file of files) {
|
|
804
|
+
const identifier = await file.getIdentifier() || 'unknown';
|
|
805
|
+
const tags = await file.getTags();
|
|
806
|
+
|
|
807
|
+
try {
|
|
808
|
+
// Parse as JSON
|
|
809
|
+
const memoryPackage = await file.json() as QuiltMemoryPackage;
|
|
810
|
+
results.push({
|
|
811
|
+
identifier,
|
|
812
|
+
memoryPackage,
|
|
813
|
+
tags,
|
|
814
|
+
retrievalTimeMs: 0 // Individual timing not tracked in batch
|
|
815
|
+
});
|
|
816
|
+
} catch (parseError) {
|
|
817
|
+
console.warn(`⚠️ Failed to parse "${identifier}" as JSON:`, parseError);
|
|
818
|
+
|
|
819
|
+
// Try partial recovery for truncated JSON
|
|
820
|
+
try {
|
|
821
|
+
const bytes = await file.bytes();
|
|
822
|
+
const recoveredPackage = this.tryRecoverTruncatedPackage(bytes);
|
|
823
|
+
if (recoveredPackage) {
|
|
824
|
+
console.log(`🔧 Partially recovered "${identifier}" (encryptedContent truncated)`);
|
|
825
|
+
results.push({
|
|
826
|
+
identifier,
|
|
827
|
+
memoryPackage: recoveredPackage,
|
|
828
|
+
tags,
|
|
829
|
+
retrievalTimeMs: 0
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
} catch {
|
|
833
|
+
// Skip files that can't be recovered
|
|
834
|
+
console.warn(`❌ Could not recover "${identifier}"`);
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
const totalTimeMs = performance.now() - startTime;
|
|
840
|
+
console.log(`✅ Retrieved ${results.length} memory packages (${totalTimeMs.toFixed(1)}ms)`);
|
|
841
|
+
|
|
842
|
+
return results;
|
|
843
|
+
|
|
844
|
+
} catch (error) {
|
|
845
|
+
console.error(`❌ Failed to retrieve memory packages:`, error);
|
|
846
|
+
throw new Error(`Failed to retrieve memory packages from Quilt ${quiltId}: ${error}`);
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* Find a specific memory in a Quilt using multiple matching strategies
|
|
852
|
+
*
|
|
853
|
+
* Strategies (in order of priority):
|
|
854
|
+
* 1. Match by tags['memory_id'] === memoryId
|
|
855
|
+
* 2. Match by identifier === `memory-${memoryId}.json`
|
|
856
|
+
* 3. Match by JSON metadata.memoryId === memoryId
|
|
857
|
+
* 4. Fallback to index-based matching (if fileIndex provided)
|
|
858
|
+
*
|
|
859
|
+
* @param quiltId - The Quilt blob ID
|
|
860
|
+
* @param memoryId - The memory ID (usually vectorId) to find
|
|
861
|
+
* @param fileIndex - Optional fallback index if other strategies fail
|
|
862
|
+
* @returns The matching memory package result, or null if not found
|
|
863
|
+
*/
|
|
864
|
+
async findMemoryInQuilt(
|
|
865
|
+
quiltId: string,
|
|
866
|
+
memoryId: string,
|
|
867
|
+
fileIndex?: number
|
|
868
|
+
): Promise<QuiltMemoryRetrieveResult | null> {
|
|
869
|
+
const startTime = performance.now();
|
|
870
|
+
|
|
871
|
+
try {
|
|
872
|
+
console.log(`🔍 Finding memory "${memoryId}" in Quilt ${quiltId.substring(0, 20)}...`);
|
|
873
|
+
|
|
874
|
+
const files = await this.getQuiltFiles(quiltId);
|
|
875
|
+
let matchedFile: WalrusFile | undefined;
|
|
876
|
+
let matchStrategy: string = '';
|
|
877
|
+
|
|
878
|
+
// Strategy 1: Match by tags['memory_id']
|
|
879
|
+
for (const f of files) {
|
|
880
|
+
const tags = await f.getTags();
|
|
881
|
+
if (tags?.['memory_id'] === memoryId) {
|
|
882
|
+
matchedFile = f;
|
|
883
|
+
matchStrategy = 'memory_id tag';
|
|
884
|
+
break;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
// Strategy 2: Match by identifier pattern "memory-{memoryId}.json"
|
|
889
|
+
if (!matchedFile) {
|
|
890
|
+
for (const f of files) {
|
|
891
|
+
const identifier = await f.getIdentifier();
|
|
892
|
+
if (identifier === `memory-${memoryId}.json`) {
|
|
893
|
+
matchedFile = f;
|
|
894
|
+
matchStrategy = 'identifier pattern';
|
|
895
|
+
break;
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// Strategy 3: Parse JSON to find matching metadata.memoryId
|
|
901
|
+
if (!matchedFile) {
|
|
902
|
+
for (const f of files) {
|
|
903
|
+
try {
|
|
904
|
+
const json = await f.json() as QuiltMemoryPackage;
|
|
905
|
+
if (json?.metadata?.memoryId === memoryId) {
|
|
906
|
+
matchedFile = f;
|
|
907
|
+
matchStrategy = 'JSON metadata.memoryId';
|
|
908
|
+
break;
|
|
909
|
+
}
|
|
910
|
+
} catch {
|
|
911
|
+
// Not valid JSON, continue
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// Strategy 4: Fallback to index-based matching
|
|
917
|
+
if (!matchedFile && fileIndex !== undefined && fileIndex < files.length) {
|
|
918
|
+
matchedFile = files[fileIndex];
|
|
919
|
+
matchStrategy = `index fallback (${fileIndex})`;
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
if (!matchedFile) {
|
|
923
|
+
console.log(`❌ Memory "${memoryId}" not found in Quilt (${files.length} files)`);
|
|
924
|
+
return null;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
const identifier = await matchedFile.getIdentifier() || 'unknown';
|
|
928
|
+
const tags = await matchedFile.getTags();
|
|
929
|
+
|
|
930
|
+
let memoryPackage: QuiltMemoryPackage;
|
|
931
|
+
try {
|
|
932
|
+
memoryPackage = await matchedFile.json() as QuiltMemoryPackage;
|
|
933
|
+
} catch (parseError) {
|
|
934
|
+
// Try recovery for truncated JSON
|
|
935
|
+
const bytes = await matchedFile.bytes();
|
|
936
|
+
const recovered = this.tryRecoverTruncatedPackage(bytes);
|
|
937
|
+
if (recovered) {
|
|
938
|
+
memoryPackage = recovered;
|
|
939
|
+
} else {
|
|
940
|
+
throw parseError;
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
const retrievalTimeMs = performance.now() - startTime;
|
|
945
|
+
console.log(`✅ Found memory "${memoryId}" via ${matchStrategy} (${identifier}) in ${retrievalTimeMs.toFixed(1)}ms`);
|
|
946
|
+
|
|
947
|
+
return {
|
|
948
|
+
identifier,
|
|
949
|
+
memoryPackage,
|
|
950
|
+
tags,
|
|
951
|
+
retrievalTimeMs
|
|
952
|
+
};
|
|
953
|
+
|
|
954
|
+
} catch (error) {
|
|
955
|
+
console.error(`❌ Failed to find memory in Quilt:`, error);
|
|
956
|
+
throw new Error(`Failed to find memory "${memoryId}" in Quilt ${quiltId}: ${error}`);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* Get memory content from a Quilt file
|
|
962
|
+
*
|
|
963
|
+
* Handles both encrypted and unencrypted content:
|
|
964
|
+
* - Unencrypted: Returns content directly from package
|
|
965
|
+
* - Encrypted: Returns decrypted content if sessionKey provided, otherwise throws
|
|
966
|
+
*
|
|
967
|
+
* @param quiltId - The Quilt blob ID
|
|
968
|
+
* @param identifier - The file identifier
|
|
969
|
+
* @param sessionKey - Optional session key for encrypted content
|
|
970
|
+
* @returns Memory content as string
|
|
971
|
+
*/
|
|
972
|
+
async getMemoryContent(
|
|
973
|
+
quiltId: string,
|
|
974
|
+
identifier: string,
|
|
975
|
+
decryptFn?: (encryptedBase64: string) => Promise<string>
|
|
976
|
+
): Promise<string> {
|
|
977
|
+
const result = await this.getMemoryPackage(quiltId, identifier);
|
|
978
|
+
const pkg = result.memoryPackage;
|
|
979
|
+
|
|
980
|
+
if (!pkg.encrypted) {
|
|
981
|
+
// Not encrypted - return content directly
|
|
982
|
+
return pkg.content;
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
if (!pkg.encryptedContent) {
|
|
986
|
+
throw new Error('Memory is marked as encrypted but no encrypted content found');
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
if (!decryptFn) {
|
|
990
|
+
throw new Error('Memory is encrypted. Provide decryptFn to decrypt content.');
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
// Decrypt using provided function
|
|
994
|
+
return await decryptFn(pkg.encryptedContent);
|
|
995
|
+
}
|
|
996
|
+
|
|
612
997
|
// ==========================================================================
|
|
613
998
|
// Utility Methods
|
|
614
999
|
// ==========================================================================
|
|
615
1000
|
|
|
1001
|
+
/**
|
|
1002
|
+
* Try to recover a partially truncated memory package
|
|
1003
|
+
*
|
|
1004
|
+
* Handles cases where JSON was truncated (e.g., in the middle of encryptedContent)
|
|
1005
|
+
* by extracting metadata and marking the encrypted content as corrupted.
|
|
1006
|
+
*
|
|
1007
|
+
* @param bytes - Raw bytes of the file
|
|
1008
|
+
* @returns Recovered QuiltMemoryPackage or null if recovery fails
|
|
1009
|
+
*/
|
|
1010
|
+
private tryRecoverTruncatedPackage(bytes: Uint8Array): QuiltMemoryPackage | null {
|
|
1011
|
+
try {
|
|
1012
|
+
const rawString = new TextDecoder().decode(bytes);
|
|
1013
|
+
|
|
1014
|
+
// Find and trim trailing null bytes
|
|
1015
|
+
let lastValidIndex = rawString.length - 1;
|
|
1016
|
+
while (lastValidIndex >= 0 && rawString.charCodeAt(lastValidIndex) === 0) {
|
|
1017
|
+
lastValidIndex--;
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
const trimmedString = rawString.slice(0, lastValidIndex + 1);
|
|
1021
|
+
|
|
1022
|
+
// First try to parse as-is (maybe nulls were the only issue)
|
|
1023
|
+
try {
|
|
1024
|
+
return JSON.parse(trimmedString) as QuiltMemoryPackage;
|
|
1025
|
+
} catch {
|
|
1026
|
+
// Continue to partial recovery
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
// Look for encryptedContent field - data likely truncated there
|
|
1030
|
+
const encryptedIdx = trimmedString.indexOf('"encryptedContent":"');
|
|
1031
|
+
if (encryptedIdx > 0) {
|
|
1032
|
+
// Extract everything before encryptedContent
|
|
1033
|
+
const beforeEncrypted = trimmedString.slice(0, encryptedIdx);
|
|
1034
|
+
// Remove trailing comma and close the object
|
|
1035
|
+
const cleanedJson = beforeEncrypted.replace(/,\s*$/, '') + '}';
|
|
1036
|
+
|
|
1037
|
+
try {
|
|
1038
|
+
const partialPackage = JSON.parse(cleanedJson);
|
|
1039
|
+
return {
|
|
1040
|
+
...partialPackage,
|
|
1041
|
+
encrypted: true,
|
|
1042
|
+
encryptedContent: '[CORRUPTED - data truncated during storage]'
|
|
1043
|
+
} as QuiltMemoryPackage;
|
|
1044
|
+
} catch {
|
|
1045
|
+
// Partial extraction failed
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
// Try to find the last complete JSON object by looking for closing brace
|
|
1050
|
+
// This handles cases where truncation happened elsewhere
|
|
1051
|
+
for (let i = trimmedString.length - 1; i >= 0; i--) {
|
|
1052
|
+
if (trimmedString[i] === '}') {
|
|
1053
|
+
try {
|
|
1054
|
+
const candidate = trimmedString.slice(0, i + 1);
|
|
1055
|
+
return JSON.parse(candidate) as QuiltMemoryPackage;
|
|
1056
|
+
} catch {
|
|
1057
|
+
// This position doesn't form valid JSON, try earlier
|
|
1058
|
+
continue;
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
return null;
|
|
1064
|
+
} catch {
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
/**
|
|
1070
|
+
* Convert Uint8Array to base64 string
|
|
1071
|
+
*/
|
|
1072
|
+
private uint8ArrayToBase64(bytes: Uint8Array): string {
|
|
1073
|
+
// Use Buffer in Node.js, btoa in browser
|
|
1074
|
+
if (typeof Buffer !== 'undefined') {
|
|
1075
|
+
return Buffer.from(bytes).toString('base64');
|
|
1076
|
+
}
|
|
1077
|
+
// Browser fallback
|
|
1078
|
+
let binary = '';
|
|
1079
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
1080
|
+
binary += String.fromCharCode(bytes[i]);
|
|
1081
|
+
}
|
|
1082
|
+
return btoa(binary);
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/**
|
|
1086
|
+
* Convert base64 string to Uint8Array
|
|
1087
|
+
*/
|
|
1088
|
+
private base64ToUint8Array(base64: string): Uint8Array {
|
|
1089
|
+
// Use Buffer in Node.js, atob in browser
|
|
1090
|
+
if (typeof Buffer !== 'undefined') {
|
|
1091
|
+
return new Uint8Array(Buffer.from(base64, 'base64'));
|
|
1092
|
+
}
|
|
1093
|
+
// Browser fallback
|
|
1094
|
+
const binary = atob(base64);
|
|
1095
|
+
const bytes = new Uint8Array(binary.length);
|
|
1096
|
+
for (let i = 0; i < binary.length; i++) {
|
|
1097
|
+
bytes[i] = binary.charCodeAt(i);
|
|
1098
|
+
}
|
|
1099
|
+
return bytes;
|
|
1100
|
+
}
|
|
1101
|
+
|
|
616
1102
|
/**
|
|
617
1103
|
* Get statistics
|
|
618
1104
|
*/
|
|
@@ -631,4 +1117,14 @@ export class QuiltBatchManager {
|
|
|
631
1117
|
? this.walrusWithRelay
|
|
632
1118
|
: this.walrusWithoutRelay;
|
|
633
1119
|
}
|
|
1120
|
+
|
|
1121
|
+
/**
|
|
1122
|
+
* Get base64 converter (for external use)
|
|
1123
|
+
*/
|
|
1124
|
+
getBase64Utils() {
|
|
1125
|
+
return {
|
|
1126
|
+
encode: this.uint8ArrayToBase64.bind(this),
|
|
1127
|
+
decode: this.base64ToUint8Array.bind(this)
|
|
1128
|
+
};
|
|
1129
|
+
}
|
|
634
1130
|
}
|