@cmdoss/memwal-sdk 0.6.1 → 0.7.0

package/dist/services/storage/QuiltBatchManager.js

@@ -87,30 +87,64 @@ export class QuiltBatchManager {
         let totalSize = 0;
         console.log(`📦 Uploading batch of ${memories.length} memories as Quilt (writeFilesFlow)...`);
         try {
-            // Create WalrusFile for each memory with plaintext tags
+            // Create WalrusFile for each memory as JSON package
+            // This format is consistent with regular memory storage
             const files = memories.map((memory, index) => {
                 const identifier = memory.id
                     ? `memory-${memory.id}.json`
                     : `memory-${Date.now()}-${index}-${Math.random().toString(36).slice(2, 9)}.json`;
-                totalSize += memory.encryptedContent.length;
+                const isEncrypted = !!memory.encryptedContent && memory.encryptedContent.length > 0;
+                const timestamp = Date.now();
+                // Create memory package (JSON format - consistent with regular storage)
+                const memoryPackage = {
+                    // Content: plaintext if not encrypted, empty if encrypted
+                    content: isEncrypted ? '' : memory.content,
+                    embedding: memory.embedding,
+                    metadata: {
+                        category: memory.category,
+                        importance: memory.importance,
+                        topic: memory.topic,
+                        ...(memory.summary ? { summary: memory.summary } : {}),
+                        ...(memory.id ? { memoryId: memory.id } : {})
+                    },
+                    timestamp,
+                    version: '2.0.0', // Quilt JSON package version
+                    encrypted: isEncrypted,
+                    // Store encrypted content as base64 for JSON compatibility
+                    ...(isEncrypted && memory.encryptedContent ? {
+                        encryptedContent: this.uint8ArrayToBase64(memory.encryptedContent)
+                    } : {})
+                };
+                // Serialize to JSON and encode as bytes
+                const jsonString = JSON.stringify(memoryPackage);
+                const contents = new TextEncoder().encode(jsonString);
+                totalSize += contents.length;
+                // Diagnostic logging for debugging Quilt corruption issues
+                console.log(`   📝 File ${index}: identifier=${identifier}`);
+                console.log(`      JSON string length: ${jsonString.length} chars`);
+                console.log(`      Encoded bytes: ${contents.length} bytes`);
+                console.log(`      Last 50 chars of JSON: ...${jsonString.slice(-50)}`);
+                console.log(`      Last 10 bytes (hex): ${Array.from(contents.slice(-10)).map(b => b.toString(16).padStart(2, '0')).join(' ')}`);
                 return WalrusFile.from({
-                    contents: memory.encryptedContent,
+                    contents,
                     identifier,
                     tags: {
-                        // Core metadata (plaintext for filtering)
+                        // Core metadata (plaintext for filtering without decryption)
+                        'content-type': 'application/json',
                         'category': memory.category,
                         'importance': memory.importance.toString(),
                         'topic': memory.topic,
-                        'timestamp': new Date().toISOString(),
-                        'created_at': new Date().toISOString(),
+                        'timestamp': new Date(timestamp).toISOString(),
+                        'created_at': new Date(timestamp).toISOString(),
                         // Encryption info
-                        'encrypted': 'true',
-                        'encryption_type': 'seal',
+                        'encrypted': isEncrypted ? 'true' : 'false',
+                        ...(isEncrypted ? { 'encryption_type': 'seal' } : {}),
                         // Owner
                         'owner': options.userAddress,
                         // Content info
-                        'content_size': memory.encryptedContent.length.toString(),
+                        'content_size': contents.length.toString(),
                         'embedding_dimensions': memory.embedding.length.toString(),
+                        'package_version': '2.0.0',
                         // Optional rich metadata
                         ...(memory.summary ? { 'summary': memory.summary } : {}),
                         ...(memory.id ? { 'memory_id': memory.id } : {})
@@ -163,22 +197,25 @@ export class QuiltBatchManager {
             console.log(`   Upload time: ${uploadTimeMs.toFixed(1)}ms`);
             console.log(`   Gas saved: ${gasSaved} vs individual uploads`);
             // Build file results using original WalrusFile objects for metadata
-            // and uploadedFilesInfo for blobId
+            // Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
+            // Match files by identifier when reading
+            const quiltId = uploadedFilesInfo[0]?.blobId || '';
             const fileResults = await Promise.all(files.map(async (originalFile, i) => {
                 const identifier = await originalFile.getIdentifier() || `file-${i}`;
                 const tags = await originalFile.getTags() || {};
-                // Get blobId from uploadedFilesInfo if available
-                const blobId = uploadedFilesInfo[i]?.blobId || '';
+                const fileInfo = uploadedFilesInfo[i];
+                // quiltPatchId is stored for reference but not used for retrieval
+                const quiltPatchId = fileInfo?.id || '';
+                console.log(`   File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
                 return {
                     identifier,
-                    blobId,
-                    quiltPatchId: undefined,
+                    // Use shared quiltId as blobId - read via getBlob(quiltId).files()
+                    blobId: quiltId,
+                    quiltPatchId,
                     tags: Object.fromEntries(Object.entries(tags).map(([k, v]) => [k, String(v)])),
-                    size: memories[i]?.encryptedContent.length || 0
+                    size: memories[i]?.encryptedContent?.length || memories[i]?.content?.length || 0
                 };
             }));
-            // Get quiltId from first uploaded file
-            const quiltId = uploadedFilesInfo[0]?.blobId || '';
             return {
                 quiltId,
                 blobObjectId: undefined, // Not available from flow
@@ -263,22 +300,25 @@ export class QuiltBatchManager {
             console.log(`   Files uploaded: ${uploadedFilesInfo.length}`);
             console.log(`   Upload time: ${uploadTimeMs.toFixed(1)}ms`);
             // Build file results using original WalrusFile objects for metadata
-            // and uploadedFilesInfo for blobId
+            // Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
+            // Match files by identifier when reading
+            const quiltId = uploadedFilesInfo[0]?.blobId || '';
             const fileResults = await Promise.all(walrusFiles.map(async (originalFile, i) => {
                 const identifier = await originalFile.getIdentifier() || files[i]?.identifier || `file-${i}`;
                 const tags = await originalFile.getTags() || {};
-                // Get blobId from uploadedFilesInfo if available
-                const blobId = uploadedFilesInfo[i]?.blobId || '';
+                const fileInfo = uploadedFilesInfo[i];
+                // quiltPatchId is stored for reference but not used for retrieval
+                const quiltPatchId = fileInfo?.id || '';
+                console.log(`   File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
                 return {
                     identifier,
-                    blobId,
-                    quiltPatchId: undefined,
+                    // Use shared quiltId as blobId - read via getBlob(quiltId).files()
+                    blobId: quiltId,
+                    quiltPatchId,
                     tags: Object.fromEntries(Object.entries(tags).map(([k, v]) => [k, String(v)])),
                     size: files[i]?.data.length || 0
                 };
             }));
-            // Get quiltId from first uploaded file
-            const quiltId = uploadedFilesInfo[0]?.blobId || '';
             return {
                 quiltId,
                 blobObjectId: undefined, // Not available from flow
@@ -299,14 +339,29 @@ export class QuiltBatchManager {
     /**
      * Retrieve all files from a Quilt
      *
-     * @param quiltId - The Quilt blob ID
+     * Uses getBlob().files() pattern which correctly parses Quilt structure
+     * and returns individual files with their identifiers and tags.
+     *
+     * @param quiltId - The Quilt blob ID (shared blobId)
      * @returns Array of WalrusFile objects
      */
     async getQuiltFiles(quiltId) {
         try {
             console.log(`📂 Retrieving files from Quilt ${quiltId}...`);
-            const files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
-            console.log(`✅ Retrieved ${files.length} files from Quilt`);
+            // Try to parse as Quilt first (getBlob().files() returns ALL files in Quilt)
+            // Fall back to getFiles() for regular blobs
+            let files;
+            try {
+                const blob = await this.suiClient.walrus.getBlob({ blobId: quiltId });
+                files = await blob.files();
+                console.log(`✅ Retrieved ${files.length} files from Quilt`);
+            }
+            catch (quiltError) {
+                // Not a Quilt - try as regular blob
+                console.log(`📄 Not a Quilt format, fetching as regular blob...`);
+                files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
+                console.log(`✅ Retrieved ${files.length} file(s) as regular blob`);
+            }
             return files;
         }
         catch (error) {
@@ -317,7 +372,9 @@ export class QuiltBatchManager {
     /**
      * Retrieve a specific file by identifier from a Quilt
      *
-     * @param quiltId - The Quilt blob ID
+     * Uses getBlob().files() to get all files then matches by identifier.
+     *
+     * @param quiltId - The Quilt blob ID (shared blobId)
      * @param identifier - The file identifier within the quilt
      * @returns QuiltRetrieveResult with content and metadata
      */
@@ -325,18 +382,22 @@ export class QuiltBatchManager {
         const startTime = performance.now();
         try {
             console.log(`📄 Retrieving file "${identifier}" from Quilt ${quiltId}...`);
-            // Get all files from quilt
-            const files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
+            // Get all files from the blob (Quilt or regular)
+            const files = await this.getQuiltFiles(quiltId);
             // Find file by identifier
-            const file = files.find(async (f) => {
+            let matchingFile;
+            for (const f of files) {
                 const fileIdentifier = await f.getIdentifier();
-                return fileIdentifier === identifier;
-            });
-            if (!file) {
+                if (fileIdentifier === identifier) {
+                    matchingFile = f;
+                    break;
+                }
+            }
+            if (!matchingFile) {
                 throw new Error(`File "${identifier}" not found in Quilt`);
             }
-            const content = await file.bytes();
-            const tags = await file.getTags();
+            const content = await matchingFile.bytes();
+            const tags = await matchingFile.getTags();
             const retrievalTimeMs = performance.now() - startTime;
             console.log(`✅ Retrieved file "${identifier}" (${content.length} bytes)`);
             return {
@@ -354,13 +415,16 @@ export class QuiltBatchManager {
     /**
      * List all patches in a Quilt with their metadata
      *
-     * @param quiltId - The Quilt blob ID
+     * Uses getBlob().files() to correctly parse Quilt structure.
+     *
+     * @param quiltId - The Quilt blob ID (shared blobId)
      * @returns Array of QuiltListResult with identifiers and tags
      */
     async listQuiltPatches(quiltId) {
         try {
             console.log(`📋 Listing patches in Quilt ${quiltId}...`);
-            const files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
+            // Get all files from the blob (Quilt or regular)
+            const files = await this.getQuiltFiles(quiltId);
             const results = await Promise.all(files.map(async (file) => {
                 const identifier = await file.getIdentifier() || 'unknown';
                 const tags = await file.getTags();
@@ -445,8 +509,347 @@ export class QuiltBatchManager {
         return matchingFiles;
     }
     // ==========================================================================
+    // JSON Memory Package Retrieval
+    // ==========================================================================
+    /**
+     * Retrieve a memory package as JSON from a Quilt
+     *
+     * Uses file.json() for efficient parsing (SDK handles it)
+     *
+     * @param quiltId - The Quilt blob ID
+     * @param identifier - The file identifier within the quilt
+     * @returns QuiltMemoryRetrieveResult with parsed memory package
+     */
+    async getMemoryPackage(quiltId, identifier) {
+        const startTime = performance.now();
+        try {
+            console.log(`📄 Retrieving memory package "${identifier}" from Quilt ${quiltId}...`);
+            // Get all files from the blob
+            const files = await this.getQuiltFiles(quiltId);
+            // Find file by identifier
+            let matchingFile;
+            for (const f of files) {
+                const fileIdentifier = await f.getIdentifier();
+                if (fileIdentifier === identifier) {
+                    matchingFile = f;
+                    break;
+                }
+            }
+            if (!matchingFile) {
+                throw new Error(`File "${identifier}" not found in Quilt`);
+            }
+            const tags = await matchingFile.getTags();
+            let memoryPackage;
+            try {
+                // Parse directly as JSON (SDK handles it!)
+                memoryPackage = await matchingFile.json();
+            }
+            catch (parseError) {
+                // Try partial recovery for truncated JSON
+                console.warn(`⚠️ JSON parse failed for "${identifier}", attempting recovery...`);
+                const bytes = await matchingFile.bytes();
+                const recovered = this.tryRecoverTruncatedPackage(bytes);
+                if (recovered) {
+                    console.log(`🔧 Partially recovered "${identifier}" (encryptedContent may be corrupted)`);
+                    memoryPackage = recovered;
+                }
+                else {
+                    throw parseError;
+                }
+            }
+            const retrievalTimeMs = performance.now() - startTime;
+            console.log(`✅ Retrieved memory package "${identifier}" (${retrievalTimeMs.toFixed(1)}ms)`);
+            return {
+                identifier,
+                memoryPackage,
+                tags,
+                retrievalTimeMs
+            };
+        }
+        catch (error) {
+            console.error(`❌ Failed to retrieve memory package:`, error);
+            throw new Error(`Failed to retrieve memory package "${identifier}": ${error}`);
+        }
+    }
+    /**
+     * Retrieve all memory packages from a Quilt as JSON
+     *
+     * @param quiltId - The Quilt blob ID
+     * @returns Array of memory packages with metadata
+     */
+    async getAllMemoryPackages(quiltId) {
+        const startTime = performance.now();
+        try {
+            console.log(`📂 Retrieving all memory packages from Quilt ${quiltId}...`);
+            const files = await this.getQuiltFiles(quiltId);
+            const results = [];
+            for (const file of files) {
+                const identifier = await file.getIdentifier() || 'unknown';
+                const tags = await file.getTags();
+                try {
+                    // Parse as JSON
+                    const memoryPackage = await file.json();
+                    results.push({
+                        identifier,
+                        memoryPackage,
+                        tags,
+                        retrievalTimeMs: 0 // Individual timing not tracked in batch
+                    });
+                }
+                catch (parseError) {
+                    console.warn(`⚠️ Failed to parse "${identifier}" as JSON:`, parseError);
+                    // Try partial recovery for truncated JSON
+                    try {
+                        const bytes = await file.bytes();
+                        const recoveredPackage = this.tryRecoverTruncatedPackage(bytes);
+                        if (recoveredPackage) {
+                            console.log(`🔧 Partially recovered "${identifier}" (encryptedContent truncated)`);
+                            results.push({
+                                identifier,
+                                memoryPackage: recoveredPackage,
+                                tags,
+                                retrievalTimeMs: 0
+                            });
+                        }
+                    }
+                    catch {
+                        // Skip files that can't be recovered
+                        console.warn(`❌ Could not recover "${identifier}"`);
+                    }
+                }
+            }
+            const totalTimeMs = performance.now() - startTime;
+            console.log(`✅ Retrieved ${results.length} memory packages (${totalTimeMs.toFixed(1)}ms)`);
+            return results;
+        }
+        catch (error) {
+            console.error(`❌ Failed to retrieve memory packages:`, error);
+            throw new Error(`Failed to retrieve memory packages from Quilt ${quiltId}: ${error}`);
+        }
+    }
+    /**
+     * Find a specific memory in a Quilt using multiple matching strategies
+     *
+     * Strategies (in order of priority):
+     * 1. Match by tags['memory_id'] === memoryId
+     * 2. Match by identifier === `memory-${memoryId}.json`
+     * 3. Match by JSON metadata.memoryId === memoryId
+     * 4. Fallback to index-based matching (if fileIndex provided)
+     *
+     * @param quiltId - The Quilt blob ID
+     * @param memoryId - The memory ID (usually vectorId) to find
+     * @param fileIndex - Optional fallback index if other strategies fail
+     * @returns The matching memory package result, or null if not found
+     */
+    async findMemoryInQuilt(quiltId, memoryId, fileIndex) {
+        const startTime = performance.now();
+        try {
+            console.log(`🔍 Finding memory "${memoryId}" in Quilt ${quiltId.substring(0, 20)}...`);
+            const files = await this.getQuiltFiles(quiltId);
+            let matchedFile;
+            let matchStrategy = '';
+            // Strategy 1: Match by tags['memory_id']
+            for (const f of files) {
+                const tags = await f.getTags();
+                if (tags?.['memory_id'] === memoryId) {
+                    matchedFile = f;
+                    matchStrategy = 'memory_id tag';
+                    break;
+                }
+            }
+            // Strategy 2: Match by identifier pattern "memory-{memoryId}.json"
+            if (!matchedFile) {
+                for (const f of files) {
+                    const identifier = await f.getIdentifier();
+                    if (identifier === `memory-${memoryId}.json`) {
+                        matchedFile = f;
+                        matchStrategy = 'identifier pattern';
+                        break;
+                    }
+                }
+            }
+            // Strategy 3: Parse JSON to find matching metadata.memoryId
+            if (!matchedFile) {
+                for (const f of files) {
+                    try {
+                        const json = await f.json();
+                        if (json?.metadata?.memoryId === memoryId) {
+                            matchedFile = f;
+                            matchStrategy = 'JSON metadata.memoryId';
+                            break;
+                        }
+                    }
+                    catch {
+                        // Not valid JSON, continue
+                    }
+                }
+            }
+            // Strategy 4: Fallback to index-based matching
+            if (!matchedFile && fileIndex !== undefined && fileIndex < files.length) {
+                matchedFile = files[fileIndex];
+                matchStrategy = `index fallback (${fileIndex})`;
+            }
+            if (!matchedFile) {
+                console.log(`❌ Memory "${memoryId}" not found in Quilt (${files.length} files)`);
+                return null;
+            }
+            const identifier = await matchedFile.getIdentifier() || 'unknown';
+            const tags = await matchedFile.getTags();
+            let memoryPackage;
+            try {
+                memoryPackage = await matchedFile.json();
+            }
+            catch (parseError) {
+                // Try recovery for truncated JSON
+                const bytes = await matchedFile.bytes();
+                const recovered = this.tryRecoverTruncatedPackage(bytes);
+                if (recovered) {
+                    memoryPackage = recovered;
+                }
+                else {
+                    throw parseError;
+                }
+            }
+            const retrievalTimeMs = performance.now() - startTime;
+            console.log(`✅ Found memory "${memoryId}" via ${matchStrategy} (${identifier}) in ${retrievalTimeMs.toFixed(1)}ms`);
+            return {
+                identifier,
+                memoryPackage,
+                tags,
+                retrievalTimeMs
+            };
+        }
+        catch (error) {
+            console.error(`❌ Failed to find memory in Quilt:`, error);
+            throw new Error(`Failed to find memory "${memoryId}" in Quilt ${quiltId}: ${error}`);
+        }
+    }
+    /**
+     * Get memory content from a Quilt file
+     *
+     * Handles both encrypted and unencrypted content:
+     * - Unencrypted: Returns content directly from package
+     * - Encrypted: Returns decrypted content if sessionKey provided, otherwise throws
+     *
+     * @param quiltId - The Quilt blob ID
+     * @param identifier - The file identifier
+     * @param sessionKey - Optional session key for encrypted content
+     * @returns Memory content as string
+     */
+    async getMemoryContent(quiltId, identifier, decryptFn) {
+        const result = await this.getMemoryPackage(quiltId, identifier);
+        const pkg = result.memoryPackage;
+        if (!pkg.encrypted) {
+            // Not encrypted - return content directly
+            return pkg.content;
+        }
+        if (!pkg.encryptedContent) {
+            throw new Error('Memory is marked as encrypted but no encrypted content found');
+        }
+        if (!decryptFn) {
+            throw new Error('Memory is encrypted. Provide decryptFn to decrypt content.');
+        }
+        // Decrypt using provided function
+        return await decryptFn(pkg.encryptedContent);
+    }
+    // ==========================================================================
     // Utility Methods
     // ==========================================================================
+    /**
+     * Try to recover a partially truncated memory package
+     *
+     * Handles cases where JSON was truncated (e.g., in the middle of encryptedContent)
+     * by extracting metadata and marking the encrypted content as corrupted.
+     *
+     * @param bytes - Raw bytes of the file
+     * @returns Recovered QuiltMemoryPackage or null if recovery fails
+     */
+    tryRecoverTruncatedPackage(bytes) {
+        try {
+            const rawString = new TextDecoder().decode(bytes);
+            // Find and trim trailing null bytes
+            let lastValidIndex = rawString.length - 1;
+            while (lastValidIndex >= 0 && rawString.charCodeAt(lastValidIndex) === 0) {
+                lastValidIndex--;
+            }
+            const trimmedString = rawString.slice(0, lastValidIndex + 1);
+            // First try to parse as-is (maybe nulls were the only issue)
+            try {
+                return JSON.parse(trimmedString);
+            }
+            catch {
+                // Continue to partial recovery
+            }
+            // Look for encryptedContent field - data likely truncated there
+            const encryptedIdx = trimmedString.indexOf('"encryptedContent":"');
+            if (encryptedIdx > 0) {
+                // Extract everything before encryptedContent
+                const beforeEncrypted = trimmedString.slice(0, encryptedIdx);
+                // Remove trailing comma and close the object
+                const cleanedJson = beforeEncrypted.replace(/,\s*$/, '') + '}';
+                try {
+                    const partialPackage = JSON.parse(cleanedJson);
+                    return {
+                        ...partialPackage,
+                        encrypted: true,
+                        encryptedContent: '[CORRUPTED - data truncated during storage]'
+                    };
+                }
+                catch {
+                    // Partial extraction failed
+                }
+            }
+            // Try to find the last complete JSON object by looking for closing brace
+            // This handles cases where truncation happened elsewhere
+            for (let i = trimmedString.length - 1; i >= 0; i--) {
+                if (trimmedString[i] === '}') {
+                    try {
+                        const candidate = trimmedString.slice(0, i + 1);
+                        return JSON.parse(candidate);
+                    }
+                    catch {
+                        // This position doesn't form valid JSON, try earlier
+                        continue;
+                    }
+                }
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Convert Uint8Array to base64 string
+     */
+    uint8ArrayToBase64(bytes) {
+        // Use Buffer in Node.js, btoa in browser
+        if (typeof Buffer !== 'undefined') {
+            return Buffer.from(bytes).toString('base64');
+        }
+        // Browser fallback
+        let binary = '';
+        for (let i = 0; i < bytes.length; i++) {
+            binary += String.fromCharCode(bytes[i]);
+        }
+        return btoa(binary);
+    }
+    /**
+     * Convert base64 string to Uint8Array
+     */
+    base64ToUint8Array(base64) {
+        // Use Buffer in Node.js, atob in browser
+        if (typeof Buffer !== 'undefined') {
+            return new Uint8Array(Buffer.from(base64, 'base64'));
+        }
+        // Browser fallback
+        const binary = atob(base64);
+        const bytes = new Uint8Array(binary.length);
+        for (let i = 0; i < binary.length; i++) {
+            bytes[i] = binary.charCodeAt(i);
+        }
+        return bytes;
+    }
     /**
      * Get statistics
      */
@@ -464,5 +867,14 @@ export class QuiltBatchManager {
             ? this.walrusWithRelay
             : this.walrusWithoutRelay;
     }
+    /**
+     * Get base64 converter (for external use)
+     */
+    getBase64Utils() {
+        return {
+            encode: this.uint8ArrayToBase64.bind(this),
+            decode: this.base64ToUint8Array.bind(this)
+        };
+    }
 }
 //# sourceMappingURL=QuiltBatchManager.js.map