@cmdoss/memwal-sdk 0.6.1 → 0.6.2

package/src/services/MemoryIndexService.ts

@@ -568,18 +568,34 @@ export class MemoryIndexService {
   }
 
   /**
-   * Load index from storage
+   * Load index from storage (local or Walrus)
+   *
+   * @param userAddress - User's wallet address
+   * @param indexBlobId - Optional Walrus blob ID to load from cloud
    */
   async loadIndex(userAddress: string, indexBlobId?: string): Promise<void> {
     const hnswService = await this.getHnswService();
+
+    // If blobId provided, try to load from Walrus first
+    if (indexBlobId && 'loadFromWalrus' in hnswService) {
+      console.log(`📥 Attempting to load index from Walrus: ${indexBlobId}`);
+      const walrusLoaded = await (hnswService as any).loadFromWalrus(userAddress, indexBlobId);
+      if (walrusLoaded) {
+        console.log(`✅ Memory index loaded from Walrus for user ${userAddress}`);
+        return;
+      }
+      console.log(`⚠️ Walrus load failed, falling back to local storage`);
+    }
+
+    // Fallback to local storage
     const loaded = await hnswService.loadIndex(userAddress);
     if (loaded) {
-      console.log(`✅ Memory index loaded for user ${userAddress}`);
+      console.log(`✅ Memory index loaded from local storage for user ${userAddress}`);
     }
   }
 
   /**
-   * Save index to storage
+   * Save index to local storage
    */
   async saveIndex(userAddress: string): Promise<void> {
     const hnswService = await this.getHnswService();
@@ -587,6 +603,72 @@ export class MemoryIndexService {
     console.log(`✅ Memory index saved for user ${userAddress}`);
   }
 
+  /**
+   * Sync index to Walrus cloud storage
+   *
+   * @param userAddress - User's wallet address
+   * @returns Walrus blob ID if successful, null if Walrus is disabled
+   */
+  async syncToWalrus(userAddress: string): Promise<string | null> {
+    const hnswService = await this.getHnswService();
+
+    if (!('syncToWalrus' in hnswService)) {
+      console.warn('⚠️ HNSW service does not support Walrus sync');
+      return null;
+    }
+
+    const blobId = await (hnswService as any).syncToWalrus(userAddress);
+    if (blobId) {
+      console.log(`☁️ Memory index synced to Walrus: ${blobId}`);
+    }
+    return blobId;
+  }
+
+  /**
+   * Load index directly from Walrus cloud storage
+   *
+   * @param userAddress - User's wallet address
+   * @param blobId - Walrus blob ID
+   * @returns true if successfully loaded
+   */
+  async loadFromWalrus(userAddress: string, blobId: string): Promise<boolean> {
+    const hnswService = await this.getHnswService();
+
+    if (!('loadFromWalrus' in hnswService)) {
+      console.warn('⚠️ HNSW service does not support Walrus load');
+      return false;
+    }
+
+    const loaded = await (hnswService as any).loadFromWalrus(userAddress, blobId);
+    if (loaded) {
+      console.log(`☁️ Memory index loaded from Walrus: ${blobId}`);
+    }
+    return loaded;
+  }
+
+  /**
+   * Get the Walrus blob ID for a user's index (if backed up)
+   *
+   * @param userAddress - User's wallet address
+   * @returns Blob ID or null if not backed up
+   */
+  getWalrusBlobId(userAddress: string): string | null {
+    if (this.hnswService && 'getWalrusBlobId' in this.hnswService) {
+      return (this.hnswService as any).getWalrusBlobId(userAddress);
+    }
+    return null;
+  }
+
+  /**
+   * Check if Walrus backup is enabled
+   */
+  isWalrusEnabled(): boolean {
+    if (this.hnswService && 'isWalrusEnabled' in this.hnswService) {
+      return (this.hnswService as any).isWalrusEnabled();
+    }
+    return false;
+  }
+
   /**
    * Clear user's index
    */

package/src/services/storage/QuiltBatchManager.ts

@@ -245,18 +245,26 @@ export class QuiltBatchManager {
       console.log(`   Gas saved: ${gasSaved} vs individual uploads`);
 
       // Build file results using original WalrusFile objects for metadata
-      // and uploadedFilesInfo for blobId
+      // Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
+      // Match files by identifier when reading
+      const quiltId = uploadedFilesInfo[0]?.blobId || '';
+
       const fileResults: QuiltFileResult[] = await Promise.all(
         files.map(async (originalFile, i) => {
           const identifier = await originalFile.getIdentifier() || `file-${i}`;
           const tags = await originalFile.getTags() || {};
-          // Get blobId from uploadedFilesInfo if available
-          const blobId = uploadedFilesInfo[i]?.blobId || '';
+          const fileInfo = uploadedFilesInfo[i];
+
+          // quiltPatchId is stored for reference but not used for retrieval
+          const quiltPatchId = fileInfo?.id || '';
+
+          console.log(`   File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
 
           return {
             identifier,
-            blobId,
-            quiltPatchId: undefined,
+            // Use shared quiltId as blobId - read via getBlob(quiltId).files()
+            blobId: quiltId,
+            quiltPatchId,
             tags: Object.fromEntries(
               Object.entries(tags).map(([k, v]) => [k, String(v)])
             ),
@@ -265,9 +273,6 @@ export class QuiltBatchManager {
         })
       );
 
-      // Get quiltId from first uploaded file
-      const quiltId = uploadedFilesInfo[0]?.blobId || '';
-
       return {
         quiltId,
         blobObjectId: undefined, // Not available from flow
@@ -373,18 +378,26 @@ export class QuiltBatchManager {
       console.log(`   Upload time: ${uploadTimeMs.toFixed(1)}ms`);
 
       // Build file results using original WalrusFile objects for metadata
-      // and uploadedFilesInfo for blobId
+      // Use shared quiltId as blobId - SDK can only read via getBlob(quiltId).files()
+      // Match files by identifier when reading
+      const quiltId = uploadedFilesInfo[0]?.blobId || '';
+
       const fileResults: QuiltFileResult[] = await Promise.all(
         walrusFiles.map(async (originalFile, i) => {
           const identifier = await originalFile.getIdentifier() || files[i]?.identifier || `file-${i}`;
           const tags = await originalFile.getTags() || {};
-          // Get blobId from uploadedFilesInfo if available
-          const blobId = uploadedFilesInfo[i]?.blobId || '';
+          const fileInfo = uploadedFilesInfo[i];
+
+          // quiltPatchId is stored for reference but not used for retrieval
+          const quiltPatchId = fileInfo?.id || '';
+
+          console.log(`   File ${i}: identifier=${identifier}, quiltId=${quiltId.substring(0, 20)}...`);
 
           return {
             identifier,
-            blobId,
-            quiltPatchId: undefined,
+            // Use shared quiltId as blobId - read via getBlob(quiltId).files()
+            blobId: quiltId,
+            quiltPatchId,
             tags: Object.fromEntries(
               Object.entries(tags).map(([k, v]) => [k, String(v)])
             ),
@@ -393,9 +406,6 @@ export class QuiltBatchManager {
         })
       );
 
-      // Get quiltId from first uploaded file
-      const quiltId = uploadedFilesInfo[0]?.blobId || '';
-
       return {
         quiltId,
         blobObjectId: undefined, // Not available from flow
@@ -418,14 +428,19 @@ export class QuiltBatchManager {
   /**
    * Retrieve all files from a Quilt
    *
-   * @param quiltId - The Quilt blob ID
+   * Uses getBlob().files() pattern which correctly parses Quilt structure
+   * and returns individual files with their identifiers and tags.
+   *
+   * @param quiltId - The Quilt blob ID (shared blobId)
    * @returns Array of WalrusFile objects
    */
   async getQuiltFiles(quiltId: string): Promise<Array<WalrusFile>> {
     try {
       console.log(`📂 Retrieving files from Quilt ${quiltId}...`);
 
-      const files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
+      // Use getBlob().files() to correctly parse Quilt and get individual files
+      const blob = await this.suiClient.walrus.getBlob({ blobId: quiltId });
+      const files = await blob.files();
 
       console.log(`✅ Retrieved ${files.length} files from Quilt`);
 
@@ -440,7 +455,9 @@ export class QuiltBatchManager {
   /**
    * Retrieve a specific file by identifier from a Quilt
    *
-   * @param quiltId - The Quilt blob ID
+   * Uses getBlob().files() to get all files then matches by identifier.
+   *
+   * @param quiltId - The Quilt blob ID (shared blobId)
    * @param identifier - The file identifier within the quilt
    * @returns QuiltRetrieveResult with content and metadata
    */
@@ -453,21 +470,26 @@ export class QuiltBatchManager {
     try {
       console.log(`📄 Retrieving file "${identifier}" from Quilt ${quiltId}...`);
 
-      // Get all files from quilt
-      const files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
+      // Get all files from quilt using getBlob().files()
+      const blob = await this.suiClient.walrus.getBlob({ blobId: quiltId });
+      const files = await blob.files();
 
       // Find file by identifier
-      const file = files.find(async f => {
+      let matchingFile: WalrusFile | undefined;
+      for (const f of files) {
         const fileIdentifier = await f.getIdentifier();
-        return fileIdentifier === identifier;
-      });
+        if (fileIdentifier === identifier) {
+          matchingFile = f;
+          break;
+        }
+      }
 
-      if (!file) {
+      if (!matchingFile) {
         throw new Error(`File "${identifier}" not found in Quilt`);
       }
 
-      const content = await file.bytes();
-      const tags = await file.getTags();
+      const content = await matchingFile.bytes();
+      const tags = await matchingFile.getTags();
       const retrievalTimeMs = performance.now() - startTime;
 
       console.log(`✅ Retrieved file "${identifier}" (${content.length} bytes)`);
@@ -488,14 +510,18 @@ export class QuiltBatchManager {
   /**
    * List all patches in a Quilt with their metadata
    *
-   * @param quiltId - The Quilt blob ID
+   * Uses getBlob().files() to correctly parse Quilt structure.
+   *
+   * @param quiltId - The Quilt blob ID (shared blobId)
    * @returns Array of QuiltListResult with identifiers and tags
    */
   async listQuiltPatches(quiltId: string): Promise<QuiltListResult[]> {
     try {
       console.log(`📋 Listing patches in Quilt ${quiltId}...`);
 
-      const files = await this.suiClient.walrus.getFiles({ ids: [quiltId] });
+      // Use getBlob().files() to correctly parse Quilt
+      const blob = await this.suiClient.walrus.getBlob({ blobId: quiltId });
+      const files = await blob.files();
 
       const results: QuiltListResult[] = await Promise.all(
         files.map(async (file) => {

package/src/utils/rebuildIndexNode.ts

@@ -27,7 +27,7 @@
  */
 
 import type { SuiClient } from '@mysten/sui/client';
-import { WalrusClient } from '@mysten/walrus';
+import { WalrusClient, WalrusFile } from '@mysten/walrus';
 
 export interface RebuildIndexNodeOptions {
   /** User's blockchain address */
@@ -212,60 +212,143 @@ export async function rebuildIndexNode(options: RebuildIndexNodeOptions): Promis
       };
     }
 
-    // Process each memory
+    // Process memories grouped by blobId (for Quilt support)
+    // In a Quilt, multiple memories share the same blobId
+    const memoriesByBlobId = new Map<string, typeof memories>();
+    for (const memory of memories) {
+      const list = memoriesByBlobId.get(memory.blobId) || [];
+      list.push(memory);
+      memoriesByBlobId.set(memory.blobId, list);
+    }
+
+    console.log(`[rebuildIndexNode] Unique blobIds: ${memoriesByBlobId.size} (${memoriesByBlobId.size < totalMemories ? 'Quilt detected' : 'individual blobs'})`);
+
     let indexedCount = 0;
     let failedCount = 0;
+    let processedCount = 0;
 
-    for (let i = 0; i < memories.length; i++) {
-      const memory = memories[i];
-      const progress = `Memory ${i + 1}/${totalMemories}`;
+    // Cache for Quilt files to avoid re-fetching
+    const quiltFileCache = new Map<string, WalrusFile[]>();
 
-      console.log(`[rebuildIndexNode] Processing ${progress}: ${memory.blobId.substring(0, 20)}...`);
-      onProgress?.(i + 1, totalMemories, `Processing ${progress}...`);
+    for (const [blobId, memoriesInBlob] of memoriesByBlobId) {
+      console.log(`[rebuildIndexNode] Processing blobId ${blobId.substring(0, 20)}... (${memoriesInBlob.length} memories)`);
 
       try {
-        // Download content from Walrus using SDK
-        const blobContent = await walrusClient.walrus.readBlob({ blobId: memory.blobId });
-
-        // Parse JSON content
-        const textDecoder = new TextDecoder();
-        const jsonString = textDecoder.decode(blobContent);
-        const memoryData: MemoryContent = JSON.parse(jsonString);
-
-        // Extract embedding
-        const embedding = memoryData.embedding;
-        if (!embedding || embedding.length !== 3072) {
-          throw new Error(`Invalid embedding: length=${embedding?.length || 0}`);
+        // Use getBlob().files() to correctly parse Quilt structure
+        // For regular blob: returns [singleFile]
+        // For Quilt: returns [file1, file2, ...] - all files in the quilt
+        let files: WalrusFile[];
+
+        if (quiltFileCache.has(blobId)) {
+          files = quiltFileCache.get(blobId)!;
+          console.log(`[rebuildIndexNode]   ♻️ Using cached files (${files.length} files)`);
+        } else {
+          const blob = await walrusClient.walrus.getBlob({ blobId });
+          files = await blob.files();
+          quiltFileCache.set(blobId, files);
+          console.log(`[rebuildIndexNode]   📥 Fetched ${files.length} file(s) from Walrus`);
         }
 
-        // Add to HNSW index
-        // Option A+: Store content in index for fast local retrieval (no Walrus fetch needed)
-        // Note: Only for unencrypted content. If content was encrypted, it won't be parseable anyway.
-        await hnswService.addVector(
-          userAddress,
-          memory.vectorId,
-          embedding,
-          {
-            blobId: memory.blobId,
-            memoryObjectId: memory.id,
-            category: memory.category,
-            importance: memory.importance,
-            topic: memoryData.metadata?.topic || '',
-            timestamp: memoryData.timestamp,
-            // Option A+: Store content for fast retrieval (avoids Walrus fetch on search)
-            content: memoryData.content,
-            isEncrypted: false // If we can parse it, it's not encrypted
+        // For each memory in this blobId
+        for (let i = 0; i < memoriesInBlob.length; i++) {
+          const memory = memoriesInBlob[i];
+          processedCount++;
+          const progress = `Memory ${processedCount}/${totalMemories}`;
+
+          console.log(`[rebuildIndexNode] Processing ${progress}: vectorId=${memory.vectorId}`);
+          onProgress?.(processedCount, totalMemories, `Processing ${progress}...`);
+
+          try {
+            // Determine which file to use
+            // For Quilt: match by index
+            // For single blob: use the only file
+            const fileIndex = files.length === 1 ? 0 : Math.min(i, files.length - 1);
+            const file = files[fileIndex];
+
+            if (!file) {
+              throw new Error(`No file found at index ${fileIndex}`);
+            }
+
+            // Get file content
+            const rawBytes = await file.bytes();
+            const rawText = new TextDecoder().decode(rawBytes);
+            const trimmedText = rawText.trim();
+
+            // Get file identifier and tags if available (for Quilts)
+            const identifier = await file.getIdentifier();
+            const tags = await file.getTags();
+
+            if (identifier) {
+              console.log(`[rebuildIndexNode]   📎 File identifier: ${identifier}`);
+            }
+
+            let content: string;
+            let embedding: number[];
+            let metadata: { category?: string; importance?: number; topic?: string } = {};
+            let timestamp = Date.now();
+
+            if (trimmedText.startsWith('{') && trimmedText.endsWith('}')) {
+              // JSON package format (correct format)
+              try {
+                const memoryData: MemoryContent = JSON.parse(trimmedText);
+                content = memoryData.content;
+                embedding = memoryData.embedding;
+                metadata = memoryData.metadata || {};
+                timestamp = memoryData.timestamp || Date.now();
+
+                if (!embedding || embedding.length !== 3072) {
+                  throw new Error(`Invalid embedding in JSON: length=${embedding?.length || 0}`);
+                }
+
+                console.log(`[rebuildIndexNode]   📦 Format: JSON package`);
+              } catch (jsonError) {
+                throw new Error(`Invalid JSON structure: ${(jsonError as Error).message}`);
+              }
+            } else if (trimmedText.length > 0 && !trimmedText.includes('\x00') && trimmedText.length < 10000) {
+              // Plain text format - cannot index without embedding
+              throw new Error('Plain text format detected but no embedding available - skip');
+            } else {
+              throw new Error('Binary, encrypted, or empty content - cannot index');
+            }
+
+            // Add to HNSW index
+            await hnswService.addVector(
+              userAddress,
+              memory.vectorId,
+              embedding,
+              {
+                blobId: memory.blobId,
+                memoryObjectId: memory.id,
+                category: metadata.category || memory.category || tags?.['category'],
+                importance: metadata.importance || memory.importance || parseInt(tags?.['importance'] || '5'),
+                topic: metadata.topic || tags?.['topic'] || '',
+                timestamp,
+                content,
+                isEncrypted: false
+              }
+            );
+
+            indexedCount++;
+            console.log(`[rebuildIndexNode]   ✓ Indexed: "${content.substring(0, 30)}..."`);
+
+          } catch (error: any) {
+            failedCount++;
+            const errorMsg = error.message || String(error);
+            errors.push({ blobId: memory.blobId, error: errorMsg });
+            console.error(`[rebuildIndexNode]   ✗ Failed: ${errorMsg}`);
           }
-        );
-
-        indexedCount++;
-        console.log(`[rebuildIndexNode]   ✓ Indexed: ${memory.blobId.substring(0, 20)}...`);
+        }
 
       } catch (error: any) {
-        failedCount++;
+        // Failed to fetch files for this blobId
         const errorMsg = error.message || String(error);
-        errors.push({ blobId: memory.blobId, error: errorMsg });
-        console.error(`[rebuildIndexNode]   ✗ Failed: ${errorMsg}`);
+        console.error(`[rebuildIndexNode]   ✗ Failed to fetch blobId: ${errorMsg}`);
+
+        for (const memory of memoriesInBlob) {
+          processedCount++;
+          failedCount++;
+          errors.push({ blobId: memory.blobId, error: `Failed to fetch blob: ${errorMsg}` });
+        }
       }
     }
 

package/src/vector/NodeHnswService.ts

@@ -248,6 +248,7 @@ export class NodeHnswService implements IHnswService {
     vector: number[],
     metadata?: Record<string, any>
   ): Promise<void> {
+    console.log(`[NodeHnswService] addVector: userAddress=${userAddress.slice(0, 10)}..., vectorId=${vectorId}, dims=${vector.length}`);
     await this.getOrCreateIndex(userAddress);
 
     const entry = this.indexCache.get(userAddress);
@@ -262,9 +263,11 @@ export class NodeHnswService implements IHnswService {
     }
     entry.isDirty = true;
     this.batchStats.pendingJobs++;
+    console.log(`[NodeHnswService] addVector: Added to pending batch. Pending: ${entry.pendingVectors.size}, isDirty: ${entry.isDirty}`);
 
     // Flush if batch is large enough
     if (entry.pendingVectors.size >= 50) {
+      console.log(`[NodeHnswService] addVector: Batch size >= 50, auto-flushing...`);
       await this.flushBatch(userAddress);
     }
   }
@@ -338,8 +341,12 @@ export class NodeHnswService implements IHnswService {
 
   async flushBatch(userAddress: string): Promise<void> {
     const entry = this.indexCache.get(userAddress);
-    if (!entry || entry.pendingVectors.size === 0) return;
+    if (!entry || entry.pendingVectors.size === 0) {
+      console.log(`[NodeHnswService] flushBatch: No pending vectors for ${userAddress.slice(0, 10)}..., skipping`);
+      return;
+    }
 
+    console.log(`[NodeHnswService] flushBatch: Flushing ${entry.pendingVectors.size} vectors for ${userAddress.slice(0, 10)}...`);
     const startTime = Date.now();
     let processed = 0;
 
@@ -348,6 +355,7 @@ export class NodeHnswService implements IHnswService {
         entry.index.addPoint(vector, vectorId);
         processed++;
       }
+      console.log(`[NodeHnswService] flushBatch: Added ${processed} points to index`);
 
       entry.pendingVectors.clear();
       entry.isDirty = true;
@@ -361,21 +369,28 @@ export class NodeHnswService implements IHnswService {
         (this.batchStats.averageProcessingTime + processingTime) / 2;
 
       // Auto-save
+      console.log(`[NodeHnswService] flushBatch: Calling saveIndex()...`);
       await this.saveIndex(userAddress);
+      console.log(`[NodeHnswService] flushBatch: Complete in ${processingTime}ms`);
     } catch (error) {
       this.batchStats.failedJobs++;
-      console.error('[NodeHnswService] Flush batch error:', error);
+      console.error('[NodeHnswService] flushBatch error:', error);
       throw error;
     }
   }
 
   async saveIndex(userAddress: string): Promise<void> {
     const entry = this.indexCache.get(userAddress);
-    if (!entry) return;
+    if (!entry) {
+      console.log(`[NodeHnswService] saveIndex: No cache entry for ${userAddress}, skipping`);
+      return;
+    }
 
     try {
       const indexPath = this.getIndexPath(userAddress);
+      console.log(`[NodeHnswService] saveIndex: Writing to ${indexPath}`);
       entry.index.writeIndex(indexPath);
+      console.log(`[NodeHnswService] saveIndex: writeIndex() complete`);
 
       // Save metadata separately
       const fs = await import('fs/promises');
@@ -385,19 +400,26 @@ export class NodeHnswService implements IHnswService {
         metadataObj[k] = v;
       }
 
-      // Preserve existing walrus blob ID if present
-      const existingMeta: Record<string, any> = {};
+      // Preserve existing metadata and merge with in-memory metadata
+      // This prevents data loss when cache is cleared between requests
+      let existingMeta: Record<string, any> = {};
       try {
         const existingContent = await fs.readFile(metadataPath, 'utf-8');
-        Object.assign(existingMeta, JSON.parse(existingContent));
+        existingMeta = JSON.parse(existingContent);
       } catch {
         // No existing metadata file
       }
 
+      // Merge: existing metadata + in-memory metadata (in-memory takes priority for same keys)
+      const mergedMetadata = {
+        ...(existingMeta.metadata || {}),
+        ...metadataObj
+      };
+
       await fs.writeFile(metadataPath, JSON.stringify({
         version: entry.version,
         dimensions: entry.dimensions,
-        metadata: metadataObj,
+        metadata: mergedMetadata,
         walrusBlobId: existingMeta.walrusBlobId,
         walrusSyncTime: existingMeta.walrusSyncTime
       }));

package/src/wallet/index.ts

@@ -1,17 +1,17 @@
-/**
- * Wallet Module
- *
- * @deprecated This module is deprecated. Use CapabilityService and ContextNamespace instead.
- * - MainWalletService → CapabilityService (from services/CapabilityService)
- * - ContextWalletService → ContextNamespace (from client/namespaces/ContextNamespace)
- *
- * These services will be removed in the next major version.
- */
-
-/** @deprecated Use CapabilityService instead */
-export { MainWalletService } from './MainWalletService';
-export type { MainWalletServiceConfig } from './MainWalletService';
-
-/** @deprecated Use ContextNamespace instead */
-export { ContextWalletService } from './ContextWalletService';
-export type { ContextWalletServiceConfig } from './ContextWalletService';
+/**
+ * Wallet Module
+ *
+ * @deprecated This module is deprecated. Use CapabilityService and ContextNamespace instead.
+ * - MainWalletService → CapabilityService (from services/CapabilityService)
+ * - ContextWalletService → ContextNamespace (from client/namespaces/ContextNamespace)
+ *
+ * These services will be removed in the next major version.
+ */
+
+/** @deprecated Use CapabilityService instead */
+export { MainWalletService } from './MainWalletService';
+export type { MainWalletServiceConfig } from './MainWalletService';
+
+/** @deprecated Use ContextNamespace instead */
+export { ContextWalletService } from './ContextWalletService';
+export type { ContextWalletServiceConfig } from './ContextWalletService';