@cmdoss/memwal-sdk 0.6.0

package/src/services/IndexManager.ts

@@ -0,0 +1,1004 @@
+/**
+ * IndexManager - Hybrid Index Persistence Manager
+ *
+ * Provides intelligent index management with:
+ * - Option 1: Full rebuild from Blockchain + Walrus (fallback)
+ * - Option 2: Binary serialization to/from Walrus (fast)
+ * - Hybrid: Combines both with incremental sync
+ * - On-chain MemoryIndex object for versioned index tracking
+ *
+ * Flow:
+ * 1. Initialize: Check for MemoryIndex on-chain → Load blob from Walrus
+ * 2. Save: Upload to Walrus → Create/Update MemoryIndex on-chain
+ *
+ * @module services/IndexManager
+ */
+
+import type { VectorService } from './VectorService';
+import type { StorageService } from './StorageService';
+import type { EmbeddingService } from './EmbeddingService';
+import type { TransactionService } from './TransactionService';
+import type { MemoryIndex } from './ViewService';
+
+/**
+ * Index state stored in localStorage/persistent storage
+ */
+export interface IndexState {
+  /** Blob ID of the saved index on Walrus */
+  blobId: string;
+  /** Graph blob ID on Walrus (for knowledge graph) */
+  graphBlobId?: string;
+  /** Timestamp when index was last saved */
+  lastSyncTimestamp: number;
+  /** Number of vectors in the index at save time */
+  vectorCount: number;
+  /** Index version for optimistic locking (matches on-chain version) */
+  version: number;
+  /** Dimension of vectors in the index */
+  dimension: number;
+  /** On-chain MemoryIndex object ID (if created) */
+  onChainIndexId?: string;
+}
+
+/**
+ * Serialized index package stored on Walrus
+ */
+export interface SerializedIndexPackage {
+  /** Package format version */
+  formatVersion: '1.0';
+  /** Space ID (user address) */
+  spaceId: string;
+  /** Index version */
+  version: number;
+  /** Vector dimension */
+  dimension: number;
+  /** Timestamp of serialization */
+  timestamp: number;
+  /** Serialized HNSW index as base64 (if supported) */
+  indexBinary?: string;
+  /** All vectors with their IDs for reconstruction */
+  vectors: Array<{
+    vectorId: number;
+    vector: number[];
+  }>;
+  /** Metadata for each vector */
+  metadata: Array<[number, any]>;
+  /** HNSW config used */
+  hnswConfig: {
+    maxElements: number;
+    m: number;
+    efConstruction: number;
+  };
+}
+
+/**
+ * Progress callback for long operations
+ */
+export type IndexProgressCallback = (
+  stage: 'loading' | 'rebuilding' | 'syncing' | 'saving' | 'complete',
+  progress: number,
+  message: string
+) => void;
+
+/**
+ * Options for IndexManager initialization
+ */
+export interface IndexManagerOptions {
+  /** Auto-save interval in ms (default: 5 minutes) */
+  autoSaveInterval?: number;
+  /** Enable auto-save (default: true) */
+  enableAutoSave?: boolean;
+  /** Storage key prefix for localStorage */
+  storageKeyPrefix?: string;
+  /** Progress callback */
+  onProgress?: IndexProgressCallback;
+  /** TransactionService for on-chain operations (optional) */
+  transactionService?: TransactionService;
+  /** Callback to get MemoryIndex from blockchain */
+  getMemoryIndexFromChain?: (userAddress: string) => Promise<MemoryIndex | null>;
+  /** Callback to execute transaction (for create/update MemoryIndex) */
+  executeTransaction?: (tx: any, signer: any) => Promise<{ digest: string; effects?: any; error?: string }>;
+}
+
+/**
+ * IndexManager - Manages HNSW index persistence with hybrid restore strategy
+ *
+ * Strategy:
+ * 1. Try to load from Walrus cache (fast, ~500ms)
+ * 2. If failed, rebuild from blockchain + Walrus (slow but complete)
+ * 3. Sync any new memories since last save
+ * 4. Auto-save periodically
+ */
+export class IndexManager {
+  private vectorService: VectorService;
+  private storageService: StorageService;
+  private embeddingService: EmbeddingService;
+  private options: Required<Omit<IndexManagerOptions, 'transactionService' | 'getMemoryIndexFromChain' | 'executeTransaction'>> & {
+    transactionService?: TransactionService;
+    getMemoryIndexFromChain?: (userAddress: string) => Promise<MemoryIndex | null>;
+    executeTransaction?: (tx: any, signer: any) => Promise<{ digest: string; effects?: any; error?: string }>;
+  };
+  private autoSaveTimer?: ReturnType<typeof setInterval>;
+  private indexStates: Map<string, IndexState> = new Map();
+  private vectorCache: Map<string, Map<number, number[]>> = new Map();
+
+  // Storage adapter (can be replaced for non-browser environments)
+  private storage: {
+    getItem: (key: string) => string | null;
+    setItem: (key: string, value: string) => void;
+    removeItem: (key: string) => void;
+  };
+
+  constructor(
+    vectorService: VectorService,
+    storageService: StorageService,
+    embeddingService: EmbeddingService,
+    options: IndexManagerOptions = {}
+  ) {
+    this.vectorService = vectorService;
+    this.storageService = storageService;
+    this.embeddingService = embeddingService;
+
+    this.options = {
+      autoSaveInterval: options.autoSaveInterval ?? 5 * 60 * 1000, // 5 minutes
+      enableAutoSave: options.enableAutoSave ?? true,
+      storageKeyPrefix: options.storageKeyPrefix ?? 'pdw_index_',
+      onProgress: options.onProgress ?? (() => {}),
+      transactionService: options.transactionService,
+      getMemoryIndexFromChain: options.getMemoryIndexFromChain,
+      executeTransaction: options.executeTransaction,
+    };
+
+    // Default to localStorage in browser, no-op in Node
+    this.storage = typeof localStorage !== 'undefined'
+      ? localStorage
+      : {
+          getItem: () => null,
+          setItem: () => {},
+          removeItem: () => {},
+        };
+  }
+
+  /**
+   * Set custom storage adapter (for React Native, Node.js, etc.)
+   */
+  setStorageAdapter(adapter: {
+    getItem: (key: string) => string | null;
+    setItem: (key: string, value: string) => void;
+    removeItem: (key: string) => void;
+  }): void {
+    this.storage = adapter;
+  }
+
+  /**
+   * Initialize index for a user with hybrid restore strategy
+   *
+   * @param spaceId - User address / space identifier
+   * @param getMemoriesFromChain - Function to fetch memories from blockchain
+   * @param getMemoryContent - Function to fetch memory content from Walrus
+   */
+  async initialize(
+    spaceId: string,
+    getMemoriesFromChain: () => Promise<Array<{
+      id: string;
+      blobId: string;
+      vectorId?: number;
+      category?: string;
+      importance?: number;
+      topic?: string;
+      createdAt?: number;
+    }>>,
+    getMemoryContent: (blobId: string) => Promise<{
+      content: string;
+      embedding?: number[];
+      metadata?: any;
+    }>
+  ): Promise<{
+    restored: boolean;
+    method: 'cache' | 'rebuild' | 'empty';
+    vectorCount: number;
+    syncedCount: number;
+    timeMs: number;
+  }> {
+    const startTime = Date.now();
+    let method: 'cache' | 'rebuild' | 'empty' = 'empty';
+    let vectorCount = 0;
+    let syncedCount = 0;
+
+    this.options.onProgress('loading', 0, 'Starting index initialization...');
+
+    // Step 0: Check for on-chain MemoryIndex first (source of truth)
+    let onChainIndex: MemoryIndex | null = null;
+    if (this.options.getMemoryIndexFromChain) {
+      try {
+        this.options.onProgress('loading', 5, 'Checking on-chain MemoryIndex...');
+        onChainIndex = await this.options.getMemoryIndexFromChain(spaceId);
+
+        if (onChainIndex) {
+          console.log(`📍 Found on-chain MemoryIndex: ${onChainIndex.id} (v${onChainIndex.version})`);
+          console.log(`   Index blob: ${onChainIndex.indexBlobId}`);
+          console.log(`   Graph blob: ${onChainIndex.graphBlobId}`);
+        }
+      } catch (error) {
+        console.warn('⚠️ Failed to fetch on-chain MemoryIndex:', error);
+      }
+    }
+
+    // Step 1: Try to load from on-chain index or localStorage cache
+    // Priority: on-chain index > localStorage cache
+    let cachedState = this.loadIndexState(spaceId);
+
+    // If on-chain index is newer, use it instead of localStorage
+    if (onChainIndex && onChainIndex.indexBlobId) {
+      const onChainVersion = onChainIndex.version;
+      const localVersion = cachedState?.version || 0;
+
+      if (onChainVersion > localVersion || !cachedState) {
+        console.log(`🔄 On-chain index (v${onChainVersion}) is newer than local (v${localVersion}), using on-chain`);
+        cachedState = {
+          blobId: onChainIndex.indexBlobId,
+          graphBlobId: onChainIndex.graphBlobId,
+          lastSyncTimestamp: 0, // Will sync all memories
+          vectorCount: 0,
+          version: onChainVersion,
+          dimension: 3072,
+          onChainIndexId: onChainIndex.id,
+        };
+      }
+    }
+
+    if (cachedState) {
+      try {
+        this.options.onProgress('loading', 10, 'Found cached index, loading from Walrus...');
+        await this.loadFromWalrus(spaceId, cachedState.blobId);
+
+        // Restore on-chain index ID if available
+        if (onChainIndex) {
+          const state = this.indexStates.get(spaceId);
+          if (state) {
+            state.onChainIndexId = onChainIndex.id;
+            this.indexStates.set(spaceId, state);
+          }
+        }
+
+        method = 'cache';
+        vectorCount = this.vectorService.getIndexStats(spaceId)?.currentElements || 0;
+
+        this.options.onProgress('loading', 50, `Loaded ${vectorCount} vectors from cache`);
+
+        // Step 2: Sync new memories since last save (with timeout)
+        this.options.onProgress('syncing', 60, 'Checking for new memories...');
+        try {
+          // Add 30 second timeout for sync operation
+          const syncPromise = this.syncNewMemories(
+            spaceId,
+            cachedState.lastSyncTimestamp,
+            getMemoriesFromChain,
+            getMemoryContent
+          );
+          const timeoutPromise = new Promise<number>((_, reject) =>
+            setTimeout(() => reject(new Error('Sync timeout after 30s')), 30000)
+          );
+          syncedCount = await Promise.race([syncPromise, timeoutPromise]);
+
+          if (syncedCount > 0) {
+            vectorCount += syncedCount;
+            this.options.onProgress('syncing', 90, `Synced ${syncedCount} new memories`);
+          }
+        } catch (syncError: any) {
+          console.warn('⚠️ Sync skipped:', syncError.message);
+          // Continue without sync - index is still usable from cache
+        }
+
+        this.options.onProgress('complete', 100, 'Index ready');
+      } catch (error) {
+        console.warn('⚠️ Failed to load cached index, falling back to rebuild:', error);
+        // Fall through to rebuild
+        method = 'rebuild';
+      }
+    }
+
+    // Step 3: Full rebuild if no cache or cache failed (Option 1 - slow but complete)
+    if (method !== 'cache') {
+      this.options.onProgress('rebuilding', 10, 'No cache found, rebuilding from blockchain...');
+
+      try {
+        // Add 60 second timeout for full rebuild
+        const rebuildPromise = this.fullRebuild(
+          spaceId,
+          getMemoriesFromChain,
+          getMemoryContent
+        );
+        const timeoutPromise = new Promise<{ memoriesCount: number }>((_, reject) =>
+          setTimeout(() => reject(new Error('Rebuild timeout after 60s')), 60000)
+        );
+        const result = await Promise.race([rebuildPromise, timeoutPromise]);
+
+        method = result.memoriesCount > 0 ? 'rebuild' : 'empty';
+        vectorCount = result.memoriesCount;
+      } catch (rebuildError: any) {
+        console.warn('⚠️ Rebuild failed or timed out:', rebuildError.message);
+        method = 'empty';
+        vectorCount = 0;
+      }
+    }
+
+    // Step 4: Mark as dirty if vectors were synced or rebuilt (save will happen when saveIndex is called)
+    // Note: We don't auto-save here because we don't have a signer
+    // The caller should call saveIndexWithSigner() when ready to persist
+
+    // Step 5: Start auto-save if enabled
+    if (this.options.enableAutoSave) {
+      this.startAutoSave(spaceId);
+    }
+
+    this.options.onProgress('complete', 100, 'Index initialization complete');
+
+    return {
+      restored: method === 'cache',
+      method,
+      vectorCount,
+      syncedCount,
+      timeMs: Date.now() - startTime,
+    };
+  }
+
+  /**
+   * Load index from Walrus (Option 2)
+   */
+  private async loadFromWalrus(spaceId: string, blobId: string): Promise<void> {
+    // Download serialized index from Walrus
+    const result = await this.storageService.retrieveMemoryPackage(blobId);
+
+    // Parse the index package - it may be in memoryPackage or need to be extracted from content
+    let pkg: SerializedIndexPackage;
+
+    console.log('📥 Loading index from Walrus, parsing format...');
+    console.log('  memoryPackage keys:', result.memoryPackage ? Object.keys(result.memoryPackage) : 'null');
+
+    if (result.memoryPackage?.formatVersion === '1.0') {
+      // Direct format match - SerializedIndexPackage at top level
+      console.log('  Format: Direct SerializedIndexPackage');
+      pkg = result.memoryPackage;
+    } else if (result.memoryPackage?.content) {
+      // Index was wrapped in memory package format
+      // content could be a string (JSON) or already parsed object
+      const content = result.memoryPackage.content;
+      console.log('  Content type:', typeof content);
+
+      if (typeof content === 'string') {
+        // Parse JSON string
+        try {
+          pkg = JSON.parse(content);
+          console.log('  Parsed content from JSON string');
+        } catch (parseErr) {
+          console.error('  Failed to parse content string:', parseErr);
+          throw new Error('Failed to parse index package from memory package content');
+        }
+      } else if (typeof content === 'object' && content !== null) {
+        // Already parsed object
+        pkg = content as SerializedIndexPackage;
+        console.log('  Content is already an object');
+      } else {
+        throw new Error('Invalid content type in memory package');
+      }
+    } else {
+      // Try to parse raw content
+      console.log('  Trying raw content parse...');
+      try {
+        const contentString = new TextDecoder().decode(result.content);
+        const parsed = JSON.parse(contentString);
+
+        // Check if it's wrapped in memory package format
+        if (parsed.content && typeof parsed.content === 'string') {
+          pkg = JSON.parse(parsed.content);
+        } else if (parsed.formatVersion === '1.0') {
+          pkg = parsed;
+        } else {
+          throw new Error('Unknown format');
+        }
+      } catch (err) {
+        console.error('  Raw parse failed:', err);
+        throw new Error('Invalid index package format - could not parse content');
+      }
+    }
+
+    // Validate format version
+    if (pkg.formatVersion !== '1.0') {
+      throw new Error(`Unsupported index format version: ${pkg.formatVersion}`);
+    }
+
+    console.log(`✅ Parsed index package: ${pkg.vectors?.length || 0} vectors, version ${pkg.version}`);
+
+    // Create new index with same config
+    await this.vectorService.createIndex(spaceId, pkg.dimension, {
+      maxElements: pkg.hnswConfig.maxElements,
+      m: pkg.hnswConfig.m,
+      efConstruction: pkg.hnswConfig.efConstruction,
+    });
+
+    // Initialize vector cache for this space
+    const vectorMap = new Map<number, number[]>();
+    this.vectorCache.set(spaceId, vectorMap);
+
+    // Restore vectors and metadata
+    const metadataMap = new Map<number, any>(pkg.metadata);
+
+    for (const { vectorId, vector } of pkg.vectors) {
+      await this.vectorService.addVector(spaceId, vectorId, vector, metadataMap.get(vectorId));
+      vectorMap.set(vectorId, vector);
+    }
+
+    // Update index state
+    this.indexStates.set(spaceId, {
+      blobId,
+      lastSyncTimestamp: pkg.timestamp,
+      vectorCount: pkg.vectors.length,
+      version: pkg.version,
+      dimension: pkg.dimension,
+    });
+
+    console.log(`✅ Loaded index from Walrus: ${pkg.vectors.length} vectors`);
+  }
+
+  /**
+   * Full rebuild from blockchain + Walrus (Option 1)
+   */
+  private async fullRebuild(
+    spaceId: string,
+    getMemoriesFromChain: () => Promise<Array<{
+      id: string;
+      blobId: string;
+      vectorId?: number;
+      category?: string;
+      importance?: number;
+      topic?: string;
+      createdAt?: number;
+    }>>,
+    getMemoryContent: (blobId: string) => Promise<{
+      content: string;
+      embedding?: number[];
+      metadata?: any;
+    }>
+  ): Promise<{ memoriesCount: number }> {
+    // Get all memories from blockchain
+    this.options.onProgress('rebuilding', 15, 'Fetching memories from blockchain...');
+    const memories = await getMemoriesFromChain();
+
+    if (memories.length === 0) {
+      console.log('No memories found on blockchain');
+      return { memoriesCount: 0 };
+    }
+
+    this.options.onProgress('rebuilding', 20, `Found ${memories.length} memories, rebuilding index...`);
+
+    // Determine dimension from first memory with embedding
+    let dimension = 3072; // Default
+
+    // Create index
+    await this.vectorService.createIndex(spaceId, dimension, {
+      maxElements: Math.max(10000, memories.length * 2),
+      m: 16,
+      efConstruction: 200,
+    });
+
+    // Initialize vector cache
+    const vectorMap = new Map<number, number[]>();
+    this.vectorCache.set(spaceId, vectorMap);
+
+    // Process each memory
+    let processed = 0;
+    const batchSize = 10;
+
+    for (let i = 0; i < memories.length; i += batchSize) {
+      const batch = memories.slice(i, i + batchSize);
+
+      await Promise.all(
+        batch.map(async (memory) => {
+          try {
+            // Fetch content from Walrus
+            const content = await getMemoryContent(memory.blobId);
+
+            let embedding = content.embedding;
+
+            // Generate embedding if not stored
+            if (!embedding || embedding.length === 0) {
+              const result = await this.embeddingService.embedText({ text: content.content });
+              embedding = result.vector;
+            }
+
+            // Update dimension if needed
+            if (processed === 0 && embedding.length !== dimension) {
+              dimension = embedding.length;
+            }
+
+            const vectorId = memory.vectorId ?? Date.now() % 4294967295;
+
+            // Add to index
+            await this.vectorService.addVector(spaceId, vectorId, embedding, {
+              blobId: memory.blobId,
+              memoryId: memory.id,
+              category: memory.category,
+              importance: memory.importance,
+              topic: memory.topic,
+              timestamp: memory.createdAt || Date.now(),
+            });
+
+            // Cache vector for serialization
+            vectorMap.set(vectorId, embedding);
+
+            processed++;
+          } catch (error) {
+            console.warn(`Failed to rebuild memory ${memory.id}:`, error);
+          }
+        })
+      );
+
+      const progress = 20 + Math.floor((processed / memories.length) * 70);
+      this.options.onProgress('rebuilding', progress, `Processed ${processed}/${memories.length} memories`);
+    }
+
+    console.log(`✅ Rebuilt index: ${processed} vectors`);
+
+    return { memoriesCount: processed };
+  }
+
+  /**
+   * Sync new memories since last save (incremental update)
+   */
+  private async syncNewMemories(
+    spaceId: string,
+    lastSyncTimestamp: number,
+    getMemoriesFromChain: () => Promise<Array<{
+      id: string;
+      blobId: string;
+      vectorId?: number;
+      category?: string;
+      importance?: number;
+      topic?: string;
+      createdAt?: number;
+    }>>,
+    getMemoryContent: (blobId: string) => Promise<{
+      content: string;
+      embedding?: number[];
+      metadata?: any;
+    }>
+  ): Promise<number> {
+    // Get all memories
+    const allMemories = await getMemoriesFromChain();
+
+    // Filter to only new memories
+    const newMemories = allMemories.filter(
+      (m) => (m.createdAt || 0) > lastSyncTimestamp
+    );
+
+    if (newMemories.length === 0) {
+      console.log('✅ Index is up to date');
+      return 0;
+    }
+
+    console.log(`🔄 Syncing ${newMemories.length} new memories...`);
+
+    // Get or create vector cache
+    let vectorMap = this.vectorCache.get(spaceId);
+    if (!vectorMap) {
+      vectorMap = new Map();
+      this.vectorCache.set(spaceId, vectorMap);
+    }
+
+    let synced = 0;
+
+    for (const memory of newMemories) {
+      try {
+        const content = await getMemoryContent(memory.blobId);
+
+        let embedding = content.embedding;
+
+        if (!embedding || embedding.length === 0) {
+          const result = await this.embeddingService.embedText({ text: content.content });
+          embedding = result.vector;
+        }
+
+        const vectorId = memory.vectorId ?? Date.now() % 4294967295;
+
+        await this.vectorService.addVector(spaceId, vectorId, embedding, {
+          blobId: memory.blobId,
+          memoryId: memory.id,
+          category: memory.category,
+          importance: memory.importance,
+          topic: memory.topic,
+          timestamp: memory.createdAt || Date.now(),
+        });
+
+        vectorMap.set(vectorId, embedding);
+        synced++;
+      } catch (error) {
+        console.warn(`Failed to sync memory ${memory.id}:`, error);
+      }
+    }
+
+    return synced;
+  }
+
+  /**
+   * Save index to Walrus
+   */
+  async saveIndex(spaceId: string): Promise<string | null> {
+    const stats = this.vectorService.getIndexStats(spaceId);
+    if (!stats || stats.currentElements === 0) {
+      console.log('No vectors to save');
+      return null;
+    }
+
+    // Get all vectors and metadata
+    const allVectors = this.vectorService.getAllVectors(spaceId);
+
+    // Try to get vectors from VectorService cache first, then fallback to local cache
+    let vectorMap = this.vectorService.getAllCachedVectors(spaceId);
+    if (vectorMap.size === 0) {
+      vectorMap = this.vectorCache.get(spaceId) || new Map();
+    }
+
+    if (vectorMap.size === 0) {
+      console.warn('Vector cache is empty, cannot save index');
+      return null;
+    }
+
+    // Build serialized package
+    const currentState = this.indexStates.get(spaceId);
+    const pkg: SerializedIndexPackage = {
+      formatVersion: '1.0',
+      spaceId,
+      version: (currentState?.version || 0) + 1,
+      dimension: currentState?.dimension || 3072,
+      timestamp: Date.now(),
+      vectors: allVectors
+        .filter(({ vectorId }) => vectorMap.has(vectorId))
+        .map(({ vectorId }) => ({
+          vectorId,
+          vector: vectorMap.get(vectorId)!,
+        })),
+      metadata: allVectors.map(({ vectorId, metadata }) => [vectorId, metadata]),
+      hnswConfig: {
+        maxElements: stats.maxElements || 10000,
+        m: 16,
+        efConstruction: 200,
+      },
+    };
+
+    // Upload to Walrus as JSON package
+    const result = await this.storageService.uploadMemoryPackage(
+      {
+        content: JSON.stringify(pkg),
+        embedding: [],
+        metadata: {
+          category: 'vector-index',
+          type: 'hnsw-index-package',
+          spaceId,
+          version: pkg.version,
+        },
+        identity: spaceId,
+      },
+      {
+        signer: null as any, // Will be provided by caller
+        epochs: 5, // Longer retention for index
+        deletable: true,
+      }
+    );
+
+    // Update state
+    const state: IndexState = {
+      blobId: result.blobId,
+      lastSyncTimestamp: pkg.timestamp,
+      vectorCount: pkg.vectors.length,
+      version: pkg.version,
+      dimension: pkg.dimension,
+    };
+
+    this.indexStates.set(spaceId, state);
+    this.saveIndexState(spaceId, state);
+
+    console.log(`💾 Index saved to Walrus: ${result.blobId} (${pkg.vectors.length} vectors)`);
+
+    return result.blobId;
+  }
+
+  /**
+   * Save index with signer (public API)
+   *
+   * This method:
+   * 1. Uploads index to Walrus
+   * 2. Creates or updates MemoryIndex on-chain (if transactionService is available)
+   * 3. Updates local state
+   *
+   * @param spaceId - User address
+   * @param signer - Wallet signer for transactions
+   * @returns Blob ID of saved index
+   */
+  async saveIndexWithSigner(
+    spaceId: string,
+    signer: any
+  ): Promise<string | null> {
+    const stats = this.vectorService.getIndexStats(spaceId);
+    if (!stats || stats.currentElements === 0) {
+      console.log('No vectors to save');
+      return null;
+    }
+
+    const allVectors = this.vectorService.getAllVectors(spaceId);
+
+    // Try to get vectors from VectorService cache first, then fallback to local cache
+    let vectorMap = this.vectorService.getAllCachedVectors(spaceId);
+    if (vectorMap.size === 0) {
+      vectorMap = this.vectorCache.get(spaceId) || new Map();
+    }
+
+    if (vectorMap.size === 0) {
+      console.warn('Vector cache is empty, cannot save index');
+      return null;
+    }
+
+    const currentState = this.indexStates.get(spaceId);
+    const newVersion = (currentState?.version || 0) + 1;
+
+    const pkg: SerializedIndexPackage = {
+      formatVersion: '1.0',
+      spaceId,
+      version: newVersion,
+      dimension: currentState?.dimension || 3072,
+      timestamp: Date.now(),
+      vectors: allVectors
+        .filter(({ vectorId }) => vectorMap.has(vectorId))
+        .map(({ vectorId }) => ({
+          vectorId,
+          vector: vectorMap.get(vectorId)!,
+        })),
+      metadata: allVectors.map(({ vectorId, metadata }) => [vectorId, metadata]),
+      hnswConfig: {
+        maxElements: stats.maxElements || 10000,
+        m: 16,
+        efConstruction: 200,
+      },
+    };
+
+    // Step 1: Upload index to Walrus
+    const result = await this.storageService.uploadMemoryPackage(
+      {
+        content: JSON.stringify(pkg),
+        embedding: [],
+        metadata: {
+          category: 'vector-index',
+          type: 'hnsw-index-package',
+          spaceId,
+          version: pkg.version,
+        },
+        identity: spaceId,
+      },
+      {
+        signer,
+        epochs: 5,
+        deletable: true,
+      }
+    );
+
+    console.log(`💾 Index uploaded to Walrus: ${result.blobId} (${pkg.vectors.length} vectors)`);
+
+    // Step 2: Create or update MemoryIndex on-chain
+    // SerialTransactionExecutor handles gas coin management and prevents equivocation
+    let onChainIndexId = currentState?.onChainIndexId;
+    const graphBlobId = currentState?.graphBlobId || ''; // Empty for now, could be knowledge graph
+
+    if (this.options.transactionService && this.options.executeTransaction) {
+      try {
+        if (!onChainIndexId) {
+          // Create new MemoryIndex on-chain
+          console.log('📝 Creating new MemoryIndex on-chain...');
+          const tx = this.options.transactionService.buildCreateMemoryIndex({
+            indexBlobId: result.blobId,
+            graphBlobId: graphBlobId,
+          });
+
+          const txResult = await this.options.executeTransaction(tx, signer);
+
+          if (txResult.digest && !txResult.error) {
+            console.log(`✅ MemoryIndex created on-chain: ${txResult.digest}`);
+
+            // Extract created object ID from transaction effects
+            if (txResult.effects?.created?.[0]?.reference?.objectId) {
+              onChainIndexId = txResult.effects.created[0].reference.objectId;
+              console.log(`   Object ID: ${onChainIndexId}`);
+            }
+          } else {
+            console.warn('⚠️ Failed to create on-chain MemoryIndex:', txResult.error);
+          }
+        } else {
+          // Update existing MemoryIndex on-chain
+          console.log(`📝 Updating MemoryIndex on-chain (v${currentState?.version} → v${newVersion})...`);
+          const tx = this.options.transactionService.buildUpdateMemoryIndex({
+            indexId: onChainIndexId,
+            expectedVersion: currentState?.version || 1,
+            newIndexBlobId: result.blobId,
+            newGraphBlobId: graphBlobId,
+          });
+
+          const txResult = await this.options.executeTransaction(tx, signer);
+
+          if (txResult.digest && !txResult.error) {
+            console.log(`✅ MemoryIndex updated on-chain: ${txResult.digest}`);
+          } else {
+            console.warn('⚠️ Failed to update on-chain MemoryIndex:', txResult.error);
+          }
+        }
+      } catch (error: any) {
+        console.warn('⚠️ Failed to sync MemoryIndex on-chain:', error.message);
+      }
+    }
+
+    // Step 3: Update local state
+    const state: IndexState = {
+      blobId: result.blobId,
+      graphBlobId: graphBlobId,
+      lastSyncTimestamp: pkg.timestamp,
+      vectorCount: pkg.vectors.length,
+      version: newVersion,
+      dimension: pkg.dimension,
+      onChainIndexId: onChainIndexId,
+    };
+
+    this.indexStates.set(spaceId, state);
+    this.saveIndexState(spaceId, state);
+
+    console.log(`💾 Index saved: ${result.blobId} (${pkg.vectors.length} vectors, v${newVersion})`);
+
+    return result.blobId;
+  }
+
+  /**
+   * Add vector and cache it for serialization
+   */
+  async addVectorWithCache(
+    spaceId: string,
+    vectorId: number,
+    vector: number[],
+    metadata?: any
+  ): Promise<void> {
+    await this.vectorService.addVector(spaceId, vectorId, vector, metadata);
+
+    // Cache vector for later serialization
+    let vectorMap = this.vectorCache.get(spaceId);
+    if (!vectorMap) {
+      vectorMap = new Map();
+      this.vectorCache.set(spaceId, vectorMap);
+    }
+    vectorMap.set(vectorId, vector);
+  }
+
+  /**
+   * Start auto-save interval
+   */
+  private startAutoSave(spaceId: string): void {
+    if (this.autoSaveTimer) {
+      clearInterval(this.autoSaveTimer);
+    }
+
+    this.autoSaveTimer = setInterval(async () => {
+      const stats = this.vectorService.getIndexStats(spaceId);
+      if (stats?.isDirty) {
+        try {
+          console.log('🔄 Auto-saving index...');
+          await this.saveIndex(spaceId);
+        } catch (error) {
+          console.error('Auto-save failed:', error);
+        }
+      }
+    }, this.options.autoSaveInterval);
+
+    console.log(`🔄 Auto-save enabled (every ${this.options.autoSaveInterval / 1000}s)`);
+  }
+
+  /**
+   * Stop auto-save
+   */
+  stopAutoSave(): void {
+    if (this.autoSaveTimer) {
+      clearInterval(this.autoSaveTimer);
+      this.autoSaveTimer = undefined;
+      console.log('Auto-save disabled');
+    }
+  }
+
+  /**
+   * Load index state from localStorage
+   */
+  private loadIndexState(spaceId: string): IndexState | null {
+    const key = `${this.options.storageKeyPrefix}${spaceId}`;
+    const data = this.storage.getItem(key);
+
+    if (!data) {
+      return null;
+    }
+
+    try {
+      return JSON.parse(data) as IndexState;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Save index state to localStorage
+   */
+  private saveIndexState(spaceId: string, state: IndexState): void {
+    const key = `${this.options.storageKeyPrefix}${spaceId}`;
+    this.storage.setItem(key, JSON.stringify(state));
+  }
+
+  /**
+   * Clear cached index state
+   */
+  clearIndexState(spaceId: string): void {
+    const key = `${this.options.storageKeyPrefix}${spaceId}`;
+    this.storage.removeItem(key);
+    this.indexStates.delete(spaceId);
+    this.vectorCache.delete(spaceId);
+    console.log(`Cleared index state for ${spaceId}`);
+  }
+
+  /**
+   * Get current index state
+   */
+  getIndexState(spaceId: string): IndexState | null {
+    return this.indexStates.get(spaceId) || this.loadIndexState(spaceId);
+  }
+
+  /**
+   * Force a full rebuild (useful for troubleshooting)
+   */
+  async forceRebuild(
+    spaceId: string,
+    getMemoriesFromChain: () => Promise<Array<{
+      id: string;
+      blobId: string;
+      vectorId?: number;
+      category?: string;
+      importance?: number;
+      topic?: string;
+      createdAt?: number;
+    }>>,
+    getMemoryContent: (blobId: string) => Promise<{
+      content: string;
+      embedding?: number[];
+      metadata?: any;
+    }>
+  ): Promise<{ memoriesCount: number }> {
+    // Clear existing state
+    this.clearIndexState(spaceId);
+
+    // Full rebuild
+    return this.fullRebuild(spaceId, getMemoriesFromChain, getMemoryContent);
+  }
+
+  /**
+   * Get statistics
+   */
+  getStats(spaceId: string): {
+    indexState: IndexState | null;
+    vectorCacheSize: number;
+    isAutoSaveEnabled: boolean;
+  } {
+    return {
+      indexState: this.getIndexState(spaceId),
+      vectorCacheSize: this.vectorCache.get(spaceId)?.size || 0,
+      isAutoSaveEnabled: !!this.autoSaveTimer,
+    };
+  }
+
+  /**
+   * Cleanup resources
+   */
+  async cleanup(): Promise<void> {
+    this.stopAutoSave();
+    this.indexStates.clear();
+    this.vectorCache.clear();
+  }
+}