@cmdoss/memwal-sdk 0.6.0

package/src/vector/VectorManager.ts

@@ -0,0 +1,479 @@
+/**
+ * VectorManager - High-level Vector Operations Orchestrator
+ *
+ * Provides a unified interface for vector operations combining
+ * embedding generation and HNSW indexing with smart caching.
+ * Uses hybrid HNSW implementation (browser: wasm, node: native).
+ */
+
+import { EmbeddingService, EmbeddingOptions, EmbeddingResult, BatchEmbeddingResult } from '../services/EmbeddingService';
+import { createHnswService, isBrowser, isNode } from './createHnswService';
+import type { IHnswService } from './IHnswService';
+import {
+  VectorEmbedding,
+  EmbeddingConfig,
+  HNSWIndexConfig,
+  BatchConfig,
+  VectorSearchOptions,
+  VectorSearchResult,
+  VectorSearchMatch,
+  HNSWSearchResult
+} from '../embedding/types';
+
+export interface VectorManagerConfig {
+  embedding: EmbeddingConfig;
+  index?: Partial<HNSWIndexConfig>;
+  batch?: Partial<BatchConfig>;
+  enableAutoIndex?: boolean;
+  enableMemoryCache?: boolean;
+}
+
+export interface VectorOperationResult {
+  success: boolean;
+  vectorId?: number;
+  embedding?: VectorEmbedding;
+  indexBlobId?: string;
+  processingTime: number;
+  error?: string;
+}
+
+export interface VectorSearchStats {
+  searchTime: number;
+  embeddingTime: number;
+  indexSearchTime: number;
+  totalResults: number;
+  cacheHits: number;
+  indexSize: number;
+}
+
+/**
+ * Unified vector operations manager
+ */
+export class VectorManager {
+  private embeddingService: EmbeddingService;
+  private indexService: IHnswService | null = null;
+  private indexServicePromise: Promise<IHnswService> | null = null;
+  private vectorIdCounter = new Map<string, number>(); // userAddress -> nextVectorId
+  private memoryCache = new Map<string, VectorEmbedding>(); // text hash -> embedding
+  private config: VectorManagerConfig;
+
+  constructor(config: VectorManagerConfig) {
+    this.config = config;
+
+    // Initialize embedding service
+    this.embeddingService = new EmbeddingService(config.embedding);
+
+    // Initialize HNSW index service asynchronously using factory
+    this.indexServicePromise = this.initializeIndexService();
+
+    const envType = isBrowser() ? 'browser (hnswlib-wasm)' : isNode() ? 'Node.js (hnswlib-node)' : 'unknown';
+    console.log(`✅ VectorManager initializing with hybrid HNSW (${envType})`);
+  }
+
+  /**
+   * Initialize HNSW service using factory
+   */
+  private async initializeIndexService(): Promise<IHnswService> {
+    try {
+      const service = await createHnswService({
+        indexConfig: {
+          dimension: this.config.index?.dimension || 3072,
+          maxElements: this.config.index?.maxElements || 10000,
+          efConstruction: this.config.index?.efConstruction || 200,
+          m: this.config.index?.m || 16
+        },
+        batchConfig: this.config.batch
+      });
+
+      this.indexService = service;
+      console.log('✅ VectorManager HNSW service initialized successfully');
+      return service;
+    } catch (error) {
+      console.error('❌ Failed to initialize HNSW service:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Get HNSW service (waits for initialization if needed)
+   */
+  private async getIndexService(): Promise<IHnswService> {
+    if (this.indexService) {
+      return this.indexService;
+    }
+    if (this.indexServicePromise) {
+      return this.indexServicePromise;
+    }
+    throw new Error('HNSW service not initialized');
+  }
+
+  /**
+   * Add text content to vector index (embed + index)
+   */
+  async addTextToIndex(
+    userAddress: string,
+    text: string,
+    options: {
+      vectorId?: number;
+      metadata?: any;
+      embeddingType?: 'content' | 'metadata' | 'query';
+      enableCache?: boolean;
+    } = {}
+  ): Promise<VectorOperationResult> {
+    const startTime = Date.now();
+
+    try {
+      // Generate or get cached embedding
+      let embedding: VectorEmbedding;
+      const textHash = this.hashText(text);
+      
+      if (options.enableCache !== false && this.config.enableMemoryCache && this.memoryCache.has(textHash)) {
+        embedding = this.memoryCache.get(textHash)!;
+        console.debug(`Using cached embedding for text hash: ${textHash.substring(0, 8)}`);
+      } else {
+        // Generate new embedding
+        const embeddingResult = await this.embeddingService.embedText({
+          text,
+          type: options.embeddingType || 'content'
+        });
+
+        embedding = {
+          vector: embeddingResult.vector,
+          dimension: embeddingResult.dimension,
+          model: embeddingResult.model,
+          metadata: {
+            contentType: 'text/plain',
+            timestamp: Date.now(),
+            source: 'vector-manager',
+            ...options.metadata
+          }
+        };
+
+        // Cache if enabled
+        if (this.config.enableMemoryCache) {
+          this.memoryCache.set(textHash, embedding);
+        }
+      }
+
+      // Get vector ID
+      const vectorId = options.vectorId || this.getNextVectorId(userAddress);
+
+      // Add to HNSW index
+      if (this.config.enableAutoIndex !== false) {
+        const indexService = await this.getIndexService();
+        await indexService.addVector(
+          userAddress,
+          vectorId,
+          embedding.vector,
+          { text, ...options.metadata }
+        );
+      }
+
+      return {
+        success: true,
+        vectorId,
+        embedding,
+        processingTime: Date.now() - startTime
+      };
+    } catch (error) {
+      return {
+        success: false,
+        processingTime: Date.now() - startTime,
+        error: error instanceof Error ? error.message : 'Unknown error'
+      };
+    }
+  }
+
+  /**
+   * Search for similar texts using vector similarity
+   */
+  async searchSimilarTexts(
+    userAddress: string,
+    queryText: string,
+    options: {
+      k?: number;
+      threshold?: number;
+      efSearch?: number;
+      category?: string;
+      metadata?: any;
+      includeEmbeddings?: boolean;
+    } = {}
+  ): Promise<{
+    results: VectorSearchMatch[];
+    stats: VectorSearchStats;
+  }> {
+    const totalStartTime = Date.now();
+    
+    try {
+      // Generate query embedding
+      const embeddingStartTime = Date.now();
+      const queryEmbedding = await this.embeddingService.embedText({
+        text: queryText,
+        type: 'query'
+      });
+      const embeddingTime = Date.now() - embeddingStartTime;
+
+      // Search in HNSW index
+      const indexStartTime = Date.now();
+      const indexService = await this.getIndexService();
+      const searchResults = await indexService.search(
+        userAddress,
+        queryEmbedding.vector,
+        {
+          k: options.k || 10,
+          ef: options.efSearch || 50,
+          filter: options.metadata ? (meta) => this.matchesMetadata(meta, options.metadata!) : undefined
+        }
+      );
+      const indexSearchTime = Date.now() - indexStartTime;
+
+      // Convert HNSW results to VectorSearchMatch format
+      const results: VectorSearchMatch[] = [];
+      for (const result of searchResults) {
+        const vectorId = result.vectorId;
+        const distance = result.distance;
+        const similarity = result.score;
+
+        // Apply threshold filter
+        if (options.threshold && similarity < options.threshold) {
+          continue;
+        }
+
+        const match: VectorSearchMatch = {
+          memoryId: `memory_${vectorId}`, // TODO: Map to actual memory ID
+          vectorId,
+          similarity,
+          distance,
+          metadata: {} // TODO: Get metadata from index
+        };
+
+        if (options.includeEmbeddings) {
+          match.embedding = {
+            vector: [], // TODO: Store and retrieve vectors
+            dimension: queryEmbedding.dimension,
+            model: queryEmbedding.model
+          };
+        }
+
+        results.push(match);
+      }
+
+      const stats: VectorSearchStats = {
+        searchTime: Date.now() - totalStartTime,
+        embeddingTime,
+        indexSearchTime,
+        totalResults: results.length,
+        cacheHits: 0, // TODO: Track cache hits
+        indexSize: 0 // TODO: Get from index service
+      };
+
+      return { results, stats };
+    } catch (error) {
+      throw new Error(`Vector search failed: ${error}`);
+    }
+  }
+
+  /**
+   * Batch add multiple texts to index
+   */
+  async addTextsBatch(
+    userAddress: string,
+    texts: Array<{
+      text: string;
+      metadata?: any;
+      embeddingType?: 'content' | 'metadata' | 'query';
+    }>,
+    options: {
+      batchSize?: number;
+      enableCache?: boolean;
+    } = {}
+  ): Promise<{
+    results: VectorOperationResult[];
+    stats: {
+      totalTime: number;
+      embeddingTime: number;
+      indexingTime: number;
+      successCount: number;
+      failureCount: number;
+    };
+  }> {
+    const startTime = Date.now();
+    const batchSize = options.batchSize || 10;
+    const results: VectorOperationResult[] = [];
+
+    // Extract texts for batch embedding
+    const textContents = texts.map(t => t.text);
+
+    // Generate embeddings in batch
+    const embeddingStartTime = Date.now();
+    const batchEmbeddings = await this.embeddingService.embedBatch(textContents);
+    const embeddingTime = Date.now() - embeddingStartTime;
+
+    // Add to index
+    const indexingStartTime = Date.now();
+    for (let i = 0; i < texts.length; i++) {
+      try {
+        const text = texts[i];
+        const vector = batchEmbeddings.vectors[i];
+        const vectorId = this.getNextVectorId(userAddress);
+
+        const embedding: VectorEmbedding = {
+          vector,
+          dimension: batchEmbeddings.dimension,
+          model: batchEmbeddings.model,
+          metadata: {
+            contentType: 'text/plain',
+            timestamp: Date.now(),
+            ...text.metadata
+          }
+        };
+
+        // Add to index
+        if (this.config.enableAutoIndex !== false) {
+          const indexService = await this.getIndexService();
+          await indexService.addVector(
+            userAddress,
+            vectorId,
+            vector,
+            text.metadata
+          );
+        }
+
+        // Cache embedding
+        if (this.config.enableMemoryCache && options.enableCache !== false) {
+          const textHash = this.hashText(text.text);
+          this.memoryCache.set(textHash, embedding);
+        }
+
+        results.push({
+          success: true,
+          vectorId,
+          embedding,
+          processingTime: 0 // Individual timing not tracked in batch
+        });
+      } catch (error) {
+        results.push({
+          success: false,
+          processingTime: 0,
+          error: error instanceof Error ? error.message : 'Unknown error'
+        });
+      }
+    }
+
+    const indexingTime = Date.now() - indexingStartTime;
+    const totalTime = Date.now() - startTime;
+
+    return {
+      results,
+      stats: {
+        totalTime,
+        embeddingTime,
+        indexingTime,
+        successCount: results.filter(r => r.success).length,
+        failureCount: results.filter(r => !r.success).length
+      }
+    };
+  }
+
+  /**
+   * Load user's vector index from storage
+   */
+  async loadUserIndex(userAddress: string, indexBlobId?: string): Promise<void> {
+    const indexService = await this.getIndexService();
+    await indexService.loadIndex(userAddress);
+  }
+
+  /**
+   * Save user's vector index to storage
+   */
+  async saveUserIndex(userAddress: string): Promise<void> {
+    const indexService = await this.getIndexService();
+    await indexService.saveIndex(userAddress);
+  }
+
+  /**
+   * Force flush pending vectors for a user
+   */
+  async forceFlushUser(userAddress: string): Promise<void> {
+    const indexService = await this.getIndexService();
+    await indexService.flushBatch(userAddress);
+  }
+
+  /**
+   * Get vector processing statistics
+   */
+  async getStats() {
+    const embeddingStats = this.embeddingService.getStats();
+
+    let indexStats = null;
+    if (this.indexService) {
+      indexStats = this.indexService.getBatchStats();
+    }
+
+    return {
+      index: indexStats,
+      embedding: embeddingStats,
+      cache: {
+        memoryCache: {
+          size: this.memoryCache.size,
+          enabled: this.config.enableMemoryCache
+        },
+        vectorIdCounters: Object.fromEntries(this.vectorIdCounter)
+      }
+    };
+  }
+
+  /**
+   * Clear cache and reset state for a user
+   */
+  async clearUserData(userAddress: string): Promise<void> {
+    const indexService = await this.getIndexService();
+    await indexService.deleteIndex(userAddress);
+    this.vectorIdCounter.delete(userAddress);
+  }
+
+  /**
+   * Cleanup resources
+   */
+  destroy(): void {
+    if (this.indexService) {
+      this.indexService.destroy();
+    }
+    this.indexService = null;
+    this.indexServicePromise = null;
+    this.memoryCache.clear();
+    this.vectorIdCounter.clear();
+  }
+
+  // ==================== PRIVATE METHODS ====================
+
+  private getNextVectorId(userAddress: string): number {
+    const current = this.vectorIdCounter.get(userAddress) || 0;
+    const next = current + 1;
+    this.vectorIdCounter.set(userAddress, next);
+    return next;
+  }
+
+  private hashText(text: string): string {
+    // Simple hash function for text caching
+    let hash = 0;
+    for (let i = 0; i < text.length; i++) {
+      const char = text.charCodeAt(i);
+      hash = ((hash << 5) - hash) + char;
+      hash = hash & hash; // Convert to 32-bit integer
+    }
+    return Math.abs(hash).toString(36);
+  }
+
+  private matchesMetadata(metadata: any, filter: any): boolean {
+    if (!metadata || !filter) return true;
+
+    for (const [key, value] of Object.entries(filter)) {
+      if (metadata[key] !== value) return false;
+    }
+
+    return true;
+  }
+}
+
+export default VectorManager;

package/src/vector/createHnswService.ts

@@ -0,0 +1,135 @@
+/**
+ * createHnswService - Factory for Environment-Aware HNSW Service
+ *
+ * Automatically detects the runtime environment and returns the appropriate
+ * HNSW implementation:
+ * - Browser: BrowserHnswIndexService (hnswlib-wasm)
+ * - Node.js: NodeHnswService (hnswlib-node)
+ *
+ * Uses singleton pattern to prevent redundant initializations.
+ */
+
+import type { IHnswService, HnswServiceConfig } from './IHnswService';
+import { isBrowser, isNode } from './IHnswService';
+
+// Re-export environment detection functions
+export { isBrowser, isNode };
+
+// Singleton instance and initialization promise
+let singletonInstance: IHnswService | null = null;
+let singletonInitPromise: Promise<IHnswService> | null = null;
+let instanceCount = 0;
+
+/**
+ * Create an HNSW service appropriate for the current environment.
+ * Uses singleton pattern - subsequent calls return the same instance.
+ *
+ * @param config - Service configuration (only used on first call)
+ * @returns Promise<IHnswService> - The appropriate HNSW service for the environment
+ *
+ * @example
+ * ```typescript
+ * const hnswService = await createHnswService({
+ *   indexConfig: { dimension: 768 }
+ * });
+ *
+ * await hnswService.addVector(userAddress, vectorId, embedding);
+ * const results = await hnswService.search(userAddress, queryEmbedding);
+ * ```
+ */
+export async function createHnswService(config: HnswServiceConfig = {}): Promise<IHnswService> {
+  // Return existing singleton if available
+  if (singletonInstance) {
+    instanceCount++;
+    console.log(`[createHnswService] Returning singleton instance (request #${instanceCount})`);
+    return singletonInstance;
+  }
+
+  // Return pending initialization promise if already in progress
+  if (singletonInitPromise) {
+    instanceCount++;
+    console.log(`[createHnswService] Waiting for pending initialization (request #${instanceCount})`);
+    return singletonInitPromise;
+  }
+
+  // Create new singleton instance
+  instanceCount = 1;
+  singletonInitPromise = createNewInstance(config);
+
+  try {
+    singletonInstance = await singletonInitPromise;
+    return singletonInstance;
+  } catch (error) {
+    // Reset on failure to allow retry
+    singletonInitPromise = null;
+    throw error;
+  }
+}
+
+/**
+ * Internal function to create a new HNSW service instance
+ */
+async function createNewInstance(config: HnswServiceConfig): Promise<IHnswService> {
+  if (isBrowser()) {
+    console.log('[createHnswService] Browser environment detected, using hnswlib-wasm');
+
+    // Dynamic import for browser service
+    const { BrowserHnswIndexService } = await import('./BrowserHnswIndexService');
+    const service = new BrowserHnswIndexService(config.indexConfig);
+    return service as unknown as IHnswService;
+  }
+
+  if (isNode()) {
+    console.log('[createHnswService] Node.js environment detected, using hnswlib-node');
+
+    // Dynamic import for Node.js service
+    const { NodeHnswService } = await import('./NodeHnswService');
+    const service = new NodeHnswService(config);
+    await service.initialize();
+    return service;
+  }
+
+  throw new Error(
+    'Unsupported environment: HNSW service requires either a browser (for hnswlib-wasm) ' +
+    'or Node.js (for hnswlib-node) environment.'
+  );
+}
+
+/**
+ * Reset the singleton instance (useful for testing)
+ */
+export function resetHnswServiceSingleton(): void {
+  if (singletonInstance) {
+    singletonInstance.destroy();
+  }
+  singletonInstance = null;
+  singletonInitPromise = null;
+  instanceCount = 0;
+  console.log('[createHnswService] Singleton instance reset');
+}
+
+/**
+ * Get singleton statistics
+ */
+export function getHnswServiceStats(): { instanceCount: number; isInitialized: boolean } {
+  return {
+    instanceCount,
+    isInitialized: singletonInstance !== null
+  };
+}
+
+/**
+ * Check if HNSW service is available in the current environment
+ */
+export function isHnswAvailable(): boolean {
+  return isBrowser() || isNode();
+}
+
+/**
+ * Get the type of HNSW service that would be used
+ */
+export function getHnswServiceType(): 'browser' | 'node' | 'none' {
+  if (isBrowser()) return 'browser';
+  if (isNode()) return 'node';
+  return 'none';
+}

package/src/vector/index.ts

@@ -0,0 +1,57 @@
+/**
+ * Vector Module - HNSW Vector Indexing and Management
+ *
+ * Exports all vector-related services and utilities for the PDW SDK.
+ * Supports both browser (hnswlib-wasm) and Node.js (hnswlib-node) environments.
+ *
+ * @example
+ * ```typescript
+ * // Auto-detect environment and create appropriate service
+ * import { createHnswService } from 'personal-data-wallet-sdk/vector';
+ * const service = await createHnswService({ indexConfig: { dimension: 768 } });
+ * ```
+ */
+
+// Interface and utilities (safe for all environments)
+export type {
+  IHnswService,
+  HnswServiceConfig,
+  IHnswIndexConfig,
+  IHnswSearchOptions,
+  IHnswSearchResultItem,
+  IHnswBatchStats,
+  WalrusBackupConfig
+} from './IHnswService';
+export { isBrowser, isNode } from './IHnswService';
+
+// Factory function (auto-detects environment, singleton pattern)
+export {
+  createHnswService,
+  isHnswAvailable,
+  getHnswServiceType,
+  resetHnswServiceSingleton,
+  getHnswServiceStats
+} from './createHnswService';
+
+// VectorManager (uses factory internally)
+export { VectorManager } from './VectorManager';
+
+// Re-export types from embedding module for convenience
+export type {
+  HNSWIndexConfig,
+  HNSWSearchResult,
+  HNSWSearchOptions,
+  BatchConfig,
+  BatchJob,
+  BatchStats,
+  VectorSearchOptions,
+  VectorSearchResult,
+  VectorSearchMatch,
+  VectorError
+} from '../embedding/types';
+
+// Default export with lazy loaders
+export default {
+  createHnswService: async () => (await import('./createHnswService')).createHnswService,
+  VectorManager: async () => (await import('./VectorManager')).VectorManager
+};