@cmdoss/memwal-sdk 0.8.0 → 0.9.0

package/src/client/namespaces/MemoryNamespace.ts

@@ -39,6 +39,8 @@ export interface CreateMemoryOptions {
   importance?: number; // 1-10
   topic?: string;
   metadata?: Record<string, any>;
+  /** Pre-generated embedding (if provided, skips embedding generation) */
+  embedding?: number[];
   onProgress?: (stage: string, percent: number) => void;
 }
 
@@ -75,6 +77,20 @@ export interface ListMemoryOptions {
   order?: 'asc' | 'desc';
 }
 
+/**
+ * Options for searching memories
+ */
+export interface SearchMemoryOptions {
+  /** Maximum number of results (default: 10) */
+  limit?: number;
+  /** Minimum similarity threshold 0-1 (default: 0.5) */
+  threshold?: number;
+  /** Filter by category */
+  category?: string;
+  /** Include full content in results (default: true) */
+  includeContent?: boolean;
+}
+
 /**
  * Memory context with related memories
  */
@@ -132,7 +148,7 @@ export class MemoryNamespace {
    * ```
    */
   async create(content: string, options: CreateMemoryOptions = {}): Promise<Memory> {
-    const { importance = 5, topic, metadata, onProgress } = options;
+    const { importance = 5, topic, metadata, onProgress, embedding: preGeneratedEmbedding } = options;
     let category: string = options.category || 'general';
 
     try {
@@ -160,17 +176,17 @@ export class MemoryNamespace {
         }
       }
 
-      // 2. Generate embedding
-      let embedding: number[] | undefined;
-      if (this.services.embedding) {
+      // 2. Use pre-generated embedding or generate new one
+      let embedding: number[] | undefined = preGeneratedEmbedding;
+      if (embedding && embedding.length > 0) {
+        console.log(`✅ Using pre-generated embedding: ${embedding.length}D`);
+      } else if (this.services.embedding) {
         onProgress?.('generating embedding', 20);
-        const embResult = await this.services.embedding.embedText({
-          text: content
-        });
+        const embResult = await this.services.embedding.embedText({ text: content });
         embedding = embResult.vector;
-        console.log(`✅ Embedding generated: ${embedding?.length || 0}D vector`);
+        console.log(`✅ Embedding generated: ${embedding?.length || 0}D`);
       } else {
-        console.warn('⚠️ EmbeddingService not available - no embedding will be stored!');
+        console.warn('⚠️ No embedding available');
       }
 
       // 3. Encrypt (if enabled) - Use capability-based encryption (v2.2)
@@ -703,6 +719,137 @@ export class MemoryNamespace {
     }
   }
 
+  /**
+   * Search memories using natural language query
+   *
+   * Unified search that:
+   * 1. Converts query text to embedding
+   * 2. Performs vector similarity search
+   * 3. Returns ranked results with content
+   *
+   * @param query - Natural language search query
+   * @param options - Search options (limit, threshold, category)
+   * @returns Array of matching memories sorted by relevance
+   *
+   * @example
+   * ```typescript
+   * // Simple search
+   * const results = await pdw.memory.search('meeting notes');
+   *
+   * // With options
+   * const results = await pdw.memory.search('TypeScript tips', {
+   *   limit: 5,
+   *   category: 'note',
+   *   threshold: 0.7
+   * });
+   * ```
+   */
+  async search(query: string, options: SearchMemoryOptions = {}): Promise<Memory[]> {
+    const {
+      limit = 10,
+      threshold = 0.5,
+      category,
+      includeContent = true
+    } = options;
+
+    try {
+      // Validate query
+      if (!query || query.trim().length === 0) {
+        return [];
+      }
+
+      // Step 1: Generate embedding for query
+      if (!this.services.embedding) {
+        throw new Error('Embedding service not available - cannot perform semantic search');
+      }
+
+      const embResult = await this.services.embedding.embedText({ text: query });
+      const queryEmbedding = embResult.vector;
+
+      // Step 2: Perform vector search
+      let searchResults: any[] = [];
+
+      // Try local index first (faster)
+      if (this.services.vector) {
+        try {
+          const spaceId = this.services.config.userAddress;
+          const searchResult = await this.services.vector.searchVectors(spaceId, queryEmbedding, { k: limit * 2 });
+          searchResults = searchResult.results.map((r: any) => ({
+            id: r.metadata?.memoryObjectId || r.memoryId || String(r.vectorId),
+            vectorId: r.vectorId,
+            content: r.metadata?.content || '',
+            category: r.metadata?.category,
+            importance: r.metadata?.importance,
+            blobId: r.metadata?.blobId || r.memoryId,
+            metadata: r.metadata,
+            similarity: r.similarity || 0,
+            encrypted: r.metadata?.isEncrypted || false,
+            createdAt: r.metadata?.timestamp || Date.now()
+          }));
+        } catch (indexError) {
+          console.warn('Local index search failed:', indexError);
+        }
+      }
+
+      // Fallback to memory index service if available
+      if (searchResults.length === 0 && this.services.memoryIndex) {
+        try {
+          const results = await this.services.memoryIndex.searchMemories({
+            userAddress: this.services.config.userAddress,
+            vector: queryEmbedding,
+            k: limit * 2
+          });
+          searchResults = results.map((r: any) => ({
+            id: r.memoryObjectId || r.id,
+            content: r.content || '',
+            category: r.category,
+            importance: r.importance || r.metadata?.importance,
+            blobId: r.blobId || r.id,
+            metadata: r.metadata,
+            similarity: r.score || r.similarity || 0,
+            encrypted: r.isEncrypted || false,
+            createdAt: r.timestamp || Date.now()
+          }));
+        } catch (memIndexError) {
+          console.warn('Memory index search failed:', memIndexError);
+        }
+      }
+
+      // Step 3: Filter by threshold and category
+      let filtered = searchResults.filter(r => r.similarity >= threshold);
+
+      if (category) {
+        filtered = filtered.filter(r => r.category === category);
+      }
+
+      // Step 4: Limit results
+      const limited = filtered.slice(0, limit);
+
+      // Step 5: Optionally fetch full content for encrypted memories
+      if (includeContent) {
+        const withContent = await Promise.all(
+          limited.map(async (r) => {
+            // If content is missing or encrypted, try to fetch from storage
+            if (!r.content && r.blobId) {
+              try {
+                const memory = await this.get(r.blobId);
+                return { ...r, content: memory.content };
+              } catch {
+                return r;
+              }
+            }
+            return r;
+          })
+        );
+        return withContent as Memory[];
+      }
+
+      return limited as Memory[];
+    } catch (error) {
+      throw new Error(`Failed to search memories: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
   /**
    * Create multiple memories in batch using Walrus Quilt
    *

package/src/client/namespaces/consolidated/AdvancedNamespace.ts

@@ -0,0 +1,264 @@
+/**
+ * Advanced Namespace - Power User Features
+ *
+ * Groups specialized features for advanced use cases:
+ * - Knowledge graph operations
+ * - Memory analytics and insights
+ * - Manual encryption/decryption
+ * - Access control and permissions
+ * - Transaction building
+ * - Processing pipelines
+ *
+ * Most users won't need these - use pdw.memory for common operations.
+ *
+ * @module client/namespaces/consolidated
+ */
+
+import type { ServiceContainer } from '../../SimplePDWClient';
+import type { GraphNamespace } from '../GraphNamespace';
+import type { AnalyticsNamespace } from '../AnalyticsNamespace';
+import type { EncryptionNamespace } from '../EncryptionNamespace';
+import type { PermissionsNamespace } from '../PermissionsNamespace';
+import type { TxNamespace } from '../TxNamespace';
+import type { PipelineNamespace } from '../PipelineNamespace';
+import type { CapabilityNamespace } from '../CapabilityNamespace';
+import type { ContextNamespace } from '../ContextNamespace';
+import type { BatchNamespace } from '../BatchNamespace';
+import type { CacheNamespace } from '../CacheNamespace';
+import type { SearchNamespace } from '../SearchNamespace';
+
+/**
+ * Advanced Namespace
+ *
+ * Access specialized features through sub-namespaces:
+ *
+ * @example
+ * ```typescript
+ * // Knowledge graph
+ * const entities = await pdw.advanced.graph.getEntities();
+ *
+ * // Analytics
+ * const insights = await pdw.advanced.analytics.generate();
+ *
+ * // Manual encryption
+ * const encrypted = await pdw.advanced.encryption.encrypt(data);
+ *
+ * // Permissions
+ * await pdw.advanced.permissions.grant(userId, memoryId);
+ *
+ * // Blockchain transactions
+ * const tx = pdw.advanced.blockchain.buildCreateMemory(params);
+ * ```
+ */
+export class AdvancedNamespace {
+  private _graph?: GraphNamespace;
+  private _analytics?: AnalyticsNamespace;
+  private _encryption?: EncryptionNamespace;
+  private _permissions?: PermissionsNamespace;
+  private _blockchain?: TxNamespace;
+  private _pipeline?: PipelineNamespace;
+  private _capability?: CapabilityNamespace;
+  private _context?: ContextNamespace;
+  private _batch?: BatchNamespace;
+  private _cache?: CacheNamespace;
+  private _search?: SearchNamespace;
+
+  constructor(
+    private services: ServiceContainer,
+    namespaces: {
+      graph?: GraphNamespace;
+      analytics?: AnalyticsNamespace;
+      encryption?: EncryptionNamespace;
+      permissions?: PermissionsNamespace;
+      blockchain?: TxNamespace;
+      pipeline?: PipelineNamespace;
+      capability?: CapabilityNamespace;
+      context?: ContextNamespace;
+      batch?: BatchNamespace;
+      cache?: CacheNamespace;
+      search?: SearchNamespace;
+    }
+  ) {
+    this._graph = namespaces.graph;
+    this._analytics = namespaces.analytics;
+    this._encryption = namespaces.encryption;
+    this._permissions = namespaces.permissions;
+    this._blockchain = namespaces.blockchain;
+    this._pipeline = namespaces.pipeline;
+    this._capability = namespaces.capability;
+    this._context = namespaces.context;
+    this._batch = namespaces.batch;
+    this._cache = namespaces.cache;
+    this._search = namespaces.search;
+  }
+
+  /**
+   * Knowledge Graph Operations
+   *
+   * Extract entities and relationships from memories,
+   * traverse the graph, and find connections.
+   *
+   * @example
+   * ```typescript
+   * const entities = await pdw.advanced.graph.getEntities();
+   * const related = await pdw.advanced.graph.traverse(entityId);
+   * ```
+   */
+  get graph(): GraphNamespace | undefined {
+    return this._graph;
+  }
+
+  /**
+   * Memory Analytics
+   *
+   * Generate insights, trends, and clustering from memories.
+   *
+   * @example
+   * ```typescript
+   * const insights = await pdw.advanced.analytics.generate();
+   * const trends = await pdw.advanced.analytics.trends();
+   * ```
+   */
+  get analytics(): AnalyticsNamespace | undefined {
+    return this._analytics;
+  }
+
+  /**
+   * Manual Encryption/Decryption
+   *
+   * Direct access to SEAL encryption for custom use cases.
+   * Note: pdw.memory.create() handles encryption automatically.
+   *
+   * @example
+   * ```typescript
+   * const encrypted = await pdw.advanced.encryption.encrypt(data, keyId);
+   * const decrypted = await pdw.advanced.encryption.decrypt(options);
+   * ```
+   */
+  get encryption(): EncryptionNamespace | undefined {
+    return this._encryption;
+  }
+
+  /**
+   * Access Control & Permissions
+   *
+   * Grant, revoke, and manage access to memories.
+   *
+   * @example
+   * ```typescript
+   * await pdw.advanced.permissions.grant(userId, memoryId);
+   * await pdw.advanced.permissions.revoke(userId, memoryId);
+   * const perms = await pdw.advanced.permissions.list(memoryId);
+   * ```
+   */
+  get permissions(): PermissionsNamespace | undefined {
+    return this._permissions;
+  }
+
+  /**
+   * Blockchain Transaction Building
+   *
+   * Build custom Sui transactions for advanced operations.
+   *
+   * @example
+   * ```typescript
+   * const tx = pdw.advanced.blockchain.buildCreateMemory(params);
+   * const result = await pdw.advanced.blockchain.execute(tx);
+   * ```
+   */
+  get blockchain(): TxNamespace | undefined {
+    return this._blockchain;
+  }
+
+  /**
+   * Processing Pipelines
+   *
+   * Create and manage memory processing pipelines.
+   *
+   * @example
+   * ```typescript
+   * const pipeline = await pdw.advanced.pipeline.create(config);
+   * await pdw.advanced.pipeline.execute(pipeline, data);
+   * ```
+   */
+  get pipeline(): PipelineNamespace | undefined {
+    return this._pipeline;
+  }
+
+  /**
+   * Capability Management
+   *
+   * Low-level MemoryCap object management for SEAL encryption.
+   *
+   * @example
+   * ```typescript
+   * const cap = await pdw.advanced.capability.getOrCreate(appId);
+   * const keyId = pdw.advanced.capability.computeKeyId(cap);
+   * ```
+   */
+  get capability(): CapabilityNamespace | undefined {
+    return this._capability;
+  }
+
+  /**
+   * App Context Management
+   *
+   * Manage application contexts for multi-app memory access.
+   *
+   * @example
+   * ```typescript
+   * const ctx = await pdw.advanced.context.getOrCreate(appId);
+   * await pdw.advanced.context.transfer(ctxId, newOwner);
+   * ```
+   */
+  get context(): ContextNamespace | undefined {
+    return this._context;
+  }
+
+  /**
+   * Batch Processing Utilities
+   *
+   * Advanced batch operations beyond createBatch.
+   *
+   * @example
+   * ```typescript
+   * const stats = await pdw.advanced.batch.stats();
+   * const progress = await pdw.advanced.batch.progress(batchId);
+   * ```
+   */
+  get batch(): BatchNamespace | undefined {
+    return this._batch;
+  }
+
+  /**
+   * Cache Management
+   *
+   * Direct access to in-memory LRU cache.
+   *
+   * @example
+   * ```typescript
+   * const cached = pdw.advanced.cache.get(key);
+   * pdw.advanced.cache.set(key, value);
+   * pdw.advanced.cache.clear();
+   * ```
+   */
+  get cache(): CacheNamespace | undefined {
+    return this._cache;
+  }
+
+  /**
+   * Advanced Search
+   *
+   * Low-level vector and semantic search operations.
+   * Note: pdw.memory.search() is simpler for most use cases.
+   *
+   * @example
+   * ```typescript
+   * const results = await pdw.advanced.search.vector(embedding, k);
+   * const results = await pdw.advanced.search.hybrid(query, filters);
+   * ```
+   */
+  get search(): SearchNamespace | undefined {
+    return this._search;
+  }
+}

package/src/client/namespaces/consolidated/index.ts

@@ -37,3 +37,5 @@ export type {
   MemoryPackage,
   UploadOptions
 } from './StorageNamespace';
+
+export { AdvancedNamespace } from './AdvancedNamespace';

package/src/config/defaults.ts

@@ -22,11 +22,15 @@ export function createDefaultConfig(): PDWConfig {
       cacheEnabled: true,
       encryptionEnabled: true,
     },
-    // Walrus Storage Configuration
+    // Walrus Storage Configuration (Testnet defaults)
+    // Publisher: Direct blob uploads (server-side, full control)
     walrusPublisherUrl: 'https://publisher.walrus-testnet.walrus.space',
+    // Upload Relay: Optimized for browser/mobile (fewer connections, faster)
+    walrusUploadRelayUrl: 'https://upload-relay.testnet.walrus.space',
+    // Aggregator: Blob retrieval
     walrusAggregatorUrl: 'https://aggregator.walrus-testnet.walrus.space',
-    walrusMaxFileSize: 1024 * 1024 * 1024, // 1GB
-    walrusTimeout: 30000, // 30 seconds
+    walrusMaxFileSize: 10 * 1024 * 1024, // 10MB (public service limit)
+    walrusTimeout: 60000, // 60 seconds for upload relay
   };
 }
 
@@ -44,8 +48,9 @@ export function createTestnetConfig(overrides: Partial<PDWConfig> = {}): PDWConf
         threshold: 2,
       },
     },
-    // Testnet Walrus endpoints (same as default for now)
+    // Testnet Walrus endpoints
     walrusPublisherUrl: 'https://publisher.walrus-testnet.walrus.space',
+    walrusUploadRelayUrl: 'https://upload-relay.testnet.walrus.space',
     walrusAggregatorUrl: 'https://aggregator.walrus-testnet.walrus.space',
     ...overrides,
   };

package/src/core/types/index.ts

@@ -37,7 +37,11 @@ export interface PDWConfig {
   /** Storage configuration */
   storageConfig?: StorageConfig;
   /** Walrus storage configuration */
+  /** Publisher URL for direct blob uploads (server-side) */
   walrusPublisherUrl?: string;
+  /** Upload Relay URL for browser/mobile uploads (fewer connections, faster) */
+  walrusUploadRelayUrl?: string;
+  /** Aggregator URL for blob retrieval */
   walrusAggregatorUrl?: string;
   walrusMaxFileSize?: number;
   walrusTimeout?: number;

package/src/infrastructure/walrus/WalrusStorageService.ts

@@ -17,8 +17,14 @@ export interface WalrusConfig {
   network?: 'testnet' | 'mainnet';
   adminAddress?: string;
   storageEpochs?: number;
+  /** Publisher URL for direct blob uploads (server-side) */
+  publisherHost?: string;
+  /** Upload Relay URL for browser/mobile uploads (fewer connections, faster) */
   uploadRelayHost?: string;
+  /** Aggregator URL for blob retrieval */
   aggregatorHost?: string;
+  /** Use upload relay instead of publisher (default: true for browser, false for server) */
+  useUploadRelay?: boolean;
   retryAttempts?: number;
   timeoutMs?: number;
   sealService?: SealService;
@@ -123,13 +129,21 @@ export class WalrusStorageService {
 
   constructor(config: Partial<WalrusConfig> = {}) {
     const network = config.network || 'testnet';
+    // Detect browser environment for default useUploadRelay
+    const isBrowser = typeof window !== 'undefined';
 
     this.config = {
       network,
       adminAddress: config.adminAddress || '',
       storageEpochs: config.storageEpochs || 12,
+      // Publisher: direct uploads (server-side)
+      publisherHost: config.publisherHost || 'https://publisher.walrus-testnet.walrus.space',
+      // Upload Relay: optimized for browser/mobile (fewer connections)
       uploadRelayHost: config.uploadRelayHost || 'https://upload-relay.testnet.walrus.space',
+      // Aggregator: blob retrieval
       aggregatorHost: config.aggregatorHost || 'https://aggregator.walrus-testnet.walrus.space',
+      // Use upload relay by default in browser, publisher on server
+      useUploadRelay: config.useUploadRelay ?? isBrowser,
       retryAttempts: config.retryAttempts || 3,
       timeoutMs: config.timeoutMs || 60000,
       sealService: config.sealService,
@@ -519,9 +533,20 @@ export class WalrusStorageService {
       }
     }
 
-    // Fallback to REST API if no signer (read-only mode or relay upload)
+    // Fallback to REST API based on useUploadRelay config
     try {
-      const response = await fetch(`${this.config.uploadRelayHost}/v1/store?epochs=${this.config.storageEpochs}`, {
+      let url: string;
+      if (this.config.useUploadRelay) {
+        // Upload Relay: optimized for browser/mobile (fewer network connections)
+        // POST to /v1/blobs endpoint
+        url = `${this.config.uploadRelayHost}/v1/blobs?epochs=${this.config.storageEpochs}`;
+      } else {
+        // Publisher: direct upload (server-side)
+        // PUT to /v1/blobs endpoint
+        url = `${this.config.publisherHost}/v1/blobs?epochs=${this.config.storageEpochs}`;
+      }
+
+      const response = await fetch(url, {
         method: 'PUT',
         headers: {
           'Content-Type': 'application/octet-stream'
@@ -568,9 +593,9 @@ export class WalrusStorageService {
       }
     }
 
-    // Fallback to REST API
+    // Fallback to REST API - Aggregator endpoint: GET /v1/blobs/<blob-id>
     try {
-      const response = await fetch(`${this.config.aggregatorHost}/v1/${blobId}`);
+      const response = await fetch(`${this.config.aggregatorHost}/v1/blobs/${blobId}`);
 
       if (!response.ok) {
         throw new Error(`Retrieval failed: ${response.status} ${response.statusText}`);

package/src/services/EmbeddingService.ts

@@ -175,7 +175,7 @@ export class EmbeddingService {
       // New behavior: Direct EmbeddingModel from ai-sdk
       this.embeddingModel = config.model;
       this.modelName = 'custom';
-      this.dimensions = config.dimensions || 3072;
+      this.dimensions = config.dimensions || 768; // Default 768 for speed (was 3072)
       this.provider = 'custom';
       console.log('✅ EmbeddingService initialized with custom ai-sdk model');
       return;
@@ -246,19 +246,26 @@ export class EmbeddingService {
 
   /**
    * Get default dimensions for provider
+   *
+   * Default is now 768 for faster performance:
+   * - 4x smaller vectors = faster indexing & search
+   * - ~4x less storage space
+   * - Minimal quality loss for most use cases
+   *
+   * Users can override via config.dimensions
    */
   private getDefaultDimensions(provider: string): number {
     switch (provider) {
       case 'google':
-        return 3072;
+        return 768; // text-embedding-004 supports output_dimensionality
       case 'openai':
-        return 1536; // text-embedding-3-small default
+        return 768; // text-embedding-3-small supports dimensions param
       case 'openrouter':
-        return 3072; // google/gemini-embedding-001 returns 3072 dimensions
+        return 768; // Most models support dimension truncation
       case 'cohere':
-        return 1024;
+        return 768; // embed-english-v3.0 supports dimensions
       default:
-        return 3072;
+        return 768; // Default to 768 for speed
     }
   }
 
@@ -362,16 +369,25 @@ export class EmbeddingService {
   /**
    * Generate embedding using OpenRouter SDK
    * Uses official @openrouter/sdk for embeddings
+   * Passes dimensions parameter to truncate output vectors
    */
   private async embedTextOpenRouter(text: string, startTime: number): Promise<EmbeddingResult> {
     if (!this.openRouterClient) {
       throw new Error('OpenRouter client not initialized');
     }
 
-    const result = await this.openRouterClient.embeddings.generate({
+    // Build request with optional dimensions parameter
+    const request: any = {
       model: this.modelName,
       input: text
-    });
+    };
+
+    // Add dimensions if configured (enables output truncation)
+    if (this.dimensions && this.dimensions < 3072) {
+      request.dimensions = this.dimensions;
+    }
+
+    const result = await this.openRouterClient.embeddings.generate(request);
 
     // Handle union type - result can be string or object
     if (typeof result === 'string') {
@@ -453,16 +469,25 @@ export class EmbeddingService {
 
   /**
    * Generate batch embeddings using OpenRouter SDK
+   * Passes dimensions parameter to truncate output vectors
    */
   private async embedBatchOpenRouter(texts: string[], startTime: number): Promise<BatchEmbeddingResult> {
     if (!this.openRouterClient) {
       throw new Error('OpenRouter client not initialized');
     }
 
-    const result = await this.openRouterClient.embeddings.generate({
+    // Build request with optional dimensions parameter
+    const request: any = {
       model: this.modelName,
       input: texts
-    });
+    };
+
+    // Add dimensions if configured (enables output truncation)
+    if (this.dimensions && this.dimensions < 3072) {
+      request.dimensions = this.dimensions;
+    }
+
+    const result = await this.openRouterClient.embeddings.generate(request);
 
     // Handle union type - result can be string or object
     if (typeof result === 'string') {