@cmdoss/memwal-sdk 0.6.0

package/src/services/storage/WalrusMetadataManager.ts

@@ -0,0 +1,271 @@
+/**
+ * WalrusMetadataManager - Walrus Blob Metadata Operations
+ *
+ * Handles metadata attachment and retrieval for Walrus Blob objects.
+ * Extracted from StorageService for better separation of concerns.
+ *
+ * Features:
+ * - Build Walrus-compatible metadata structures
+ * - Attach metadata to Blob objects via dynamic fields
+ * - Retrieve metadata from Blob objects
+ * - Content integrity via blob_id
+ */
+
+import type { SuiClient } from '@mysten/sui/client';
+import type { Signer } from '@mysten/sui/cryptography';
+import { Transaction } from '@mysten/sui/transactions';
+
+export interface WalrusMemoryMetadata {
+  // Content identification
+  content_type: string;
+  content_size: string;
+
+  // Memory classification
+  category: string;
+  topic: string;
+  importance: string;
+
+  // Vector embedding
+  embedding_dimensions: string;
+  embedding_model: string;
+  embedding_blob_id?: string;
+
+  // Knowledge graph
+  graph_entity_count: string;
+  graph_relationship_count: string;
+  graph_blob_id?: string;
+  graph_entity_ids?: string;
+
+  // Vector index
+  vector_id: string;
+  vector_status: string;
+
+  // Lifecycle
+  created_at: string;
+  updated_at: string;
+  deleted_at?: string;
+
+  // Encryption
+  encrypted: string;
+  encryption_type?: string;
+  seal_identity?: string;
+
+  // Extensible custom fields
+  [key: string]: string | undefined;
+}
+
+export interface MetadataBuildOptions {
+  category?: string;
+  topic?: string;
+  importance?: number;
+  embedding?: number[];
+  embeddingBlobId?: string;
+  graphBlobId?: string;
+  graphEntityIds?: string[];
+  graphEntityCount?: number;
+  graphRelationshipCount?: number;
+  vectorId?: number;
+  isEncrypted?: boolean;
+  encryptionType?: string;
+  sealIdentity?: string;
+  customFields?: Record<string, string>;
+}
+
+/**
+ * WalrusMetadataManager - Manages Walrus Blob metadata
+ *
+ * Handles:
+ * - Metadata structure building
+ * - Dynamic field attachment to Blob objects
+ * - Metadata retrieval and parsing
+ */
+export class WalrusMetadataManager {
+  constructor(private suiClient: SuiClient) {}
+
+  /**
+   * Build Walrus metadata structure
+   *
+   * NOTE: No content_hash field needed! Walrus blob_id already serves as content hash.
+   * blob_id = blake2b256(bcs(root_hash, encoding_type, size))
+   */
+  buildWalrusMetadata(
+    contentSize: number,
+    options: MetadataBuildOptions
+  ): WalrusMemoryMetadata {
+    // Determine content type
+    const contentType = options.isEncrypted
+      ? 'application/octet-stream'
+      : options.customFields?.['content-type'] || 'text/plain';
+
+    // Build base metadata (all values as strings for VecMap<String, String>)
+    const metadata: WalrusMemoryMetadata = {
+      // Content identification
+      content_type: contentType,
+      content_size: contentSize.toString(),
+
+      // Memory classification
+      category: options.category || 'general',
+      topic: options.topic || '',
+      importance: (options.importance || 5).toString(),
+
+      // Vector embedding
+      embedding_dimensions: (options.embedding?.length || 3072).toString(),
+      embedding_model: 'text-embedding-004',
+      embedding_blob_id: options.embeddingBlobId || '',
+
+      // Knowledge graph
+      graph_entity_count: (options.graphEntityCount || 0).toString(),
+      graph_relationship_count: (options.graphRelationshipCount || 0).toString(),
+      graph_blob_id: options.graphBlobId || '',
+      graph_entity_ids: options.graphEntityIds ? JSON.stringify(options.graphEntityIds) : '',
+
+      // Vector index
+      vector_id: options.vectorId?.toString() || '',
+      vector_status: options.vectorId ? '1' : '2', // 1=active, 2=pending
+
+      // Lifecycle
+      created_at: Date.now().toString(),
+      updated_at: Date.now().toString(),
+
+      // Encryption
+      encrypted: (options.isEncrypted || false).toString(),
+      encryption_type: options.isEncrypted ? (options.encryptionType || 'seal') : undefined,
+      seal_identity: options.sealIdentity || undefined,
+    };
+
+    // Add custom fields
+    if (options.customFields) {
+      Object.entries(options.customFields).forEach(([key, value]) => {
+        if (value !== undefined && !metadata[key]) {
+          metadata[key] = value;
+        }
+      });
+    }
+
+    return metadata;
+  }
+
+  /**
+   * Attach metadata to a Walrus Blob object
+   *
+   * NOTE: Based on research, Walrus metadata requires separate queries to retrieve,
+   * making it less efficient than on-chain Memory structs for filtering.
+   */
+  async attachMetadataToBlob(
+    blobId: string,
+    metadata: WalrusMemoryMetadata,
+    signer: Signer,
+    walrusPackageId: string
+  ): Promise<{ digest: string; effects: any }> {
+    try {
+      const tx = new Transaction();
+
+      // Convert WalrusMemoryMetadata to VecMap<String, String> format
+      const metadataEntries: Array<[string, string]> = [];
+      Object.entries(metadata).forEach(([key, value]) => {
+        if (value !== undefined && value !== '') {
+          metadataEntries.push([key, value]);
+        }
+      });
+
+      console.log(`📋 Attaching ${metadataEntries.length} metadata fields to blob ${blobId}`);
+
+      // Call Walrus blob::add_or_replace_metadata()
+      tx.moveCall({
+        target: `${walrusPackageId}::blob::add_or_replace_metadata`,
+        arguments: [
+          tx.object(blobId),
+          // Metadata construction would go here
+        ],
+      });
+
+      tx.setSender(signer.toSuiAddress());
+
+      const result = await signer.signAndExecuteTransaction({
+        transaction: tx,
+        client: this.suiClient,
+      });
+
+      console.log(`✅ Metadata attached successfully. Digest: ${result.digest}`);
+
+      return {
+        digest: result.digest,
+        effects: result.effects,
+      };
+
+    } catch (error) {
+      console.error(`❌ Failed to attach metadata to blob ${blobId}:`, error);
+      throw new Error(`Metadata attachment failed: ${error}`);
+    }
+  }
+
+  /**
+   * Retrieve metadata from a Walrus Blob object
+   *
+   * NOTE: This queries dynamic fields, which is slower than querying on-chain structs.
+   */
+  async retrieveBlobMetadata(blobObjectId: string): Promise<WalrusMemoryMetadata | null> {
+    try {
+      // Query dynamic fields on the Blob object
+      const dynamicFields = await this.suiClient.getDynamicFields({
+        parentId: blobObjectId,
+      });
+
+      // Look for the metadata dynamic field
+      const metadataField = dynamicFields.data.find(
+        (field: any) => field.name.value === 'metadata'
+      );
+
+      if (!metadataField) {
+        console.log(`No metadata found for blob object ${blobObjectId}`);
+        return null;
+      }
+
+      // Retrieve the metadata object
+      const metadataObject = await this.suiClient.getObject({
+        id: metadataField.objectId,
+        options: { showContent: true },
+      });
+
+      if (!metadataObject.data?.content || !('fields' in metadataObject.data.content)) {
+        return null;
+      }
+
+      const fields = metadataObject.data.content.fields as any;
+
+      // Parse VecMap<String, String> into WalrusMemoryMetadata
+      const metadata: WalrusMemoryMetadata = {
+        content_type: '',
+        content_size: '',
+        category: '',
+        topic: '',
+        importance: '',
+        embedding_dimensions: '',
+        embedding_model: '',
+        graph_entity_count: '',
+        graph_relationship_count: '',
+        vector_id: '',
+        vector_status: '',
+        created_at: '',
+        updated_at: '',
+        encrypted: '',
+      };
+
+      // Parse the VecMap contents
+      if (fields.contents && Array.isArray(fields.contents)) {
+        fields.contents.forEach((entry: any) => {
+          if (entry.key && entry.value) {
+            (metadata as any)[entry.key] = entry.value;
+          }
+        });
+      }
+
+      console.log(`✅ Retrieved metadata for blob object ${blobObjectId}`);
+      return metadata;
+
+    } catch (error) {
+      console.error(`❌ Failed to retrieve metadata for blob ${blobObjectId}:`, error);
+      return null;
+    }
+  }
+}

package/src/services/storage/WalrusStorageManager.ts

@@ -0,0 +1,290 @@
+/**
+ * WalrusStorageManager - Core Walrus Storage Operations
+ *
+ * Handles blob upload/download using the Walrus distributed storage network.
+ * Extracted from StorageService for better separation of concerns.
+ *
+ * Features:
+ * - writeBlobFlow() for single blob uploads
+ * - Upload relay support (preferred on testnet)
+ * - Content integrity via blob_id
+ * - Direct blob retrieval from Walrus
+ *
+ * Performance: ~10-13 seconds per upload on testnet
+ */
+
+import { SuiClient, getFullnodeUrl } from '@mysten/sui/client';
+import { WalrusClient } from '@mysten/walrus';
+import type { Signer } from '@mysten/sui/cryptography';
+import type { ClientWithExtensions } from '@mysten/sui/experimental';
+
+export interface WalrusStorageConfig {
+  suiClient?: SuiClient;
+  network?: 'testnet' | 'mainnet';
+  maxFileSize?: number;
+  timeout?: number;
+  useUploadRelay?: boolean;
+  epochs?: number;
+}
+
+export interface BlobUploadOptions {
+  signer: Signer;
+  epochs?: number;
+  deletable?: boolean;
+  useUploadRelay?: boolean;
+  encrypt?: boolean;
+  metadata?: Record<string, string>;
+}
+
+export interface WalrusUploadResult {
+  blobId: string;
+  blobObjectId?: string;
+  isEncrypted: boolean;
+  storageEpochs: number;
+  uploadTimeMs: number;
+  contentSize: number;
+}
+
+export interface WalrusRetrievalResult {
+  content: Uint8Array;
+  source: 'walrus';
+  retrievalTime: number;
+  blobSize: number;
+}
+
+/**
+ * WalrusStorageManager - Manages core Walrus storage operations
+ *
+ * Handles all low-level Walrus interactions including:
+ * - Client creation and configuration
+ * - Blob upload via writeBlobFlow
+ * - Blob retrieval from distributed storage
+ */
+export class WalrusStorageManager {
+  private suiClient: ClientWithExtensions<{ jsonRpc: SuiClient; walrus: WalrusClient }>;
+  private walrusWithRelay: WalrusClient;
+  private walrusWithoutRelay: WalrusClient;
+  private config: WalrusStorageConfig;
+
+  constructor(config: WalrusStorageConfig) {
+    this.config = config;
+
+    // Setup clients synchronously
+    const clients = this.createClients();
+    this.suiClient = clients.suiClient;
+    this.walrusWithRelay = clients.walrusWithRelay;
+    this.walrusWithoutRelay = clients.walrusWithoutRelay;
+  }
+
+  /**
+   * Create Walrus clients with upload relay support
+   */
+  private createClients() {
+    const network = this.config.network || 'testnet';
+    const baseClient = this.config.suiClient || new SuiClient({
+      url: getFullnodeUrl(network),
+      network: network,
+    });
+
+    const uploadRelayHost = network === 'mainnet'
+      ? 'https://upload-relay.mainnet.walrus.space'
+      : 'https://upload-relay.testnet.walrus.space';
+
+    // Client with upload relay (preferred)
+    const clientWithRelay = baseClient.$extend(
+      WalrusClient.experimental_asClientExtension({
+        network: network,
+        uploadRelay: {
+          host: uploadRelayHost,
+          sendTip: { max: 1_000 },
+          timeout: 60_000,
+        },
+        storageNodeClientOptions: {
+          timeout: 60_000,
+        },
+      })
+    );
+
+    // Client without upload relay (fallback)
+    const clientWithoutRelay = baseClient.$extend(
+      WalrusClient.experimental_asClientExtension({
+        network: network,
+        storageNodeClientOptions: {
+          timeout: 60_000,
+        },
+      })
+    );
+
+    return {
+      suiClient: clientWithRelay,
+      walrusWithRelay: clientWithRelay.walrus,
+      walrusWithoutRelay: clientWithoutRelay.walrus,
+    };
+  }
+
+  /**
+   * Upload blob using writeBlobFlow pattern
+   *
+   * This is the core upload method for single blobs.
+   * Uses the official Walrus writeBlobFlow: encode → register → upload → certify
+   *
+   * @param data - Blob data as Uint8Array
+   * @param options - Upload options including signer and metadata
+   * @returns Upload result with blob ID and timing
+   */
+  async uploadBlob(
+    data: Uint8Array,
+    options: BlobUploadOptions
+  ): Promise<WalrusUploadResult> {
+    const startTime = performance.now();
+
+    try {
+      // Select client based on upload relay preference
+      const walrusClient = (options.useUploadRelay ?? this.config.useUploadRelay ?? true)
+        ? this.walrusWithRelay
+        : this.walrusWithoutRelay;
+
+      // Detect SEAL encryption from metadata
+      const isSealEncrypted = !!(
+        options.metadata?.['encryption-type']?.includes('seal') &&
+        options.metadata?.['encrypted'] === 'true'
+      );
+
+      if (isSealEncrypted) {
+        console.log(`🔐 Uploading SEAL encrypted binary data (${data.length} bytes)`);
+      }
+
+      // Create writeBlobFlow
+      const flow = walrusClient.writeBlobFlow({ blob: data });
+
+      // Step 1: Encode blob
+      await flow.encode();
+
+      // Step 2: Register blob on-chain
+      const registerTx = flow.register({
+        epochs: options.epochs || this.config.epochs || 3,
+        deletable: options.deletable ?? true,
+        owner: options.signer.toSuiAddress(),
+      });
+
+      registerTx.setSender(options.signer.toSuiAddress());
+      const { digest: registerDigest } = await options.signer.signAndExecuteTransaction({
+        transaction: registerTx,
+        client: this.suiClient,
+      });
+
+      // Step 3: Upload to storage
+      await flow.upload({ digest: registerDigest });
+
+      // Step 4: Certify blob on-chain
+      const certifyTx = flow.certify();
+      certifyTx.setSender(options.signer.toSuiAddress());
+      await options.signer.signAndExecuteTransaction({
+        transaction: certifyTx,
+        client: this.suiClient,
+      });
+
+      // Get blob info
+      const blob = await flow.getBlob();
+      const uploadTimeMs = performance.now() - startTime;
+
+      console.log(`✅ Blob uploaded successfully`);
+      console.log(`   Blob ID: ${blob.blobId}`);
+      console.log(`   Size: ${data.length} bytes`);
+      console.log(`   Upload time: ${uploadTimeMs.toFixed(1)}ms`);
+
+      return {
+        blobId: blob.blobId,
+        blobObjectId: blob.blobObject?.id?.id,
+        isEncrypted: isSealEncrypted,
+        storageEpochs: options.epochs || this.config.epochs || 3,
+        uploadTimeMs,
+        contentSize: data.length,
+      };
+
+    } catch (error) {
+      throw new Error(`Blob upload failed: ${error}`);
+    }
+  }
+
+  /**
+   * Retrieve blob by ID from Walrus
+   *
+   * @param blobId - The Walrus blob ID
+   * @returns Blob content as Uint8Array
+   */
+  async getBlob(blobId: string): Promise<Uint8Array> {
+    try {
+      console.log(`📥 Retrieving blob ${blobId} from Walrus...`);
+      const content = await this.suiClient.walrus.readBlob({ blobId });
+      console.log(`✅ Retrieved ${content.length} bytes from Walrus`);
+      return content;
+    } catch (error) {
+      console.error(`❌ Failed to retrieve blob ${blobId}:`, error);
+      throw new Error(`Failed to retrieve blob ${blobId}: ${error}`);
+    }
+  }
+
+  /**
+   * Retrieve blob with detailed timing and metadata
+   *
+   * @param blobId - The Walrus blob ID
+   * @returns Retrieval result with timing information
+   */
+  async retrieveBlob(blobId: string): Promise<WalrusRetrievalResult> {
+    const startTime = Date.now();
+
+    try {
+      console.log(`🔄 WALRUS RETRIEVAL: ${blobId}`);
+
+      const content = await this.suiClient.walrus.readBlob({ blobId });
+      const retrievalTime = Date.now() - startTime;
+
+      console.log(`✅ WALRUS RETRIEVAL SUCCESS:`);
+      console.log(`   Blob ID: ${blobId}`);
+      console.log(`   Content size: ${content.length} bytes`);
+      console.log(`   Retrieval time: ${retrievalTime}ms`);
+
+      return {
+        content,
+        source: 'walrus' as const,
+        retrievalTime,
+        blobSize: content.length
+      };
+
+    } catch (error) {
+      const retrievalTime = Date.now() - startTime;
+      console.error(`❌ WALRUS RETRIEVAL FAILED:`);
+      console.error(`   Blob ID: ${blobId}`);
+      console.error(`   Time elapsed: ${retrievalTime}ms`);
+
+      throw new Error(`Walrus retrieval failed for ${blobId}: ${error}`);
+    }
+  }
+
+  /**
+   * Get Walrus client (with or without relay)
+   */
+  getWalrusClient(useRelay: boolean = true): WalrusClient {
+    return useRelay ? this.walrusWithRelay : this.walrusWithoutRelay;
+  }
+
+  /**
+   * Get Sui client with Walrus extension
+   */
+  getSuiClient(): ClientWithExtensions<{ jsonRpc: SuiClient; walrus: WalrusClient }> {
+    return this.suiClient;
+  }
+
+  /**
+   * Get storage statistics
+   */
+  getStats() {
+    return {
+      network: this.config.network || 'testnet',
+      useUploadRelay: this.config.useUploadRelay ?? true,
+      epochs: this.config.epochs || 3,
+      maxFileSize: this.config.maxFileSize || Number.MAX_SAFE_INTEGER,
+    };
+  }
+}

package/src/services/storage/index.ts

@@ -0,0 +1,52 @@
+/**
+ * Storage Managers - Modular Storage Operations
+ *
+ * This module provides focused managers extracted from StorageService:
+ * - WalrusStorageManager: Core blob upload/download
+ * - MemorySearchManager: HNSW indexing and semantic search
+ * - KnowledgeGraphManager: Entity extraction and graph operations
+ * - WalrusMetadataManager: Blob metadata attachment
+ * - QuiltBatchManager: Batch uploads via Quilts
+ * - BlobAttributesManager: Dynamic field operations
+ */
+
+export { WalrusStorageManager } from './WalrusStorageManager';
+export type {
+  WalrusStorageConfig,
+  BlobUploadOptions,
+  WalrusUploadResult,
+  WalrusRetrievalResult
+} from './WalrusStorageManager';
+
+export { MemorySearchManager } from './MemorySearchManager';
+export type {
+  MemoryMetadata,
+  MetadataSearchQuery,
+  MetadataSearchResult,
+  IndexingResult
+} from './MemorySearchManager';
+
+export { KnowledgeGraphManager } from './KnowledgeGraphManager';
+export type {
+  GraphCache,
+  GraphQueryOptions,
+  GraphInitConfig
+} from './KnowledgeGraphManager';
+
+export { WalrusMetadataManager } from './WalrusMetadataManager';
+export type {
+  WalrusMemoryMetadata,
+  MetadataBuildOptions
+} from './WalrusMetadataManager';
+
+export { QuiltBatchManager } from './QuiltBatchManager';
+export type {
+  BatchMemory,
+  QuiltUploadOptions,
+  QuiltUploadResult
+} from './QuiltBatchManager';
+
+export { BlobAttributesManager } from './BlobAttributesManager';
+export type {
+  BlobQueryResult
+} from './BlobAttributesManager';

package/src/types/index.ts

@@ -0,0 +1,13 @@
+/**
+ * Core TypeScript types for Personal Data Wallet SDK
+ *
+ * @deprecated This module is deprecated. Import from '@personal-data-wallet/sdk/core/types' instead.
+ * This file now re-exports from the core module for backward compatibility.
+ *
+ * Migration guide:
+ * - Old: import { PDWConfig } from '@personal-data-wallet/sdk/types'
+ * - New: import { PDWConfig } from '@personal-data-wallet/sdk/core/types'
+ */
+
+// Re-export all types from core for backward compatibility
+export * from '../core/types/index';

package/src/utils/index.ts

@@ -0,0 +1,68 @@
+/**
+ * Utilities Module - Barrel Export
+ * 
+ * Central location for utility functions and helpers.
+ * Currently empty - ready for future utility additions.
+ * 
+ * Recommended structure:
+ * - utils/crypto/ - Cryptographic helpers
+ * - utils/encoding/ - Encoding/decoding utilities
+ * - utils/validation/ - Input validation (see errors/validation.ts)
+ * - utils/formatting/ - Data formatting utilities
+ * - utils/network/ - Network/HTTP helpers
+ * 
+ * Note: Configuration utilities are in src/config/
+ * Note: Validation utilities are in src/errors/validation.ts
+ */
+
+// Re-export commonly used validation utilities for convenience
+export {
+  isString,
+  isNumber,
+  isBoolean,
+  isObject,
+  isArray,
+  isUint8Array,
+  isValidAddress,
+  isValidObjectId,
+  validateRequired,
+  validateString,
+  validateNumber,
+  validateBoolean,
+  validateObject,
+  validateArray,
+  validateSuiAddress,
+  validateObjectId,
+} from '../errors/validation';
+
+// Re-export configuration helpers for convenience
+export {
+  ConfigurationHelper,
+  type SDKConfig,
+  type EnvironmentConfig,
+} from '../config/ConfigurationHelper';
+
+// Rebuild index utility for re-indexing existing memories (Browser)
+export {
+  rebuildIndex,
+  hasExistingIndex,
+  clearIndex,
+  type RebuildIndexOptions,
+  type RebuildIndexResult,
+} from './rebuildIndex';
+
+// Rebuild index utility for Node.js environments
+export {
+  rebuildIndexNode,
+  hasExistingIndexNode,
+  clearIndexNode,
+  type RebuildIndexNodeOptions,
+  type RebuildIndexNodeResult,
+} from './rebuildIndexNode';
+
+// Future utility exports will go here
+// export * from './crypto';
+// export * from './encoding';
+// export * from './formatting';
+// export * from './network';
+