@soulcraft/brainy 3.50.1 → 4.0.0

package/dist/storage/adapters/azureBlobStorage.d.ts

@@ -0,0 +1,545 @@
+/**
+ * Azure Blob Storage Adapter (Native)
+ * Uses the native @azure/storage-blob library for optimal performance and authentication
+ *
+ * Supports multiple authentication methods:
+ * 1. DefaultAzureCredential (Managed Identity) - Automatic in Azure environments
+ * 2. Connection String
+ * 3. Storage Account Key
+ * 4. SAS Token
+ * 5. Azure AD (OAuth2) via DefaultAzureCredential
+ *
+ * v4.0.0: Fully compatible with metadata/vector separation architecture
+ */
+import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
+import { BaseStorage } from '../baseStorage.js';
+type HNSWNode = HNSWNoun;
+type Edge = HNSWVerb;
+/**
+ * Native Azure Blob Storage adapter for server environments
+ * Uses the @azure/storage-blob library with DefaultAzureCredential
+ *
+ * Authentication priority:
+ * 1. DefaultAzureCredential (Managed Identity) - if no credentials provided
+ * 2. Connection String - if connectionString provided
+ * 3. Storage Account Key - if accountName + accountKey provided
+ * 4. SAS Token - if accountName + sasToken provided
+ */
+export declare class AzureBlobStorage extends BaseStorage {
+    private blobServiceClient;
+    private containerClient;
+    private containerName;
+    private accountName?;
+    private accountKey?;
+    private connectionString?;
+    private sasToken?;
+    private nounPrefix;
+    private verbPrefix;
+    private metadataPrefix;
+    private verbMetadataPrefix;
+    private systemPrefix;
+    protected statisticsCache: StatisticsData | null;
+    private pendingOperations;
+    private consecutiveErrors;
+    private lastErrorReset;
+    private backpressure;
+    private nounWriteBuffer;
+    private verbWriteBuffer;
+    private requestCoalescer;
+    private highVolumeMode;
+    private lastVolumeCheck;
+    private volumeCheckInterval;
+    private forceHighVolumeMode;
+    private nounCacheManager;
+    private verbCacheManager;
+    private logger;
+    /**
+     * Initialize the storage adapter
+     * @param options Configuration options for Azure Blob Storage
+     */
+    constructor(options: {
+        containerName: string;
+        connectionString?: string;
+        accountName?: string;
+        accountKey?: string;
+        sasToken?: string;
+        cacheConfig?: {
+            hotCacheMaxSize?: number;
+            hotCacheEvictionThreshold?: number;
+            warmCacheTTL?: number;
+        };
+        readOnly?: boolean;
+    });
+    /**
+     * Initialize the storage adapter
+     */
+    init(): Promise<void>;
+    /**
+     * Get the Azure blob name for a noun using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/nouns/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getNounKey('ab123456-1234-5678-9abc-def012345678')
+     * // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json'
+     */
+    private getNounKey;
+    /**
+     * Get the Azure blob name for a verb using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/verbs/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getVerbKey('cd987654-4321-8765-cba9-fed543210987')
+     * // returns 'entities/verbs/vectors/cd/cd987654-4321-8765-cba9-fed543210987.json'
+     */
+    private getVerbKey;
+    /**
+     * Override base class method to detect Azure-specific throttling errors
+     */
+    protected isThrottlingError(error: any): boolean;
+    /**
+     * Override base class to enable smart batching for cloud storage
+     *
+     * Azure Blob Storage is cloud storage with network latency (~50ms per write).
+     * Smart batching reduces writes from 1000 ops → 100 batches.
+     *
+     * @returns true (Azure is cloud storage)
+     */
+    protected isCloudStorage(): boolean;
+    /**
+     * Apply backpressure before starting an operation
+     * @returns Request ID for tracking
+     */
+    private applyBackpressure;
+    /**
+     * Release backpressure after completing an operation
+     * @param success Whether the operation succeeded
+     * @param requestId Request ID from applyBackpressure()
+     */
+    private releaseBackpressure;
+    /**
+     * Check if high-volume mode should be enabled
+     */
+    private checkVolumeMode;
+    /**
+     * Flush noun buffer to Azure
+     */
+    private flushNounBuffer;
+    /**
+     * Flush verb buffer to Azure
+     */
+    private flushVerbBuffer;
+    /**
+     * Save a noun to storage (internal implementation)
+     */
+    protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
+    /**
+     * Save a node to storage
+     */
+    protected saveNode(node: HNSWNode): Promise<void>;
+    /**
+     * Save a node directly to Azure (bypass buffer)
+     */
+    private saveNodeDirect;
+    /**
+     * Get a noun from storage (internal implementation)
+     * v4.0.0: Returns ONLY vector data (no metadata field)
+     * Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
+     */
+    protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
+    /**
+     * Get a node from storage
+     */
+    protected getNode(id: string): Promise<HNSWNode | null>;
+    /**
+     * Delete a noun from storage (internal implementation)
+     */
+    protected deleteNoun_internal(id: string): Promise<void>;
+    /**
+     * Write an object to a specific path in Azure
+     * Primitive operation required by base class
+     * @protected
+     */
+    protected writeObjectToPath(path: string, data: any): Promise<void>;
+    /**
+     * Read an object from a specific path in Azure
+     * Primitive operation required by base class
+     * @protected
+     */
+    protected readObjectFromPath(path: string): Promise<any | null>;
+    /**
+     * Delete an object from a specific path in Azure
+     * Primitive operation required by base class
+     * @protected
+     */
+    protected deleteObjectFromPath(path: string): Promise<void>;
+    /**
+     * Batch delete multiple blobs from Azure Blob Storage
+     * Deletes up to 256 blobs per batch (Azure limit)
+     * Handles throttling, retries, and partial failures
+     *
+     * @param keys - Array of blob names (paths) to delete
+     * @param options - Configuration options for batch deletion
+     * @returns Statistics about successful and failed deletions
+     */
+    batchDelete(keys: string[], options?: {
+        maxRetries?: number;
+        retryDelayMs?: number;
+        continueOnError?: boolean;
+    }): Promise<{
+        totalRequested: number;
+        successfulDeletes: number;
+        failedDeletes: number;
+        errors: Array<{
+            key: string;
+            error: string;
+        }>;
+    }>;
+    /**
+     * List all objects under a specific prefix in Azure
+     * Primitive operation required by base class
+     * @protected
+     */
+    protected listObjectsUnderPath(prefix: string): Promise<string[]>;
+    /**
+     * Helper: Convert Azure stream to buffer
+     */
+    private streamToBuffer;
+    /**
+     * Save a verb to storage (internal implementation)
+     */
+    protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
+    /**
+     * Save an edge to storage
+     */
+    protected saveEdge(edge: Edge): Promise<void>;
+    /**
+     * Save an edge directly to Azure (bypass buffer)
+     */
+    private saveEdgeDirect;
+    /**
+     * Get a verb from storage (internal implementation)
+     * v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
+     * Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
+     */
+    protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
+    /**
+     * Get an edge from storage
+     */
+    protected getEdge(id: string): Promise<Edge | null>;
+    /**
+     * Delete a verb from storage (internal implementation)
+     */
+    protected deleteVerb_internal(id: string): Promise<void>;
+    /**
+     * Get nouns with pagination
+     * v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
+     * Iterates through all UUID-based shards (00-ff) for consistent pagination
+     */
+    getNounsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: HNSWNounWithMetadata[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get nouns by noun type (internal implementation)
+     */
+    protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Get verbs by source ID (internal implementation)
+     */
+    protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
+    /**
+     * Get verbs by target ID (internal implementation)
+     */
+    protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
+    /**
+     * Get verbs by type (internal implementation)
+     */
+    protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
+    /**
+     * Clear all data from storage
+     */
+    clear(): Promise<void>;
+    /**
+     * Get storage status
+     */
+    getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    /**
+     * Save statistics data to storage
+     */
+    protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage
+     */
+    protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Initialize counts from storage
+     */
+    protected initializeCounts(): Promise<void>;
+    /**
+     * Initialize counts from storage scan (expensive - only for first-time init)
+     */
+    private initializeCountsFromScan;
+    /**
+     * Persist counts to storage
+     */
+    protected persistCounts(): Promise<void>;
+    /**
+     * Get a noun's vector for HNSW rebuild
+     */
+    getNounVector(id: string): Promise<number[] | null>;
+    /**
+     * Save HNSW graph data for a noun
+     */
+    saveHNSWData(nounId: string, hnswData: {
+        level: number;
+        connections: Record<string, string[]>;
+    }): Promise<void>;
+    /**
+     * Get HNSW graph data for a noun
+     */
+    getHNSWData(nounId: string): Promise<{
+        level: number;
+        connections: Record<string, string[]>;
+    } | null>;
+    /**
+     * Save HNSW system data (entry point, max level)
+     */
+    saveHNSWSystem(systemData: {
+        entryPointId: string | null;
+        maxLevel: number;
+    }): Promise<void>;
+    /**
+     * Get HNSW system data (entry point, max level)
+     */
+    getHNSWSystem(): Promise<{
+        entryPointId: string | null;
+        maxLevel: number;
+    } | null>;
+    /**
+     * Set the access tier for a specific blob (v4.0.0 cost optimization)
+     * Azure Blob Storage tiers:
+     * - Hot: $0.0184/GB/month - Frequently accessed data
+     * - Cool: $0.01/GB/month - Infrequently accessed data (45% cheaper)
+     * - Archive: $0.00099/GB/month - Rarely accessed data (99% cheaper!)
+     *
+     * @param blobName - Name of the blob to change tier
+     * @param tier - Target access tier ('Hot', 'Cool', or 'Archive')
+     * @returns Promise that resolves when tier is set
+     *
+     * @example
+     * // Move old vectors to Archive tier (99% cost savings)
+     * await storage.setBlobTier('entities/nouns/vectors/ab/old-id.json', 'Archive')
+     */
+    setBlobTier(blobName: string, tier: 'Hot' | 'Cool' | 'Archive'): Promise<void>;
+    /**
+     * Get the current access tier for a blob
+     *
+     * @param blobName - Name of the blob
+     * @returns Promise that resolves to the current tier or null if not found
+     *
+     * @example
+     * const tier = await storage.getBlobTier('entities/nouns/vectors/ab/id.json')
+     * console.log(`Current tier: ${tier}`) // 'Hot', 'Cool', or 'Archive'
+     */
+    getBlobTier(blobName: string): Promise<string | null>;
+    /**
+     * Set access tier for multiple blobs in batch (v4.0.0 cost optimization)
+     * Efficiently move large numbers of blobs between tiers for cost optimization
+     *
+     * @param blobs - Array of blob names and their target tiers
+     * @param options - Configuration options
+     * @returns Promise with statistics about tier changes
+     *
+     * @example
+     * // Move old data to Archive tier for 99% cost savings
+     * const oldBlobs = await storage.listObjectsUnderPath('entities/nouns/vectors/')
+     * await storage.setBlobTierBatch(
+     *   oldBlobs.map(name => ({ blobName: name, tier: 'Archive' }))
+     * )
+     */
+    setBlobTierBatch(blobs: Array<{
+        blobName: string;
+        tier: 'Hot' | 'Cool' | 'Archive';
+    }>, options?: {
+        maxRetries?: number;
+        retryDelayMs?: number;
+        continueOnError?: boolean;
+    }): Promise<{
+        totalRequested: number;
+        successfulChanges: number;
+        failedChanges: number;
+        errors: Array<{
+            blobName: string;
+            error: string;
+        }>;
+    }>;
+    /**
+     * Check if a blob in Archive tier has been rehydrated and is ready to read
+     * Archive tier blobs must be rehydrated before they can be read
+     *
+     * @param blobName - Name of the blob to check
+     * @returns Promise that resolves to rehydration status
+     *
+     * @example
+     * const status = await storage.checkRehydrationStatus('entities/nouns/vectors/ab/id.json')
+     * if (status.isRehydrated) {
+     *   // Blob is ready to read
+     *   const data = await storage.readObjectFromPath('entities/nouns/vectors/ab/id.json')
+     * }
+     */
+    checkRehydrationStatus(blobName: string): Promise<{
+        isArchived: boolean;
+        isRehydrating: boolean;
+        isRehydrated: boolean;
+        rehydratePriority?: string;
+    }>;
+    /**
+     * Rehydrate an archived blob (move from Archive to Hot or Cool tier)
+     * Note: Rehydration can take several hours depending on priority
+     *
+     * @param blobName - Name of the blob to rehydrate
+     * @param targetTier - Target tier after rehydration ('Hot' or 'Cool')
+     * @param priority - Rehydration priority ('Standard' or 'High')
+     *                  Standard: Up to 15 hours, cheaper
+     *                  High: Up to 1 hour, more expensive
+     * @returns Promise that resolves when rehydration is initiated
+     *
+     * @example
+     * // Rehydrate with standard priority (cheaper, slower)
+     * await storage.rehydrateBlob('entities/nouns/vectors/ab/id.json', 'Cool', 'Standard')
+     *
+     * // Check status
+     * const status = await storage.checkRehydrationStatus('entities/nouns/vectors/ab/id.json')
+     * console.log(`Rehydrating: ${status.isRehydrating}`)
+     */
+    rehydrateBlob(blobName: string, targetTier: 'Hot' | 'Cool', priority?: 'Standard' | 'High'): Promise<void>;
+    /**
+     * Set lifecycle management policy for automatic tier transitions and deletions (v4.0.0)
+     * Automates cost optimization by moving old data to cheaper tiers or deleting it
+     *
+     * Azure Lifecycle Management rules run once per day and apply to the entire container.
+     * Rules are evaluated against blob properties like lastModifiedTime and lastAccessTime.
+     *
+     * @param options - Lifecycle policy configuration
+     * @returns Promise that resolves when policy is set
+     *
+     * @example
+     * // Auto-archive old vectors for 99% cost savings
+     * await storage.setLifecyclePolicy({
+     *   rules: [
+     *     {
+     *       name: 'archiveOldVectors',
+     *       enabled: true,
+     *       type: 'Lifecycle',
+     *       definition: {
+     *         filters: {
+     *           blobTypes: ['blockBlob'],
+     *           prefixMatch: ['entities/nouns/vectors/']
+     *         },
+     *         actions: {
+     *           baseBlob: {
+     *             tierToCool: { daysAfterModificationGreaterThan: 30 },
+     *             tierToArchive: { daysAfterModificationGreaterThan: 90 },
+     *             delete: { daysAfterModificationGreaterThan: 365 }
+     *           }
+     *         }
+     *       }
+     *     }
+     *   ]
+     * })
+     */
+    setLifecyclePolicy(options: {
+        rules: Array<{
+            name: string;
+            enabled: boolean;
+            type: 'Lifecycle';
+            definition: {
+                filters: {
+                    blobTypes: string[];
+                    prefixMatch?: string[];
+                };
+                actions: {
+                    baseBlob: {
+                        tierToCool?: {
+                            daysAfterModificationGreaterThan: number;
+                        };
+                        tierToArchive?: {
+                            daysAfterModificationGreaterThan: number;
+                        };
+                        delete?: {
+                            daysAfterModificationGreaterThan: number;
+                        };
+                    };
+                };
+            };
+        }>;
+    }): Promise<void>;
+    /**
+     * Get the current lifecycle management policy
+     *
+     * @returns Promise that resolves to the current policy or null if not set
+     *
+     * @example
+     * const policy = await storage.getLifecyclePolicy()
+     * if (policy) {
+     *   console.log(`Found ${policy.rules.length} lifecycle rules`)
+     * }
+     */
+    getLifecyclePolicy(): Promise<{
+        rules: Array<{
+            name: string;
+            enabled: boolean;
+            type: string;
+            definition: {
+                filters: {
+                    blobTypes: string[];
+                    prefixMatch?: string[];
+                };
+                actions: {
+                    baseBlob: {
+                        tierToCool?: {
+                            daysAfterModificationGreaterThan: number;
+                        };
+                        tierToArchive?: {
+                            daysAfterModificationGreaterThan: number;
+                        };
+                        delete?: {
+                            daysAfterModificationGreaterThan: number;
+                        };
+                    };
+                };
+            };
+        }>;
+    } | null>;
+    /**
+     * Remove the lifecycle management policy
+     * All automatic tier transitions and deletions will stop
+     *
+     * @returns Promise that resolves when policy is removed
+     *
+     * @example
+     * await storage.removeLifecyclePolicy()
+     * console.log('Lifecycle policy removed - auto-archival disabled')
+     */
+    removeLifecyclePolicy(): Promise<void>;
+}
+export {};