@soulcraft/brainy 3.25.2 → 3.27.0

package/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.27.0](https://github.com/soulcraftlabs/brainy/compare/v3.26.0...v3.27.0) (2025-10-08)
+
+- test: skip incomplete clusterByDomain tests pending implementation (19aa4af)
+- feat: add native Google Cloud Storage adapter with ADC support (e2aa8e3)
+
+
+## [3.26.0](https://github.com/soulcraftlabs/brainy/compare/v3.25.2...v3.26.0) (2025-10-08)
+
+
+### ⚠ BREAKING CHANGES
+
+* Requires data migration for existing S3/GCS/R2/OpFS deployments.
+See .strategy/UNIFIED-UUID-SHARDING.md for migration guidance.
+
+### 🐛 Bug Fixes
+
+* implement unified UUID-based sharding for metadata across all storage adapters ([2f33571](https://github.com/soulcraftlabs/brainy/commit/2f3357132d06c70cd74532d22cbfbf6abb92903a))
+
 ### [3.25.2](https://github.com/soulcraftlabs/brainy/compare/v3.25.1...v3.25.2) (2025-10-08)
 
 

package/dist/storage/adapters/fileSystemStorage.js

@@ -510,7 +510,11 @@ export class FileSystemStorage extends BaseStorage {
      */
     async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
-        const filePath = path.join(this.nounMetadataDir, `${id}.json`);
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const filePath = this.getShardedPath(this.nounMetadataDir, id);
+        // Ensure shard directory exists
+        const shardDir = path.dirname(filePath);
+        await fs.promises.mkdir(shardDir, { recursive: true });
         await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
     }
     /**
@@ -518,7 +522,8 @@ export class FileSystemStorage extends BaseStorage {
      */
     async getNounMetadata(id) {
         await this.ensureInitialized();
-        const filePath = path.join(this.nounMetadataDir, `${id}.json`);
+        // Use UUID-based sharding for metadata (consistent with noun vectors)
+        const filePath = this.getShardedPath(this.nounMetadataDir, id);
         try {
             const data = await fs.promises.readFile(filePath, 'utf-8');
             return JSON.parse(data);

package/dist/storage/adapters/gcsStorage.d.ts

@@ -0,0 +1,334 @@
+/**
+ * Google Cloud Storage Adapter (Native)
+ * Uses the native @google-cloud/storage library for optimal performance and authentication
+ *
+ * Supports multiple authentication methods:
+ * 1. Application Default Credentials (ADC) - Automatic in Cloud Run/GCE
+ * 2. Service Account Key File
+ * 3. Service Account Credentials Object
+ * 4. HMAC Keys (fallback for backward compatibility)
+ */
+import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
+import { BaseStorage } from '../baseStorage.js';
+type HNSWNode = HNSWNoun;
+type Edge = HNSWVerb;
+/**
+ * Native Google Cloud Storage adapter for server environments
+ * Uses the @google-cloud/storage library with Application Default Credentials
+ *
+ * Authentication priority:
+ * 1. Application Default Credentials (if no credentials provided)
+ * 2. Service Account Key File (if keyFilename provided)
+ * 3. Service Account Credentials Object (if credentials provided)
+ * 4. HMAC Keys (if accessKeyId/secretAccessKey provided)
+ */
+export declare class GcsStorage extends BaseStorage {
+    private storage;
+    private bucket;
+    private bucketName;
+    private keyFilename?;
+    private credentials?;
+    private accessKeyId?;
+    private secretAccessKey?;
+    private nounPrefix;
+    private verbPrefix;
+    private metadataPrefix;
+    private verbMetadataPrefix;
+    private systemPrefix;
+    protected statisticsCache: StatisticsData | null;
+    private pendingOperations;
+    private maxConcurrentOperations;
+    private baseBatchSize;
+    private currentBatchSize;
+    private lastMemoryCheck;
+    private memoryCheckInterval;
+    private consecutiveErrors;
+    private lastErrorReset;
+    private backpressure;
+    private nounWriteBuffer;
+    private verbWriteBuffer;
+    private requestCoalescer;
+    private highVolumeMode;
+    private lastVolumeCheck;
+    private volumeCheckInterval;
+    private forceHighVolumeMode;
+    private nounCacheManager;
+    private verbCacheManager;
+    private logger;
+    /**
+     * Initialize the storage adapter
+     * @param options Configuration options for Google Cloud Storage
+     */
+    constructor(options: {
+        bucketName: string;
+        keyFilename?: string;
+        credentials?: object;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+        cacheConfig?: {
+            hotCacheMaxSize?: number;
+            hotCacheEvictionThreshold?: number;
+            warmCacheTTL?: number;
+        };
+        readOnly?: boolean;
+    });
+    /**
+     * Initialize the storage adapter
+     */
+    init(): Promise<void>;
+    /**
+     * Get the GCS object key for a noun using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/nouns/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getNounKey('ab123456-1234-5678-9abc-def012345678')
+     * // returns 'entities/nouns/vectors/ab/ab123456-1234-5678-9abc-def012345678.json'
+     */
+    private getNounKey;
+    /**
+     * Get the GCS object key for a verb using UUID-based sharding
+     *
+     * Uses first 2 hex characters of UUID for consistent sharding.
+     * Path format: entities/verbs/vectors/{shardId}/{uuid}.json
+     *
+     * @example
+     * getVerbKey('cd987654-4321-8765-cba9-fed543210987')
+     * // returns 'entities/verbs/vectors/cd/cd987654-4321-8765-cba9-fed543210987.json'
+     */
+    private getVerbKey;
+    /**
+     * Override base class method to detect GCS-specific throttling errors
+     */
+    protected isThrottlingError(error: any): boolean;
+    /**
+     * Apply backpressure before starting an operation
+     * @returns Request ID for tracking
+     */
+    private applyBackpressure;
+    /**
+     * Release backpressure after completing an operation
+     * @param success Whether the operation succeeded
+     * @param requestId Request ID from applyBackpressure()
+     */
+    private releaseBackpressure;
+    /**
+     * Check if high-volume mode should be enabled
+     */
+    private checkVolumeMode;
+    /**
+     * Flush noun buffer to GCS
+     */
+    private flushNounBuffer;
+    /**
+     * Flush verb buffer to GCS
+     */
+    private flushVerbBuffer;
+    /**
+     * Save a noun to storage (internal implementation)
+     */
+    protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
+    /**
+     * Save a node to storage
+     */
+    protected saveNode(node: HNSWNode): Promise<void>;
+    /**
+     * Save a node directly to GCS (bypass buffer)
+     */
+    private saveNodeDirect;
+    /**
+     * Get a noun from storage (internal implementation)
+     */
+    protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
+    /**
+     * Get a node from storage
+     */
+    protected getNode(id: string): Promise<HNSWNode | null>;
+    /**
+     * Delete a noun from storage (internal implementation)
+     */
+    protected deleteNoun_internal(id: string): Promise<void>;
+    /**
+     * Save noun metadata to storage (internal implementation)
+     */
+    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
+    /**
+     * Save metadata to storage (public API - delegates to saveNounMetadata_internal)
+     */
+    saveMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Get metadata from storage (public API - delegates to getNounMetadata)
+     */
+    getMetadata(id: string): Promise<any | null>;
+    /**
+     * Get noun metadata from storage
+     */
+    getNounMetadata(id: string): Promise<any | null>;
+    /**
+     * Save verb metadata to storage (internal implementation)
+     */
+    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
+    /**
+     * Get verb metadata from storage
+     */
+    getVerbMetadata(id: string): Promise<any | null>;
+    /**
+     * Save a verb to storage (internal implementation)
+     */
+    protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
+    /**
+     * Save an edge to storage
+     */
+    protected saveEdge(edge: Edge): Promise<void>;
+    /**
+     * Save an edge directly to GCS (bypass buffer)
+     */
+    private saveEdgeDirect;
+    /**
+     * Get a verb from storage (internal implementation)
+     */
+    protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
+    /**
+     * Get an edge from storage
+     */
+    protected getEdge(id: string): Promise<Edge | null>;
+    /**
+     * Delete a verb from storage (internal implementation)
+     */
+    protected deleteVerb_internal(id: string): Promise<void>;
+    /**
+     * Get nouns with pagination
+     * Iterates through all UUID-based shards (00-ff) for consistent pagination
+     */
+    getNounsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: HNSWNoun[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get nodes with pagination (internal implementation)
+     * Iterates through UUID-based shards for consistent pagination
+     */
+    private getNodesWithPagination;
+    /**
+     * Get nouns by noun type (internal implementation)
+     */
+    protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
+    /**
+     * Get verbs by source ID (internal implementation)
+     */
+    protected getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by target ID (internal implementation)
+     */
+    protected getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs by type (internal implementation)
+     */
+    protected getVerbsByType_internal(type: string): Promise<GraphVerb[]>;
+    /**
+     * Get verbs with pagination
+     */
+    getVerbsWithPagination(options?: {
+        limit?: number;
+        cursor?: string;
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: GraphVerb[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get nouns with filtering and pagination (public API)
+     */
+    getNouns(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            nounType?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: any[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Get verbs with filtering and pagination (public API)
+     */
+    getVerbs(options?: {
+        pagination?: {
+            offset?: number;
+            limit?: number;
+            cursor?: string;
+        };
+        filter?: {
+            verbType?: string | string[];
+            sourceId?: string | string[];
+            targetId?: string | string[];
+            service?: string | string[];
+            metadata?: Record<string, any>;
+        };
+    }): Promise<{
+        items: GraphVerb[];
+        totalCount?: number;
+        hasMore: boolean;
+        nextCursor?: string;
+    }>;
+    /**
+     * Clear all data from storage
+     */
+    clear(): Promise<void>;
+    /**
+     * Get storage status
+     */
+    getStorageStatus(): Promise<{
+        type: string;
+        used: number;
+        quota: number | null;
+        details?: Record<string, any>;
+    }>;
+    /**
+     * Save statistics data to storage
+     */
+    protected saveStatisticsData(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage
+     */
+    protected getStatisticsData(): Promise<StatisticsData | null>;
+    /**
+     * Initialize counts from storage
+     */
+    protected initializeCounts(): Promise<void>;
+    /**
+     * Initialize counts from storage scan (expensive - only for first-time init)
+     */
+    private initializeCountsFromScan;
+    /**
+     * Persist counts to storage
+     */
+    protected persistCounts(): Promise<void>;
+}
+export {};