@soulcraft/brainy 0.48.0 → 0.49.0

package/dist/storage/backwardCompatibility.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Backward Compatibility Layer for Storage Migration
+ *
+ * Handles the transition from 'index' to '_system' directory
+ * Ensures services running different versions can coexist
+ */
+import { StatisticsData } from '../coreTypes.js';
+export interface MigrationMetadata {
+    schemaVersion: number;
+    migrationStarted?: string;
+    migrationCompleted?: string;
+    lastUpdatedBy?: string;
+}
+/**
+ * Backward compatibility strategy for directory migration
+ */
+export declare class StorageCompatibilityLayer {
+    private migrationMetadata;
+    /**
+     * Determines the read strategy based on what's available
+     * @returns Priority-ordered list of directories to try
+     */
+    static getReadPriority(): string[];
+    /**
+     * Determines write strategy based on migration state
+     * @param migrationComplete Whether migration is complete
+     * @returns List of directories to write to
+     */
+    static getWriteTargets(migrationComplete?: boolean): string[];
+    /**
+     * Check if we should perform migration based on service coordination
+     * @param existingStats Statistics from storage
+     * @returns Whether to initiate migration
+     */
+    static shouldMigrate(existingStats: StatisticsData | null): boolean;
+    /**
+     * Creates migration metadata
+     */
+    static createMigrationMetadata(): MigrationMetadata;
+    /**
+     * Merge statistics from multiple locations (deduplication)
+     */
+    static mergeStatistics(primary: StatisticsData | null, fallback: StatisticsData | null): StatisticsData | null;
+    /**
+     * Determines if dual-write is needed based on environment
+     * @param storageType The type of storage being used
+     * @returns Whether to write to both old and new locations
+     */
+    static needsDualWrite(storageType: string): boolean;
+    /**
+     * Grace period for migration (30 days default)
+     * After this period, services can stop reading from old location
+     */
+    static getMigrationGracePeriodMs(): number;
+    /**
+     * Check if migration grace period has expired
+     */
+    static isGracePeriodExpired(migrationStarted: string): boolean;
+    /**
+     * Log migration events for monitoring
+     */
+    static logMigrationEvent(event: string, details?: any): void;
+}
+/**
+ * Storage paths helper for migration
+ */
+export declare class StoragePaths {
+    /**
+     * Get the statistics file path for a given directory
+     */
+    static getStatisticsPath(baseDir: string, filename?: string): string;
+    /**
+     * Get distributed config path
+     */
+    static getDistributedConfigPath(baseDir: string): string;
+    /**
+     * Check if a path is using the old structure
+     */
+    static isLegacyPath(path: string): boolean;
+    /**
+     * Convert legacy path to new structure
+     */
+    static modernizePath(path: string): string;
+}

package/dist/storage/backwardCompatibility.js

@@ -0,0 +1,141 @@
+/**
+ * Backward Compatibility Layer for Storage Migration
+ *
+ * Handles the transition from 'index' to '_system' directory
+ * Ensures services running different versions can coexist
+ */
+/**
+ * Backward compatibility strategy for directory migration
+ */
+export class StorageCompatibilityLayer {
+    constructor() {
+        this.migrationMetadata = null;
+    }
+    /**
+     * Determines the read strategy based on what's available
+     * @returns Priority-ordered list of directories to try
+     */
+    static getReadPriority() {
+        return ['_system', 'index']; // Try new location first, fallback to old
+    }
+    /**
+     * Determines write strategy based on migration state
+     * @param migrationComplete Whether migration is complete
+     * @returns List of directories to write to
+     */
+    static getWriteTargets(migrationComplete = false) {
+        if (migrationComplete) {
+            return ['_system']; // Only write to new location
+        }
+        // During migration, write to both for compatibility
+        return ['_system', 'index'];
+    }
+    /**
+     * Check if we should perform migration based on service coordination
+     * @param existingStats Statistics from storage
+     * @returns Whether to initiate migration
+     */
+    static shouldMigrate(existingStats) {
+        if (!existingStats)
+            return true; // No data yet, use new structure
+        // Check if we have migration metadata in stats
+        const migrationData = existingStats.migrationMetadata;
+        if (!migrationData)
+            return true; // No migration data, start migration
+        // Check schema version
+        if (migrationData.schemaVersion < 2)
+            return true;
+        // Already migrated
+        return false;
+    }
+    /**
+     * Creates migration metadata
+     */
+    static createMigrationMetadata() {
+        return {
+            schemaVersion: 2,
+            migrationStarted: new Date().toISOString(),
+            lastUpdatedBy: process.env.HOSTNAME || process.env.INSTANCE_ID || 'unknown'
+        };
+    }
+    /**
+     * Merge statistics from multiple locations (deduplication)
+     */
+    static mergeStatistics(primary, fallback) {
+        if (!primary && !fallback)
+            return null;
+        if (!fallback)
+            return primary;
+        if (!primary)
+            return fallback;
+        // Return the most recently updated
+        const primaryTime = new Date(primary.lastUpdated).getTime();
+        const fallbackTime = new Date(fallback.lastUpdated).getTime();
+        return primaryTime >= fallbackTime ? primary : fallback;
+    }
+    /**
+     * Determines if dual-write is needed based on environment
+     * @param storageType The type of storage being used
+     * @returns Whether to write to both old and new locations
+     */
+    static needsDualWrite(storageType) {
+        // Only need dual-write for shared storage systems
+        const sharedStorageTypes = ['s3', 'r2', 'gcs', 'filesystem'];
+        return sharedStorageTypes.includes(storageType.toLowerCase());
+    }
+    /**
+     * Grace period for migration (30 days default)
+     * After this period, services can stop reading from old location
+     */
+    static getMigrationGracePeriodMs() {
+        const days = parseInt(process.env.BRAINY_MIGRATION_GRACE_DAYS || '30', 10);
+        return days * 24 * 60 * 60 * 1000;
+    }
+    /**
+     * Check if migration grace period has expired
+     */
+    static isGracePeriodExpired(migrationStarted) {
+        const startTime = new Date(migrationStarted).getTime();
+        const now = Date.now();
+        const gracePeriod = this.getMigrationGracePeriodMs();
+        return (now - startTime) > gracePeriod;
+    }
+    /**
+     * Log migration events for monitoring
+     */
+    static logMigrationEvent(event, details) {
+        if (process.env.NODE_ENV !== 'test') {
+            console.log(`[Brainy Storage Migration] ${event}`, details || '');
+        }
+    }
+}
+/**
+ * Storage paths helper for migration
+ */
+export class StoragePaths {
+    /**
+     * Get the statistics file path for a given directory
+     */
+    static getStatisticsPath(baseDir, filename = 'statistics') {
+        return `${baseDir}/${filename}.json`;
+    }
+    /**
+     * Get distributed config path
+     */
+    static getDistributedConfigPath(baseDir) {
+        return `${baseDir}/distributed_config.json`;
+    }
+    /**
+     * Check if a path is using the old structure
+     */
+    static isLegacyPath(path) {
+        return path.includes('/index/') || path.endsWith('/index');
+    }
+    /**
+     * Convert legacy path to new structure
+     */
+    static modernizePath(path) {
+        return path.replace('/index/', '/_system/').replace('/index', '/_system');
+    }
+}
+//# sourceMappingURL=backwardCompatibility.js.map

package/dist/storage/backwardCompatibility.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backwardCompatibility.js","sourceRoot":"","sources":["../../src/storage/backwardCompatibility.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH;;GAEG;AACH,MAAM,OAAO,yBAAyB;IAAtC;QACU,sBAAiB,GAA6B,IAAI,CAAA;IA8G5D,CAAC;IA5GC;;;OAGG;IACH,MAAM,CAAC,eAAe;QACpB,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA,CAAE,0CAA0C;IACzE,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,eAAe,CAAC,oBAA6B,KAAK;QACvD,IAAI,iBAAiB,EAAE,CAAC;YACtB,OAAO,CAAC,SAAS,CAAC,CAAA,CAAE,6BAA6B;QACnD,CAAC;QACD,oDAAoD;QACpD,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC7B,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,aAAa,CAAC,aAAoC;QACvD,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAA,CAAE,iCAAiC;QAElE,+CAA+C;QAC/C,MAAM,aAAa,GAAI,aAAqB,CAAC,iBAAiB,CAAA;QAC9D,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAA,CAAE,qCAAqC;QAEtE,uBAAuB;QACvB,IAAI,aAAa,CAAC,aAAa,GAAG,CAAC;YAAE,OAAO,IAAI,CAAA;QAEhD,mBAAmB;QACnB,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,uBAAuB;QAC5B,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,SAAS;SAC5E,CAAA;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe,CACpB,OAA8B,EAC9B,QAA+B;QAE/B,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAA;QACtC,IAAI,CAAC,QAAQ;YAAE,OAAO,OAAO,CAAA;QAC7B,IAAI,CAAC,OAAO;YAAE,OAAO,QAAQ,CAAA;QAE7B,mCAAmC;QACnC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAA;QAC3D,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAA;QAE7D,OAAO,WAAW,IAAI,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAA;IACzD,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,cAAc,CAAC,WAAmB;QACvC,kDAAkD;QAClD,MAAM,kBAAkB,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,CAAC,CAAA;QAC5D,OAAO,kBAAkB,CAAC,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,CAAA;IAC/D,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,yBAAyB;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;QAC1E,OAAO,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;IACnC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,oBAAoB,CAAC,gBAAwB;QAClD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAA;QAEpD,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,WAAW,CAAA;IACxC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,iBAAiB,CAAC,KAAa,EAAE,OAAa;QACnD,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,8BAA8B,KAAK,EAAE,EAAE,OAAO,IAAI,EAAE,CAAC,CAAA;QACnE,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAY;IACvB;;OAEG;IACH,MAAM,CAAC,iBAAiB,CAAC,OAAe,EAAE,WAAmB,YAAY;QACvE,OAAO,GAAG,OAAO,IAAI,QAAQ,OAAO,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,wBAAwB,CAAC,OAAe;QAC7C,OAAO,GAAG,OAAO,0BAA0B,CAAA;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,YAAY,CAAC,IAAY;QAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAC5D,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,aAAa,CAAC,IAAY;QAC/B,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC3E,CAAC;CACF"}

package/dist/storage/baseStorage.d.ts

@@ -4,13 +4,27 @@
  */
 import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
+export declare const ENTITIES_DIR = "entities";
+export declare const NOUNS_VECTOR_DIR = "entities/nouns/vectors";
+export declare const NOUNS_METADATA_DIR = "entities/nouns/metadata";
+export declare const VERBS_VECTOR_DIR = "entities/verbs/vectors";
+export declare const VERBS_METADATA_DIR = "entities/verbs/metadata";
+export declare const INDEXES_DIR = "indexes";
+export declare const METADATA_INDEX_DIR = "indexes/metadata";
 export declare const NOUNS_DIR = "nouns";
 export declare const VERBS_DIR = "verbs";
 export declare const METADATA_DIR = "metadata";
 export declare const NOUN_METADATA_DIR = "noun-metadata";
 export declare const VERB_METADATA_DIR = "verb-metadata";
 export declare const INDEX_DIR = "index";
+export declare const SYSTEM_DIR = "_system";
 export declare const STATISTICS_KEY = "statistics";
+export declare const STORAGE_SCHEMA_VERSION = 3;
+export declare const USE_ENTITY_BASED_STRUCTURE = true;
+/**
+ * Get the appropriate directory path based on configuration
+ */
+export declare function getDirectoryPath(entityType: 'noun' | 'verb', dataType: 'vector' | 'metadata'): string;
 /**
  * Base storage adapter that implements common functionality
  * This is an abstract class that should be extended by specific storage adapters
@@ -35,12 +49,6 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Get a noun from storage
      */
     getNoun(id: string): Promise<HNSWNoun | null>;
-    /**
-     * Get all nouns from storage
-     * @deprecated This method is deprecated and will be removed in a future version.
-     * It can cause memory issues with large datasets. Use getNouns() with pagination instead.
-     */
-    getAllNouns(): Promise<HNSWNoun[]>;
     /**
      * Get nouns by noun type
      * @param nounType The noun type to filter by
@@ -65,10 +73,10 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
     protected convertHNSWVerbToGraphVerb(hnswVerb: HNSWVerb): Promise<GraphVerb | null>;
     /**
      * Get all verbs from storage
-     * @deprecated This method is deprecated and will be removed in a future version.
-     * It can cause memory issues with large datasets. Use getVerbs() with pagination instead.
+     * @returns Promise that resolves to an array of all HNSWVerbs
+     * @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
      */
-    getAllVerbs(): Promise<GraphVerb[]>;
+    getAllVerbs(): Promise<HNSWVerb[]>;
     /**
      * Get verbs by source
      */
@@ -81,6 +89,12 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Get verbs by type
      */
     getVerbsByType(type: string): Promise<GraphVerb[]>;
+    /**
+     * Get all nouns from storage
+     * @returns Promise that resolves to an array of all nouns
+     * @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
+     */
+    getAllNouns(): Promise<HNSWNoun[]>;
     /**
      * Get nouns with pagination and filtering
      * @param options Pagination and filtering options
@@ -186,11 +200,6 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * This method should be implemented by each specific adapter
      */
     protected abstract getNoun_internal(id: string): Promise<HNSWNoun | null>;
-    /**
-     * Get all nouns from storage
-     * This method should be implemented by each specific adapter
-     */
-    protected abstract getAllNouns_internal(): Promise<HNSWNoun[]>;
     /**
      * Get nouns by noun type
      * This method should be implemented by each specific adapter
@@ -211,11 +220,6 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * This method should be implemented by each specific adapter
      */
     protected abstract getVerb_internal(id: string): Promise<HNSWVerb | null>;
-    /**
-     * Get all verbs from storage
-     * This method should be implemented by each specific adapter
-     */
-    protected abstract getAllVerbs_internal(): Promise<HNSWVerb[]>;
     /**
      * Get verbs by source
      * This method should be implemented by each specific adapter
@@ -240,6 +244,16 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Helper method to convert a Map to a plain object for serialization
      */
     protected mapToObject<K extends string | number, V>(map: Map<K, V>, valueTransformer?: (value: V) => any): Record<string, any>;
+    /**
+     * Save statistics data to storage (public interface)
+     * @param statistics The statistics data to save
+     */
+    saveStatistics(statistics: StatisticsData): Promise<void>;
+    /**
+     * Get statistics data from storage (public interface)
+     * @returns Promise that resolves to the statistics data or null if not found
+     */
+    getStatistics(): Promise<StatisticsData | null>;
     /**
      * Save statistics data to storage
      * This method should be implemented by each specific adapter