@soulcraft/brainy 3.0.0 → 3.1.0

package/dist/storage/backwardCompatibility.js

@@ -1,141 +1,26 @@
 /**
- * Backward Compatibility Layer for Storage Migration
- *
- * Handles the transition from 'index' to '_system' directory
- * Ensures services running different versions can coexist
- */
-/**
- * Backward compatibility strategy for directory migration
+ * Storage backward compatibility layer for legacy data migrations
  */
 export class StorageCompatibilityLayer {
-    constructor() {
-        this.migrationMetadata = null;
-    }
-    /**
-     * Determines the read strategy based on what's available
-     * @returns Priority-ordered list of directories to try
-     */
-    static getReadPriority() {
-        return ['_system', 'index']; // Try new location first, fallback to old
-    }
-    /**
-     * Determines write strategy based on migration state
-     * @param migrationComplete Whether migration is complete
-     * @returns List of directories to write to
-     */
-    static getWriteTargets(migrationComplete = false) {
-        if (migrationComplete) {
-            return ['_system']; // Only write to new location
-        }
-        // During migration, write to both for compatibility
-        return ['_system', 'index'];
-    }
-    /**
-     * Check if we should perform migration based on service coordination
-     * @param existingStats Statistics from storage
-     * @returns Whether to initiate migration
-     */
-    static shouldMigrate(existingStats) {
-        if (!existingStats)
-            return true; // No data yet, use new structure
-        // Check if we have migration metadata in stats
-        const migrationData = existingStats.migrationMetadata;
-        if (!migrationData)
-            return true; // No migration data, start migration
-        // Check schema version
-        if (migrationData.schemaVersion < 2)
-            return true;
-        // Already migrated
-        return false;
-    }
-    /**
-     * Creates migration metadata
-     */
-    static createMigrationMetadata() {
-        return {
-            schemaVersion: 2,
-            migrationStarted: new Date().toISOString(),
-            lastUpdatedBy: process.env.HOSTNAME || process.env.INSTANCE_ID || 'unknown'
-        };
-    }
-    /**
-     * Merge statistics from multiple locations (deduplication)
-     */
-    static mergeStatistics(primary, fallback) {
-        if (!primary && !fallback)
-            return null;
-        if (!fallback)
-            return primary;
-        if (!primary)
-            return fallback;
-        // Return the most recently updated
-        const primaryTime = new Date(primary.lastUpdated).getTime();
-        const fallbackTime = new Date(fallback.lastUpdated).getTime();
-        return primaryTime >= fallbackTime ? primary : fallback;
-    }
-    /**
-     * Determines if dual-write is needed based on environment
-     * @param storageType The type of storage being used
-     * @returns Whether to write to both old and new locations
-     */
-    static needsDualWrite(storageType) {
-        // Only need dual-write for shared storage systems
-        const sharedStorageTypes = ['s3', 'r2', 'gcs', 'filesystem'];
-        return sharedStorageTypes.includes(storageType.toLowerCase());
-    }
-    /**
-     * Grace period for migration (30 days default)
-     * After this period, services can stop reading from old location
-     */
-    static getMigrationGracePeriodMs() {
-        const days = parseInt(process.env.BRAINY_MIGRATION_GRACE_DAYS || '30', 10);
-        return days * 24 * 60 * 60 * 1000;
-    }
-    /**
-     * Check if migration grace period has expired
-     */
-    static isGracePeriodExpired(migrationStarted) {
-        const startTime = new Date(migrationStarted).getTime();
-        const now = Date.now();
-        const gracePeriod = this.getMigrationGracePeriodMs();
-        return (now - startTime) > gracePeriod;
-    }
-    /**
-     * Log migration events for monitoring
-     */
     static logMigrationEvent(event, details) {
-        if (process.env.NODE_ENV !== 'test') {
-            console.log(`[Brainy Storage Migration] ${event}`, details || '');
+        // Simplified logging for migration events
+        if (process.env.DEBUG_MIGRATION) {
+            console.log(`[Migration] ${event}`, details);
         }
     }
-}
-/**
- * Storage paths helper for migration
- */
-export class StoragePaths {
-    /**
-     * Get the statistics file path for a given directory
-     */
-    static getStatisticsPath(baseDir, filename = 'statistics') {
-        return `${baseDir}/${filename}.json`;
-    }
-    /**
-     * Get distributed config path
-     */
-    static getDistributedConfigPath(baseDir) {
-        return `${baseDir}/distributed_config.json`;
-    }
-    /**
-     * Check if a path is using the old structure
-     */
-    static isLegacyPath(path) {
-        return path.includes('/index/') || path.endsWith('/index');
-    }
-    /**
-     * Convert legacy path to new structure
-     */
-    static modernizePath(path) {
-        return path.replace('/index/', '/_system/').replace('/index', '/_system');
+    static async migrateIfNeeded(storagePath) {
+        // No-op for now - can be extended later if needed
     }
 }
+// Helper to get default paths
+export function getDefaultStoragePaths(basePath) {
+    return {
+        nouns: `${basePath}/nouns`,
+        verbs: `${basePath}/verbs`,
+        metadata: `${basePath}/metadata`,
+        index: `${basePath}/index`,
+        system: `${basePath}/system`,
+        statistics: `${basePath}/statistics.json`
+    };
+}
 //# sourceMappingURL=backwardCompatibility.js.map

package/dist/storage/baseStorage.d.ts

@@ -2,6 +2,7 @@
  * Base Storage Adapter
  * Provides common functionality for all storage adapters
  */
+import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
 export declare const ENTITIES_DIR = "entities";
@@ -31,6 +32,7 @@ export declare function getDirectoryPath(entityType: 'noun' | 'verb', dataType:
  */
 export declare abstract class BaseStorage extends BaseStorageAdapter {
     protected isInitialized: boolean;
+    protected graphIndex?: GraphAdjacencyIndex;
     protected readOnly: boolean;
     /**
      * Initialize the storage adapter
@@ -143,6 +145,10 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Delete a verb from storage
      */
     deleteVerb(id: string): Promise<void>;
+    /**
+     * Get graph index (lazy initialization)
+     */
+    getGraphIndex(): Promise<GraphAdjacencyIndex>;
     /**
      * Clear all data from storage
      * This method should be implemented by each specific adapter
@@ -172,7 +178,12 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Save noun metadata to storage
      * This method should be implemented by each specific adapter
      */
-    abstract saveNounMetadata(id: string, metadata: any): Promise<void>;
+    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Internal method for saving noun metadata
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get noun metadata from storage
      * This method should be implemented by each specific adapter
@@ -182,7 +193,12 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Save verb metadata to storage
      * This method should be implemented by each specific adapter
      */
-    abstract saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    /**
+     * Internal method for saving verb metadata
+     * This method should be implemented by each specific adapter
+     */
+    protected abstract saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get verb metadata from storage
      * This method should be implemented by each specific adapter

package/dist/storage/baseStorage.js

@@ -2,7 +2,9 @@
  * Base Storage Adapter
  * Provides common functionality for all storage adapters
  */
+import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
+import { validateNounType, validateVerbType } from '../utils/typeValidation.js';
 // Common directory/prefix names
 // Option A: Entity-Based Directory Structure
 export const ENTITIES_DIR = 'entities';
@@ -71,6 +73,11 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async saveNoun(noun) {
         await this.ensureInitialized();
+        // Validate noun type before saving - storage boundary protection
+        const metadata = await this.getNounMetadata(noun.id);
+        if (metadata?.noun) {
+            validateNounType(metadata.noun);
+        }
         return this.saveNoun_internal(noun);
     }
     /**
@@ -101,6 +108,10 @@ export class BaseStorage extends BaseStorageAdapter {
      */
     async saveVerb(verb) {
         await this.ensureInitialized();
+        // Validate verb type before saving - storage boundary protection
+        if (verb.verb) {
+            validateVerbType(verb.verb);
+        }
         // Extract the lightweight HNSWVerb data
         const hnswVerb = {
             id: verb.id,
@@ -123,9 +134,31 @@ export class BaseStorage extends BaseStorageAdapter {
             createdBy: verb.createdBy,
             embedding: verb.embedding
         };
-        // Save both the HNSWVerb and metadata
-        await this.saveVerb_internal(hnswVerb);
-        await this.saveVerbMetadata(verb.id, metadata);
+        // Save both the HNSWVerb and metadata atomically
+        try {
+            console.log(`[DEBUG] Saving verb ${verb.id}: sourceId=${verb.sourceId}, targetId=${verb.targetId}`);
+            // Save the HNSWVerb first
+            await this.saveVerb_internal(hnswVerb);
+            console.log(`[DEBUG] Successfully saved HNSWVerb file for ${verb.id}`);
+            // Then save the metadata
+            await this.saveVerbMetadata(verb.id, metadata);
+            console.log(`[DEBUG] Successfully saved metadata file for ${verb.id}`);
+        }
+        catch (error) {
+            console.error(`[ERROR] Failed to save verb ${verb.id}:`, error);
+            // Attempt cleanup - remove verb file if metadata failed
+            try {
+                const verbExists = await this.getVerb_internal(verb.id);
+                if (verbExists) {
+                    console.log(`[CLEANUP] Attempting to remove orphaned verb file ${verb.id}`);
+                    await this.deleteVerb_internal(verb.id);
+                }
+            }
+            catch (cleanupError) {
+                console.error(`[ERROR] Failed to cleanup orphaned verb ${verb.id}:`, cleanupError);
+            }
+            throw new Error(`Failed to save verb ${verb.id}: ${error instanceof Error ? error.message : String(error)}`);
+        }
     }
     /**
      * Get a verb from storage
@@ -488,6 +521,44 @@ export class BaseStorage extends BaseStorageAdapter {
         await this.ensureInitialized();
         return this.deleteVerb_internal(id);
     }
+    /**
+     * Get graph index (lazy initialization)
+     */
+    async getGraphIndex() {
+        if (!this.graphIndex) {
+            console.log('Initializing GraphAdjacencyIndex...');
+            this.graphIndex = new GraphAdjacencyIndex(this);
+            // Check if we need to rebuild from existing data
+            const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } });
+            if (sampleVerbs.items.length > 0) {
+                console.log('Found existing verbs, rebuilding graph index...');
+                await this.graphIndex.rebuild();
+            }
+        }
+        return this.graphIndex;
+    }
+    /**
+     * Save noun metadata to storage
+     * This method should be implemented by each specific adapter
+     */
+    async saveNounMetadata(id, metadata) {
+        // Validate noun type in metadata - storage boundary protection
+        if (metadata?.noun) {
+            validateNounType(metadata.noun);
+        }
+        return this.saveNounMetadata_internal(id, metadata);
+    }
+    /**
+     * Save verb metadata to storage
+     * This method should be implemented by each specific adapter
+     */
+    async saveVerbMetadata(id, metadata) {
+        // Validate verb type in metadata - storage boundary protection
+        if (metadata?.verb) {
+            validateVerbType(metadata.verb);
+        }
+        return this.saveVerbMetadata_internal(id, metadata);
+    }
     /**
      * Helper method to convert a Map to a plain object for serialization
      */

package/dist/storage/cacheManager.js

@@ -120,7 +120,7 @@ export class CacheManager {
                 (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0;
             // Determine if we're dealing with a large dataset (>100K items)
             const isLargeDataset = totalItems > 100000;
-            // Check if we're in read-only mode (from parent BrainyData instance)
+            // Check if we're in read-only mode (from parent Brainy instance)
             const isReadOnly = this.options?.readOnly || false;
             // In Node.js, use available system memory with enhanced allocation
             if (this.environment === Environment.NODE) {
@@ -220,7 +220,7 @@ export class CacheManager {
                 (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0;
             // Determine if we're dealing with a large dataset (>100K items)
             const isLargeDataset = totalItems > 100000;
-            // Check if we're in read-only mode (from parent BrainyData instance)
+            // Check if we're in read-only mode (from parent Brainy instance)
             const isReadOnly = this.options?.readOnly || false;
             // Get memory information based on environment
             const memoryInfo = await this.detectAvailableMemory();

package/dist/storage/readOnlyOptimizations.js

@@ -309,9 +309,14 @@ export class ReadOnlyOptimizations {
      * Load segment data from storage
      */
     async loadSegmentFromStorage(segment) {
-        // This would integrate with your S3 storage adapter
-        // For now, return a placeholder
-        console.log(`Loading segment ${segment.id} from storage`);
+        // Load segment from memory-mapped buffer if available
+        const cached = this.memoryMappedBuffers.get(segment.id);
+        if (cached) {
+            return cached;
+        }
+        // This feature requires actual storage backend integration (S3, file system, etc)
+        // Return empty buffer as this is an optional optimization feature
+        console.warn(`Segment loading optimization not available for segment ${segment.id}. Using standard storage.`);
         return new ArrayBuffer(0);
     }
     /**

package/dist/streaming/pipeline.d.ts

@@ -0,0 +1,154 @@
+/**
+ * Streaming Pipeline System for Brainy
+ *
+ * Real implementation of streaming data pipelines with:
+ * - Async iterators for streaming
+ * - Backpressure handling
+ * - Auto-scaling workers
+ * - Checkpointing for recovery
+ * - Error boundaries
+ */
+import { Brainy } from '../brainy.js';
+/**
+ * Pipeline stage types
+ */
+export type StageType = 'source' | 'transform' | 'filter' | 'batch' | 'sink' | 'branch' | 'merge' | 'window' | 'reduce';
+/**
+ * Pipeline execution options
+ */
+export interface PipelineOptions {
+    workers?: number | 'auto';
+    checkpoint?: boolean | string;
+    monitoring?: boolean;
+    maxThroughput?: number;
+    backpressure?: 'drop' | 'buffer' | 'pause';
+    retries?: number;
+    errorHandler?: (error: Error, item: any) => void;
+    bufferSize?: number;
+}
+/**
+ * Base interface for pipeline stages
+ */
+export interface PipelineStage<T = any, R = any> {
+    type: StageType;
+    name: string;
+    process(input: AsyncIterable<T>): AsyncIterable<R>;
+}
+/**
+ * Streaming Pipeline Builder
+ */
+export declare class Pipeline<T = any> {
+    private brainyInstance?;
+    private stages;
+    private running;
+    private abortController?;
+    private metrics;
+    constructor(brainyInstance?: (Brainy | Brainy<any>) | undefined);
+    /**
+     * Add a data source
+     */
+    source<S>(generator: AsyncIterable<S> | (() => AsyncIterable<S>) | AsyncGeneratorFunction): Pipeline<S>;
+    /**
+     * Transform data
+     */
+    map<R>(fn: (item: T) => R | Promise<R>): Pipeline<R>;
+    /**
+     * Filter data
+     */
+    filter(predicate: (item: T) => boolean | Promise<boolean>): Pipeline<T>;
+    /**
+     * Batch items for efficiency
+     */
+    batch(size: number, timeoutMs?: number): Pipeline<T[]>;
+    /**
+     * Sink data to a destination
+     */
+    sink(handler: (item: T) => Promise<void> | void): Pipeline<void>;
+    /**
+     * Sink data to Brainy
+     */
+    toBrainy(options?: {
+        type?: string;
+        metadata?: any;
+        batchSize?: number;
+    }): Pipeline<void>;
+    /**
+     * Sink with rate limiting
+     */
+    throttledSink(handler: (item: T) => Promise<void> | void, rateLimit: number): Pipeline<void>;
+    /**
+     * Parallel sink with worker pool
+     */
+    parallelSink(handler: (item: T) => Promise<void> | void, workers?: number): Pipeline<void>;
+    /**
+     * Collect all results
+     */
+    collect(): Promise<T[]>;
+    /**
+     * Window operations for time-based processing
+     */
+    window(size: number, type?: 'tumbling' | 'sliding'): Pipeline<T[]>;
+    /**
+     * Flatmap operation - map and flatten results
+     */
+    flatMap<R>(fn: (item: T) => R[] | Promise<R[]>): Pipeline<R>;
+    /**
+     * Tap into the pipeline for side effects without modifying data
+     */
+    tap(fn: (item: T) => void | Promise<void>): Pipeline<T>;
+    /**
+     * Retry failed operations
+     */
+    retry<R>(fn: (item: T) => R | Promise<R>, maxRetries?: number, backoff?: number): Pipeline<R>;
+    /**
+     * Buffer with backpressure handling
+     */
+    buffer(size: number, strategy?: 'drop' | 'block'): Pipeline<T>;
+    /**
+     * Fork the pipeline into multiple branches
+     */
+    fork(...branches: Array<(pipeline: Pipeline<T>) => Pipeline<any>>): Pipeline<T>;
+    /**
+     * Reduce operation
+     */
+    reduce<R>(reducer: (acc: R, item: T) => R, initial: R): Pipeline<R>;
+    /**
+     * Run the pipeline with metrics tracking
+     */
+    run(options?: PipelineOptions): Promise<void>;
+    /**
+     * Start the pipeline (alias for run)
+     */
+    start(options?: PipelineOptions): Promise<void>;
+    /**
+     * Stop the pipeline
+     */
+    stop(): void;
+    /**
+     * Monitor pipeline metrics
+     */
+    monitor(dashboard?: string): Pipeline<T>;
+}
+/**
+ * Pipeline factory function
+ */
+export declare function createPipeline(brain?: Brainy): Pipeline;
+/**
+ * Backward compatibility exports
+ */
+export declare const pipeline: Pipeline<any>;
+export declare enum ExecutionMode {
+    SEQUENTIAL = "sequential",
+    PARALLEL = "parallel",
+    FIRST_SUCCESS = "firstSuccess",
+    FIRST_RESULT = "firstResult",
+    THREADED = "threaded"
+}
+export type PipelineResult<T> = {
+    success: boolean;
+    data: T;
+    error?: string;
+};
+export type StreamlinedPipelineOptions = PipelineOptions;
+export type StreamlinedPipelineResult<T> = PipelineResult<T>;
+export { ExecutionMode as StreamlinedExecutionMode };