@soulcraft/brainy 2.15.0 → 3.0.1

package/dist/storage/adapters/fileSystemStorage.js

@@ -1180,7 +1180,60 @@ export class FileSystemStorage extends BaseStorage {
             }
         }
         // Merge statistics from both locations
-        return StorageCompatibilityLayer.mergeStatistics(newStats, oldStats);
+        return this.mergeStatistics(newStats, oldStats);
+    }
+    /**
+     * Merge statistics from multiple sources
+     */
+    mergeStatistics(storageStats, localStats) {
+        // Handle null cases
+        if (!storageStats && !localStats) {
+            return {
+                nounCount: {},
+                verbCount: {},
+                metadataCount: {},
+                hnswIndexSize: 0,
+                totalNodes: 0,
+                totalEdges: 0,
+                lastUpdated: new Date().toISOString()
+            };
+        }
+        if (!storageStats)
+            return localStats;
+        if (!localStats)
+            return storageStats;
+        // Merge noun counts by taking the maximum of each type
+        const mergedNounCount = {
+            ...storageStats.nounCount
+        };
+        for (const [type, count] of Object.entries(localStats.nounCount)) {
+            mergedNounCount[type] = Math.max(mergedNounCount[type] || 0, count);
+        }
+        // Merge verb counts by taking the maximum of each type
+        const mergedVerbCount = {
+            ...storageStats.verbCount
+        };
+        for (const [type, count] of Object.entries(localStats.verbCount)) {
+            mergedVerbCount[type] = Math.max(mergedVerbCount[type] || 0, count);
+        }
+        // Merge metadata counts by taking the maximum of each type
+        const mergedMetadataCount = {
+            ...storageStats.metadataCount
+        };
+        for (const [type, count] of Object.entries(localStats.metadataCount)) {
+            mergedMetadataCount[type] = Math.max(mergedMetadataCount[type] || 0, count);
+        }
+        return {
+            nounCount: mergedNounCount,
+            verbCount: mergedVerbCount,
+            metadataCount: mergedMetadataCount,
+            hnswIndexSize: Math.max(storageStats.hnswIndexSize || 0, localStats.hnswIndexSize || 0),
+            totalNodes: Math.max(storageStats.totalNodes || 0, localStats.totalNodes || 0),
+            totalEdges: Math.max(storageStats.totalEdges || 0, localStats.totalEdges || 0),
+            totalMetadata: Math.max(storageStats.totalMetadata || 0, localStats.totalMetadata || 0),
+            operations: storageStats.operations || localStats.operations,
+            lastUpdated: new Date().toISOString()
+        };
     }
 }
 //# sourceMappingURL=fileSystemStorage.js.map

package/dist/storage/adapters/memoryStorage.js

@@ -35,7 +35,8 @@ export class MemoryStorage extends BaseStorage {
             id: noun.id,
             vector: [...noun.vector],
             connections: new Map(),
-            level: noun.level || 0
+            level: noun.level || 0,
+            metadata: noun.metadata
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -59,7 +60,8 @@ export class MemoryStorage extends BaseStorage {
             id: noun.id,
             vector: [...noun.vector],
             connections: new Map(),
-            level: noun.level || 0
+            level: noun.level || 0,
+            metadata: noun.metadata
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -89,12 +91,14 @@ export class MemoryStorage extends BaseStorage {
         const matchingIds = [];
         // Iterate through all nouns to find matches
         for (const [nounId, noun] of this.nouns.entries()) {
-            // Get the metadata to check filters
-            const metadata = await this.getMetadata(nounId);
-            if (!metadata)
-                continue;
+            // Check the noun's embedded metadata field
+            const nounMetadata = noun.metadata || {};
+            // Also check separate metadata store for backward compatibility
+            const separateMetadata = await this.getMetadata(nounId);
+            // Merge both metadata sources (noun.metadata takes precedence)
+            const metadata = { ...separateMetadata, ...nounMetadata };
             // Filter by noun type if specified
-            if (nounTypes && !nounTypes.includes(metadata.noun)) {
+            if (nounTypes && metadata.noun && !nounTypes.includes(metadata.noun)) {
                 continue;
             }
             // Filter by service if specified
@@ -133,7 +137,8 @@ export class MemoryStorage extends BaseStorage {
                 id: noun.id,
                 vector: [...noun.vector],
                 connections: new Map(),
-                level: noun.level || 0
+                level: noun.level || 0,
+                metadata: noun.metadata
             };
             // Copy connections
             for (const [level, connections] of noun.connections.entries()) {

package/dist/storage/backwardCompatibility.d.ts

@@ -1,84 +1,16 @@
 /**
- * Backward Compatibility Layer for Storage Migration
- *
- * Handles the transition from 'index' to '_system' directory
- * Ensures services running different versions can coexist
- */
-import { StatisticsData } from '../coreTypes.js';
-export interface MigrationMetadata {
-    schemaVersion: number;
-    migrationStarted?: string;
-    migrationCompleted?: string;
-    lastUpdatedBy?: string;
-}
-/**
- * Backward compatibility strategy for directory migration
+ * Storage backward compatibility layer for legacy data migrations
  */
 export declare class StorageCompatibilityLayer {
-    private migrationMetadata;
-    /**
-     * Determines the read strategy based on what's available
-     * @returns Priority-ordered list of directories to try
-     */
-    static getReadPriority(): string[];
-    /**
-     * Determines write strategy based on migration state
-     * @param migrationComplete Whether migration is complete
-     * @returns List of directories to write to
-     */
-    static getWriteTargets(migrationComplete?: boolean): string[];
-    /**
-     * Check if we should perform migration based on service coordination
-     * @param existingStats Statistics from storage
-     * @returns Whether to initiate migration
-     */
-    static shouldMigrate(existingStats: StatisticsData | null): boolean;
-    /**
-     * Creates migration metadata
-     */
-    static createMigrationMetadata(): MigrationMetadata;
-    /**
-     * Merge statistics from multiple locations (deduplication)
-     */
-    static mergeStatistics(primary: StatisticsData | null, fallback: StatisticsData | null): StatisticsData | null;
-    /**
-     * Determines if dual-write is needed based on environment
-     * @param storageType The type of storage being used
-     * @returns Whether to write to both old and new locations
-     */
-    static needsDualWrite(storageType: string): boolean;
-    /**
-     * Grace period for migration (30 days default)
-     * After this period, services can stop reading from old location
-     */
-    static getMigrationGracePeriodMs(): number;
-    /**
-     * Check if migration grace period has expired
-     */
-    static isGracePeriodExpired(migrationStarted: string): boolean;
-    /**
-     * Log migration events for monitoring
-     */
     static logMigrationEvent(event: string, details?: any): void;
+    static migrateIfNeeded(storagePath: string): Promise<void>;
 }
-/**
- * Storage paths helper for migration
- */
-export declare class StoragePaths {
-    /**
-     * Get the statistics file path for a given directory
-     */
-    static getStatisticsPath(baseDir: string, filename?: string): string;
-    /**
-     * Get distributed config path
-     */
-    static getDistributedConfigPath(baseDir: string): string;
-    /**
-     * Check if a path is using the old structure
-     */
-    static isLegacyPath(path: string): boolean;
-    /**
-     * Convert legacy path to new structure
-     */
-    static modernizePath(path: string): string;
+export interface StoragePaths {
+    nouns: string;
+    verbs: string;
+    metadata: string;
+    index: string;
+    system: string;
+    statistics: string;
 }
+export declare function getDefaultStoragePaths(basePath: string): StoragePaths;

package/dist/storage/backwardCompatibility.js

@@ -1,141 +1,26 @@
 /**
- * Backward Compatibility Layer for Storage Migration
- *
- * Handles the transition from 'index' to '_system' directory
- * Ensures services running different versions can coexist
- */
-/**
- * Backward compatibility strategy for directory migration
+ * Storage backward compatibility layer for legacy data migrations
  */
 export class StorageCompatibilityLayer {
-    constructor() {
-        this.migrationMetadata = null;
-    }
-    /**
-     * Determines the read strategy based on what's available
-     * @returns Priority-ordered list of directories to try
-     */
-    static getReadPriority() {
-        return ['_system', 'index']; // Try new location first, fallback to old
-    }
-    /**
-     * Determines write strategy based on migration state
-     * @param migrationComplete Whether migration is complete
-     * @returns List of directories to write to
-     */
-    static getWriteTargets(migrationComplete = false) {
-        if (migrationComplete) {
-            return ['_system']; // Only write to new location
-        }
-        // During migration, write to both for compatibility
-        return ['_system', 'index'];
-    }
-    /**
-     * Check if we should perform migration based on service coordination
-     * @param existingStats Statistics from storage
-     * @returns Whether to initiate migration
-     */
-    static shouldMigrate(existingStats) {
-        if (!existingStats)
-            return true; // No data yet, use new structure
-        // Check if we have migration metadata in stats
-        const migrationData = existingStats.migrationMetadata;
-        if (!migrationData)
-            return true; // No migration data, start migration
-        // Check schema version
-        if (migrationData.schemaVersion < 2)
-            return true;
-        // Already migrated
-        return false;
-    }
-    /**
-     * Creates migration metadata
-     */
-    static createMigrationMetadata() {
-        return {
-            schemaVersion: 2,
-            migrationStarted: new Date().toISOString(),
-            lastUpdatedBy: process.env.HOSTNAME || process.env.INSTANCE_ID || 'unknown'
-        };
-    }
-    /**
-     * Merge statistics from multiple locations (deduplication)
-     */
-    static mergeStatistics(primary, fallback) {
-        if (!primary && !fallback)
-            return null;
-        if (!fallback)
-            return primary;
-        if (!primary)
-            return fallback;
-        // Return the most recently updated
-        const primaryTime = new Date(primary.lastUpdated).getTime();
-        const fallbackTime = new Date(fallback.lastUpdated).getTime();
-        return primaryTime >= fallbackTime ? primary : fallback;
-    }
-    /**
-     * Determines if dual-write is needed based on environment
-     * @param storageType The type of storage being used
-     * @returns Whether to write to both old and new locations
-     */
-    static needsDualWrite(storageType) {
-        // Only need dual-write for shared storage systems
-        const sharedStorageTypes = ['s3', 'r2', 'gcs', 'filesystem'];
-        return sharedStorageTypes.includes(storageType.toLowerCase());
-    }
-    /**
-     * Grace period for migration (30 days default)
-     * After this period, services can stop reading from old location
-     */
-    static getMigrationGracePeriodMs() {
-        const days = parseInt(process.env.BRAINY_MIGRATION_GRACE_DAYS || '30', 10);
-        return days * 24 * 60 * 60 * 1000;
-    }
-    /**
-     * Check if migration grace period has expired
-     */
-    static isGracePeriodExpired(migrationStarted) {
-        const startTime = new Date(migrationStarted).getTime();
-        const now = Date.now();
-        const gracePeriod = this.getMigrationGracePeriodMs();
-        return (now - startTime) > gracePeriod;
-    }
-    /**
-     * Log migration events for monitoring
-     */
     static logMigrationEvent(event, details) {
-        if (process.env.NODE_ENV !== 'test') {
-            console.log(`[Brainy Storage Migration] ${event}`, details || '');
+        // Simplified logging for migration events
+        if (process.env.DEBUG_MIGRATION) {
+            console.log(`[Migration] ${event}`, details);
         }
     }
-}
-/**
- * Storage paths helper for migration
- */
-export class StoragePaths {
-    /**
-     * Get the statistics file path for a given directory
-     */
-    static getStatisticsPath(baseDir, filename = 'statistics') {
-        return `${baseDir}/${filename}.json`;
-    }
-    /**
-     * Get distributed config path
-     */
-    static getDistributedConfigPath(baseDir) {
-        return `${baseDir}/distributed_config.json`;
-    }
-    /**
-     * Check if a path is using the old structure
-     */
-    static isLegacyPath(path) {
-        return path.includes('/index/') || path.endsWith('/index');
-    }
-    /**
-     * Convert legacy path to new structure
-     */
-    static modernizePath(path) {
-        return path.replace('/index/', '/_system/').replace('/index', '/_system');
+    static async migrateIfNeeded(storagePath) {
+        // No-op for now - can be extended later if needed
     }
 }
+// Helper to get default paths
+export function getDefaultStoragePaths(basePath) {
+    return {
+        nouns: `${basePath}/nouns`,
+        verbs: `${basePath}/verbs`,
+        metadata: `${basePath}/metadata`,
+        index: `${basePath}/index`,
+        system: `${basePath}/system`,
+        statistics: `${basePath}/statistics.json`
+    };
+}
 //# sourceMappingURL=backwardCompatibility.js.map

package/dist/storage/baseStorage.d.ts

@@ -2,6 +2,7 @@
  * Base Storage Adapter
  * Provides common functionality for all storage adapters
  */
+import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
 export declare const ENTITIES_DIR = "entities";
@@ -31,6 +32,7 @@ export declare function getDirectoryPath(entityType: 'noun' | 'verb', dataType:
  */
 export declare abstract class BaseStorage extends BaseStorageAdapter {
     protected isInitialized: boolean;
+    protected graphIndex?: GraphAdjacencyIndex;
     protected readOnly: boolean;
     /**
      * Initialize the storage adapter
@@ -143,6 +145,10 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
      * Delete a verb from storage
      */
     deleteVerb(id: string): Promise<void>;
+    /**
+     * Get graph index (lazy initialization)
+     */
+    getGraphIndex(): Promise<GraphAdjacencyIndex>;
     /**
      * Clear all data from storage
      * This method should be implemented by each specific adapter

package/dist/storage/baseStorage.js

@@ -2,6 +2,7 @@
  * Base Storage Adapter
  * Provides common functionality for all storage adapters
  */
+import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
 import { validateNounType, validateVerbType } from '../utils/typeValidation.js';
 // Common directory/prefix names
@@ -520,6 +521,22 @@ export class BaseStorage extends BaseStorageAdapter {
         await this.ensureInitialized();
         return this.deleteVerb_internal(id);
     }
+    /**
+     * Get graph index (lazy initialization)
+     */
+    async getGraphIndex() {
+        if (!this.graphIndex) {
+            console.log('Initializing GraphAdjacencyIndex...');
+            this.graphIndex = new GraphAdjacencyIndex(this);
+            // Check if we need to rebuild from existing data
+            const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } });
+            if (sampleVerbs.items.length > 0) {
+                console.log('Found existing verbs, rebuilding graph index...');
+                await this.graphIndex.rebuild();
+            }
+        }
+        return this.graphIndex;
+    }
     /**
      * Save noun metadata to storage
      * This method should be implemented by each specific adapter

package/dist/storage/cacheManager.js

@@ -120,7 +120,7 @@ export class CacheManager {
                 (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0;
             // Determine if we're dealing with a large dataset (>100K items)
             const isLargeDataset = totalItems > 100000;
-            // Check if we're in read-only mode (from parent BrainyData instance)
+            // Check if we're in read-only mode (from parent Brainy instance)
             const isReadOnly = this.options?.readOnly || false;
             // In Node.js, use available system memory with enhanced allocation
             if (this.environment === Environment.NODE) {
@@ -220,7 +220,7 @@ export class CacheManager {
                 (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0;
             // Determine if we're dealing with a large dataset (>100K items)
             const isLargeDataset = totalItems > 100000;
-            // Check if we're in read-only mode (from parent BrainyData instance)
+            // Check if we're in read-only mode (from parent Brainy instance)
             const isReadOnly = this.options?.readOnly || false;
             // Get memory information based on environment
             const memoryInfo = await this.detectAvailableMemory();

package/dist/storage/readOnlyOptimizations.js

@@ -309,9 +309,14 @@ export class ReadOnlyOptimizations {
      * Load segment data from storage
      */
     async loadSegmentFromStorage(segment) {
-        // This would integrate with your S3 storage adapter
-        // For now, return a placeholder
-        console.log(`Loading segment ${segment.id} from storage`);
+        // Load segment from memory-mapped buffer if available
+        const cached = this.memoryMappedBuffers.get(segment.id);
+        if (cached) {
+            return cached;
+        }
+        // This feature requires actual storage backend integration (S3, file system, etc)
+        // Return empty buffer as this is an optional optimization feature
+        console.warn(`Segment loading optimization not available for segment ${segment.id}. Using standard storage.`);
         return new ArrayBuffer(0);
     }
     /**

package/dist/streaming/pipeline.d.ts

@@ -0,0 +1,154 @@
+/**
+ * Streaming Pipeline System for Brainy
+ *
+ * Real implementation of streaming data pipelines with:
+ * - Async iterators for streaming
+ * - Backpressure handling
+ * - Auto-scaling workers
+ * - Checkpointing for recovery
+ * - Error boundaries
+ */
+import { Brainy } from '../brainy.js';
+/**
+ * Pipeline stage types
+ */
+export type StageType = 'source' | 'transform' | 'filter' | 'batch' | 'sink' | 'branch' | 'merge' | 'window' | 'reduce';
+/**
+ * Pipeline execution options
+ */
+export interface PipelineOptions {
+    workers?: number | 'auto';
+    checkpoint?: boolean | string;
+    monitoring?: boolean;
+    maxThroughput?: number;
+    backpressure?: 'drop' | 'buffer' | 'pause';
+    retries?: number;
+    errorHandler?: (error: Error, item: any) => void;
+    bufferSize?: number;
+}
+/**
+ * Base interface for pipeline stages
+ */
+export interface PipelineStage<T = any, R = any> {
+    type: StageType;
+    name: string;
+    process(input: AsyncIterable<T>): AsyncIterable<R>;
+}
+/**
+ * Streaming Pipeline Builder
+ */
+export declare class Pipeline<T = any> {
+    private brainyInstance?;
+    private stages;
+    private running;
+    private abortController?;
+    private metrics;
+    constructor(brainyInstance?: (Brainy | Brainy<any>) | undefined);
+    /**
+     * Add a data source
+     */
+    source<S>(generator: AsyncIterable<S> | (() => AsyncIterable<S>) | AsyncGeneratorFunction): Pipeline<S>;
+    /**
+     * Transform data
+     */
+    map<R>(fn: (item: T) => R | Promise<R>): Pipeline<R>;
+    /**
+     * Filter data
+     */
+    filter(predicate: (item: T) => boolean | Promise<boolean>): Pipeline<T>;
+    /**
+     * Batch items for efficiency
+     */
+    batch(size: number, timeoutMs?: number): Pipeline<T[]>;
+    /**
+     * Sink data to a destination
+     */
+    sink(handler: (item: T) => Promise<void> | void): Pipeline<void>;
+    /**
+     * Sink data to Brainy
+     */
+    toBrainy(options?: {
+        type?: string;
+        metadata?: any;
+        batchSize?: number;
+    }): Pipeline<void>;
+    /**
+     * Sink with rate limiting
+     */
+    throttledSink(handler: (item: T) => Promise<void> | void, rateLimit: number): Pipeline<void>;
+    /**
+     * Parallel sink with worker pool
+     */
+    parallelSink(handler: (item: T) => Promise<void> | void, workers?: number): Pipeline<void>;
+    /**
+     * Collect all results
+     */
+    collect(): Promise<T[]>;
+    /**
+     * Window operations for time-based processing
+     */
+    window(size: number, type?: 'tumbling' | 'sliding'): Pipeline<T[]>;
+    /**
+     * Flatmap operation - map and flatten results
+     */
+    flatMap<R>(fn: (item: T) => R[] | Promise<R[]>): Pipeline<R>;
+    /**
+     * Tap into the pipeline for side effects without modifying data
+     */
+    tap(fn: (item: T) => void | Promise<void>): Pipeline<T>;
+    /**
+     * Retry failed operations
+     */
+    retry<R>(fn: (item: T) => R | Promise<R>, maxRetries?: number, backoff?: number): Pipeline<R>;
+    /**
+     * Buffer with backpressure handling
+     */
+    buffer(size: number, strategy?: 'drop' | 'block'): Pipeline<T>;
+    /**
+     * Fork the pipeline into multiple branches
+     */
+    fork(...branches: Array<(pipeline: Pipeline<T>) => Pipeline<any>>): Pipeline<T>;
+    /**
+     * Reduce operation
+     */
+    reduce<R>(reducer: (acc: R, item: T) => R, initial: R): Pipeline<R>;
+    /**
+     * Run the pipeline with metrics tracking
+     */
+    run(options?: PipelineOptions): Promise<void>;
+    /**
+     * Start the pipeline (alias for run)
+     */
+    start(options?: PipelineOptions): Promise<void>;
+    /**
+     * Stop the pipeline
+     */
+    stop(): void;
+    /**
+     * Monitor pipeline metrics
+     */
+    monitor(dashboard?: string): Pipeline<T>;
+}
+/**
+ * Pipeline factory function
+ */
+export declare function createPipeline(brain?: Brainy): Pipeline;
+/**
+ * Backward compatibility exports
+ */
+export declare const pipeline: Pipeline<any>;
+export declare enum ExecutionMode {
+    SEQUENTIAL = "sequential",
+    PARALLEL = "parallel",
+    FIRST_SUCCESS = "firstSuccess",
+    FIRST_RESULT = "firstResult",
+    THREADED = "threaded"
+}
+export type PipelineResult<T> = {
+    success: boolean;
+    data: T;
+    error?: string;
+};
+export type StreamlinedPipelineOptions = PipelineOptions;
+export type StreamlinedPipelineResult<T> = PipelineResult<T>;
+export { ExecutionMode as StreamlinedExecutionMode };