@soulcraft/brainy 3.0.0 → 3.1.0

package/dist/storage/adapters/fileSystemStorage.js

@@ -433,7 +433,7 @@ export class FileSystemStorage extends BaseStorage {
     /**
      * Save noun metadata to storage
      */
-    async saveNounMetadata(id, metadata) {
+    async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
         const filePath = path.join(this.nounMetadataDir, `${id}.json`);
         await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
@@ -458,7 +458,7 @@ export class FileSystemStorage extends BaseStorage {
     /**
      * Save verb metadata to storage
      */
-    async saveVerbMetadata(id, metadata) {
+    async saveVerbMetadata_internal(id, metadata) {
         await this.ensureInitialized();
         const filePath = path.join(this.verbMetadataDir, `${id}.json`);
         await fs.promises.writeFile(filePath, JSON.stringify(metadata, null, 2));
@@ -770,28 +770,178 @@ export class FileSystemStorage extends BaseStorage {
      * Get verbs by source
      */
     async getVerbsBySource_internal(sourceId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsBySource_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        console.log(`[DEBUG] getVerbsBySource_internal called for sourceId: ${sourceId}`);
+        // Use the working pagination method with source filter
+        const result = await this.getVerbsWithPagination({
+            limit: 10000,
+            filter: { sourceId: [sourceId] }
+        });
+        console.log(`[DEBUG] Found ${result.items.length} verbs for source ${sourceId}`);
+        return result.items;
     }
     /**
      * Get verbs by target
      */
     async getVerbsByTarget_internal(targetId) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsByTarget_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        console.log(`[DEBUG] getVerbsByTarget_internal called for targetId: ${targetId}`);
+        // Use the working pagination method with target filter
+        const result = await this.getVerbsWithPagination({
+            limit: 10000,
+            filter: { targetId: [targetId] }
+        });
+        console.log(`[DEBUG] Found ${result.items.length} verbs for target ${targetId}`);
+        return result.items;
     }
     /**
      * Get verbs by type
      */
     async getVerbsByType_internal(type) {
-        // This method is deprecated and would require loading metadata for each edge
-        // For now, return empty array since this is not efficiently implementable with new storage pattern
-        console.warn('getVerbsByType_internal is deprecated and not efficiently supported in new storage pattern');
-        return [];
+        console.log(`[DEBUG] getVerbsByType_internal called for type: ${type}`);
+        // Use the working pagination method with type filter
+        const result = await this.getVerbsWithPagination({
+            limit: 10000,
+            filter: { verbType: [type] }
+        });
+        console.log(`[DEBUG] Found ${result.items.length} verbs for type ${type}`);
+        return result.items;
+    }
+    /**
+     * Get verbs with pagination
+     * This method reads verb files from the filesystem and returns them with pagination
+     */
+    async getVerbsWithPagination(options = {}) {
+        await this.ensureInitialized();
+        const limit = options.limit || 100;
+        const startIndex = options.cursor ? parseInt(options.cursor, 10) : 0;
+        try {
+            // List all verb files in the verbs directory
+            // Note: For very large directories (millions of files), this could be memory-intensive
+            // Future optimization: Use fs.opendir() for streaming directory reads
+            const files = await fs.promises.readdir(this.verbsDir);
+            const verbFiles = files.filter((f) => f.endsWith('.json'));
+            // Sort files for consistent ordering
+            verbFiles.sort();
+            // Calculate pagination
+            const totalCount = verbFiles.length;
+            const endIndex = Math.min(startIndex + limit, totalCount);
+            const hasMore = endIndex < totalCount;
+            // Safety check for large datasets
+            if (totalCount > 100000) {
+                console.warn(`Large verb dataset detected (${totalCount} verbs). Consider using a database for better performance.`);
+            }
+            // Load the requested page of verbs
+            const verbs = [];
+            for (let i = startIndex; i < endIndex; i++) {
+                const file = verbFiles[i];
+                const id = file.replace('.json', '');
+                try {
+                    // Read the verb data (HNSWVerb stored as edge)
+                    const filePath = path.join(this.verbsDir, file);
+                    const data = await fs.promises.readFile(filePath, 'utf-8');
+                    const edge = JSON.parse(data);
+                    // Get metadata which contains the actual verb information
+                    const metadata = await this.getVerbMetadata(id);
+                    // If no metadata exists, try to reconstruct basic metadata from filename
+                    if (!metadata) {
+                        console.warn(`Verb ${id} has no metadata, trying to create minimal verb`);
+                        // Create minimal GraphVerb without full metadata
+                        const minimalVerb = {
+                            id: edge.id,
+                            vector: edge.vector,
+                            connections: edge.connections || new Map(),
+                            sourceId: 'unknown',
+                            targetId: 'unknown',
+                            source: 'unknown',
+                            target: 'unknown',
+                            type: 'relationship',
+                            verb: 'relatedTo'
+                        };
+                        verbs.push(minimalVerb);
+                        continue;
+                    }
+                    // Convert connections Map to proper format if needed
+                    let connections = edge.connections;
+                    if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
+                        const connectionsMap = new Map();
+                        for (const [level, nodeIds] of Object.entries(connections)) {
+                            connectionsMap.set(Number(level), new Set(nodeIds));
+                        }
+                        connections = connectionsMap;
+                    }
+                    // Properly reconstruct GraphVerb from HNSWVerb + metadata
+                    const verb = {
+                        id: edge.id,
+                        vector: edge.vector, // Include the vector field!
+                        connections: connections,
+                        sourceId: metadata.sourceId || metadata.source,
+                        targetId: metadata.targetId || metadata.target,
+                        source: metadata.source || metadata.sourceId,
+                        target: metadata.target || metadata.targetId,
+                        verb: metadata.verb || metadata.type,
+                        type: metadata.type || metadata.verb,
+                        weight: metadata.weight,
+                        metadata: metadata.metadata || metadata,
+                        data: metadata.data,
+                        createdAt: metadata.createdAt,
+                        updatedAt: metadata.updatedAt,
+                        createdBy: metadata.createdBy,
+                        embedding: metadata.embedding || edge.vector
+                    };
+                    // Apply filters if provided
+                    if (options.filter) {
+                        const filter = options.filter;
+                        // Check verbType filter
+                        if (filter.verbType) {
+                            const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
+                            const verbType = verb.type || verb.verb;
+                            if (verbType && !types.includes(verbType))
+                                continue;
+                        }
+                        // Check sourceId filter
+                        if (filter.sourceId) {
+                            const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
+                            const sourceId = verb.sourceId || verb.source;
+                            if (!sourceId || !sources.includes(sourceId))
+                                continue;
+                        }
+                        // Check targetId filter
+                        if (filter.targetId) {
+                            const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
+                            const targetId = verb.targetId || verb.target;
+                            if (!targetId || !targets.includes(targetId))
+                                continue;
+                        }
+                        // Check service filter
+                        if (filter.service && metadata?.service) {
+                            const services = Array.isArray(filter.service) ? filter.service : [filter.service];
+                            if (!services.includes(metadata.service))
+                                continue;
+                        }
+                    }
+                    verbs.push(verb);
+                }
+                catch (error) {
+                    console.warn(`Failed to read verb ${id}:`, error);
+                }
+            }
+            return {
+                items: verbs,
+                totalCount,
+                hasMore,
+                nextCursor: hasMore ? String(endIndex) : undefined
+            };
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                // Verbs directory doesn't exist yet
+                return {
+                    items: [],
+                    totalCount: 0,
+                    hasMore: false
+                };
+            }
+            throw error;
+        }
     }
     /**
      * Delete a verb from storage
@@ -1030,7 +1180,60 @@ export class FileSystemStorage extends BaseStorage {
             }
         }
         // Merge statistics from both locations
-        return StorageCompatibilityLayer.mergeStatistics(newStats, oldStats);
+        return this.mergeStatistics(newStats, oldStats);
+    }
+    /**
+     * Merge statistics from multiple sources
+     */
+    mergeStatistics(storageStats, localStats) {
+        // Handle null cases
+        if (!storageStats && !localStats) {
+            return {
+                nounCount: {},
+                verbCount: {},
+                metadataCount: {},
+                hnswIndexSize: 0,
+                totalNodes: 0,
+                totalEdges: 0,
+                lastUpdated: new Date().toISOString()
+            };
+        }
+        if (!storageStats)
+            return localStats;
+        if (!localStats)
+            return storageStats;
+        // Merge noun counts by taking the maximum of each type
+        const mergedNounCount = {
+            ...storageStats.nounCount
+        };
+        for (const [type, count] of Object.entries(localStats.nounCount)) {
+            mergedNounCount[type] = Math.max(mergedNounCount[type] || 0, count);
+        }
+        // Merge verb counts by taking the maximum of each type
+        const mergedVerbCount = {
+            ...storageStats.verbCount
+        };
+        for (const [type, count] of Object.entries(localStats.verbCount)) {
+            mergedVerbCount[type] = Math.max(mergedVerbCount[type] || 0, count);
+        }
+        // Merge metadata counts by taking the maximum of each type
+        const mergedMetadataCount = {
+            ...storageStats.metadataCount
+        };
+        for (const [type, count] of Object.entries(localStats.metadataCount)) {
+            mergedMetadataCount[type] = Math.max(mergedMetadataCount[type] || 0, count);
+        }
+        return {
+            nounCount: mergedNounCount,
+            verbCount: mergedVerbCount,
+            metadataCount: mergedMetadataCount,
+            hnswIndexSize: Math.max(storageStats.hnswIndexSize || 0, localStats.hnswIndexSize || 0),
+            totalNodes: Math.max(storageStats.totalNodes || 0, localStats.totalNodes || 0),
+            totalEdges: Math.max(storageStats.totalEdges || 0, localStats.totalEdges || 0),
+            totalMetadata: Math.max(storageStats.totalMetadata || 0, localStats.totalMetadata || 0),
+            operations: storageStats.operations || localStats.operations,
+            lastUpdated: new Date().toISOString()
+        };
     }
 }
 //# sourceMappingURL=fileSystemStorage.js.map

package/dist/storage/adapters/memoryStorage.d.ts

@@ -131,17 +131,17 @@ export declare class MemoryStorage extends BaseStorage {
      */
     getMetadataBatch(ids: string[]): Promise<Map<string, any>>;
     /**
-     * Save noun metadata to storage
+     * Save noun metadata to storage (internal implementation)
      */
-    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get noun metadata from storage
      */
     getNounMetadata(id: string): Promise<any | null>;
     /**
-     * Save verb metadata to storage
+     * Save verb metadata to storage (internal implementation)
      */
-    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get verb metadata from storage
      */

package/dist/storage/adapters/memoryStorage.js

@@ -35,7 +35,8 @@ export class MemoryStorage extends BaseStorage {
             id: noun.id,
             vector: [...noun.vector],
             connections: new Map(),
-            level: noun.level || 0
+            level: noun.level || 0,
+            metadata: noun.metadata
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -59,7 +60,8 @@ export class MemoryStorage extends BaseStorage {
             id: noun.id,
             vector: [...noun.vector],
             connections: new Map(),
-            level: noun.level || 0
+            level: noun.level || 0,
+            metadata: noun.metadata
         };
         // Copy connections
         for (const [level, connections] of noun.connections.entries()) {
@@ -89,12 +91,14 @@ export class MemoryStorage extends BaseStorage {
         const matchingIds = [];
         // Iterate through all nouns to find matches
         for (const [nounId, noun] of this.nouns.entries()) {
-            // Get the metadata to check filters
-            const metadata = await this.getMetadata(nounId);
-            if (!metadata)
-                continue;
+            // Check the noun's embedded metadata field
+            const nounMetadata = noun.metadata || {};
+            // Also check separate metadata store for backward compatibility
+            const separateMetadata = await this.getMetadata(nounId);
+            // Merge both metadata sources (noun.metadata takes precedence)
+            const metadata = { ...separateMetadata, ...nounMetadata };
             // Filter by noun type if specified
-            if (nounTypes && !nounTypes.includes(metadata.noun)) {
+            if (nounTypes && metadata.noun && !nounTypes.includes(metadata.noun)) {
                 continue;
             }
             // Filter by service if specified
@@ -133,7 +137,8 @@ export class MemoryStorage extends BaseStorage {
                 id: noun.id,
                 vector: [...noun.vector],
                 connections: new Map(),
-                level: noun.level || 0
+                level: noun.level || 0,
+                metadata: noun.metadata
             };
             // Copy connections
             for (const [level, connections] of noun.connections.entries()) {
@@ -424,9 +429,9 @@ export class MemoryStorage extends BaseStorage {
         return results;
     }
     /**
-     * Save noun metadata to storage
+     * Save noun metadata to storage (internal implementation)
      */
-    async saveNounMetadata(id, metadata) {
+    async saveNounMetadata_internal(id, metadata) {
         this.nounMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
     }
     /**
@@ -440,9 +445,9 @@ export class MemoryStorage extends BaseStorage {
         return JSON.parse(JSON.stringify(metadata));
     }
     /**
-     * Save verb metadata to storage
+     * Save verb metadata to storage (internal implementation)
      */
-    async saveVerbMetadata(id, metadata) {
+    async saveVerbMetadata_internal(id, metadata) {
         this.verbMetadata.set(id, JSON.parse(JSON.stringify(metadata)));
     }
     /**

package/dist/storage/adapters/opfsStorage.d.ts

@@ -144,7 +144,7 @@ export declare class OPFSStorage extends BaseStorage {
     /**
      * Save verb metadata to storage
      */
-    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get verb metadata from storage
      */
@@ -152,7 +152,7 @@ export declare class OPFSStorage extends BaseStorage {
     /**
      * Save noun metadata to storage
      */
-    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get noun metadata from storage
      */

package/dist/storage/adapters/opfsStorage.js

@@ -540,7 +540,7 @@ export class OPFSStorage extends BaseStorage {
     /**
      * Save verb metadata to storage
      */
-    async saveVerbMetadata(id, metadata) {
+    async saveVerbMetadata_internal(id, metadata) {
         await this.ensureInitialized();
         const fileName = `${id}.json`;
         const fileHandle = await this.verbMetadataDir.getFileHandle(fileName, { create: true });
@@ -570,7 +570,7 @@ export class OPFSStorage extends BaseStorage {
     /**
      * Save noun metadata to storage
      */
-    async saveNounMetadata(id, metadata) {
+    async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
         const fileName = `${id}.json`;
         const fileHandle = await this.nounMetadataDir.getFileHandle(fileName, { create: true });

package/dist/storage/adapters/s3CompatibleStorage.d.ts

@@ -328,7 +328,7 @@ export declare class S3CompatibleStorage extends BaseStorage {
     /**
      * Save verb metadata to storage
      */
-    saveVerbMetadata(id: string, metadata: any): Promise<void>;
+    protected saveVerbMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get verb metadata from storage
      */
@@ -336,7 +336,7 @@ export declare class S3CompatibleStorage extends BaseStorage {
     /**
      * Save noun metadata to storage
      */
-    saveNounMetadata(id: string, metadata: any): Promise<void>;
+    protected saveNounMetadata_internal(id: string, metadata: any): Promise<void>;
     /**
      * Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
      * This is the solution to the metadata reading socket exhaustion during initialization

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -1450,7 +1450,7 @@ export class S3CompatibleStorage extends BaseStorage {
     /**
      * Save verb metadata to storage
      */
-    async saveVerbMetadata(id, metadata) {
+    async saveVerbMetadata_internal(id, metadata) {
         await this.ensureInitialized();
         try {
             // Import the PutObjectCommand only when needed
@@ -1523,7 +1523,7 @@ export class S3CompatibleStorage extends BaseStorage {
     /**
      * Save noun metadata to storage
      */
-    async saveNounMetadata(id, metadata) {
+    async saveNounMetadata_internal(id, metadata) {
         await this.ensureInitialized();
         try {
             // Import the PutObjectCommand only when needed

package/dist/storage/backwardCompatibility.d.ts

@@ -1,84 +1,16 @@
 /**
- * Backward Compatibility Layer for Storage Migration
- *
- * Handles the transition from 'index' to '_system' directory
- * Ensures services running different versions can coexist
- */
-import { StatisticsData } from '../coreTypes.js';
-export interface MigrationMetadata {
-    schemaVersion: number;
-    migrationStarted?: string;
-    migrationCompleted?: string;
-    lastUpdatedBy?: string;
-}
-/**
- * Backward compatibility strategy for directory migration
+ * Storage backward compatibility layer for legacy data migrations
  */
 export declare class StorageCompatibilityLayer {
-    private migrationMetadata;
-    /**
-     * Determines the read strategy based on what's available
-     * @returns Priority-ordered list of directories to try
-     */
-    static getReadPriority(): string[];
-    /**
-     * Determines write strategy based on migration state
-     * @param migrationComplete Whether migration is complete
-     * @returns List of directories to write to
-     */
-    static getWriteTargets(migrationComplete?: boolean): string[];
-    /**
-     * Check if we should perform migration based on service coordination
-     * @param existingStats Statistics from storage
-     * @returns Whether to initiate migration
-     */
-    static shouldMigrate(existingStats: StatisticsData | null): boolean;
-    /**
-     * Creates migration metadata
-     */
-    static createMigrationMetadata(): MigrationMetadata;
-    /**
-     * Merge statistics from multiple locations (deduplication)
-     */
-    static mergeStatistics(primary: StatisticsData | null, fallback: StatisticsData | null): StatisticsData | null;
-    /**
-     * Determines if dual-write is needed based on environment
-     * @param storageType The type of storage being used
-     * @returns Whether to write to both old and new locations
-     */
-    static needsDualWrite(storageType: string): boolean;
-    /**
-     * Grace period for migration (30 days default)
-     * After this period, services can stop reading from old location
-     */
-    static getMigrationGracePeriodMs(): number;
-    /**
-     * Check if migration grace period has expired
-     */
-    static isGracePeriodExpired(migrationStarted: string): boolean;
-    /**
-     * Log migration events for monitoring
-     */
     static logMigrationEvent(event: string, details?: any): void;
+    static migrateIfNeeded(storagePath: string): Promise<void>;
 }
-/**
- * Storage paths helper for migration
- */
-export declare class StoragePaths {
-    /**
-     * Get the statistics file path for a given directory
-     */
-    static getStatisticsPath(baseDir: string, filename?: string): string;
-    /**
-     * Get distributed config path
-     */
-    static getDistributedConfigPath(baseDir: string): string;
-    /**
-     * Check if a path is using the old structure
-     */
-    static isLegacyPath(path: string): boolean;
-    /**
-     * Convert legacy path to new structure
-     */
-    static modernizePath(path: string): string;
+export interface StoragePaths {
+    nouns: string;
+    verbs: string;
+    metadata: string;
+    index: string;
+    system: string;
+    statistics: string;
 }
+export declare function getDefaultStoragePaths(basePath: string): StoragePaths;