@soulcraft/brainy 4.10.3 → 4.11.0

package/dist/import/ImportCoordinator.js

@@ -499,17 +499,114 @@ export class ImportCoordinator {
             });
             console.log(`✅ Document entity created: ${documentEntityId}`);
         }
-        // Create entities in graph
-        for (const row of rows) {
-            const entity = row.entity || row;
-            // Find corresponding VFS file
-            const vfsFile = vfsResult.files.find((f) => f.entityId === entity.id);
-            // Create or merge entity
-            try {
-                const importSource = vfsResult.rootPath;
-                let entityId;
-                let wasMerged = false;
-                if (actuallyEnableDeduplication) {
+        // ============================================
+        // v4.11.0: Batch entity creation using addMany()
+        // Replaces entity-by-entity loop for 10-100x performance improvement on cloud storage
+        // ============================================
+        if (!actuallyEnableDeduplication) {
+            // FAST PATH: Batch creation without deduplication (recommended for imports > 100 entities)
+            const importSource = vfsResult.rootPath;
+            // Prepare all entity parameters upfront
+            const entityParams = rows.map((row) => {
+                const entity = row.entity || row;
+                const vfsFile = vfsResult.files.find((f) => f.entityId === entity.id);
+                return {
+                    data: entity.description || entity.name,
+                    type: entity.type,
+                    metadata: {
+                        ...entity.metadata,
+                        name: entity.name,
+                        confidence: entity.confidence,
+                        vfsPath: vfsFile?.path,
+                        importedFrom: 'import-coordinator',
+                        imports: [importSource],
+                        ...(trackingContext && {
+                            importIds: [trackingContext.importId],
+                            projectId: trackingContext.projectId,
+                            importedAt: trackingContext.importedAt,
+                            importFormat: trackingContext.importFormat,
+                            importSource: trackingContext.importSource,
+                            sourceRow: row.rowNumber,
+                            sourceSheet: row.sheet,
+                            ...trackingContext.customMetadata
+                        })
+                    }
+                };
+            });
+            // Batch create all entities (storage-aware batching handles rate limits automatically)
+            const addResult = await this.brain.addMany({
+                items: entityParams,
+                continueOnError: true,
+                onProgress: (done, total) => {
+                    options.onProgress?.({
+                        stage: 'storing-graph',
+                        message: `Creating entities: ${done}/${total}`,
+                        processed: done,
+                        total,
+                        entities: done
+                    });
+                }
+            });
+            // Map results to entities array and update rows with new IDs
+            for (let i = 0; i < addResult.successful.length; i++) {
+                const entityId = addResult.successful[i];
+                const row = rows[i];
+                const entity = row.entity || row;
+                const vfsFile = vfsResult.files.find((f) => f.entityId === entity.id);
+                entity.id = entityId;
+                entities.push({
+                    id: entityId,
+                    name: entity.name,
+                    type: entity.type,
+                    vfsPath: vfsFile?.path
+                });
+                newCount++;
+            }
+            // Handle failed entities
+            if (addResult.failed.length > 0) {
+                console.warn(`⚠️  ${addResult.failed.length} entities failed to create`);
+            }
+            // Create provenance links in batch
+            if (documentEntityId && options.createProvenanceLinks !== false && entities.length > 0) {
+                const provenanceParams = entities.map((entity, idx) => {
+                    const row = rows[idx];
+                    return {
+                        from: documentEntityId,
+                        to: entity.id,
+                        type: VerbType.Contains,
+                        metadata: {
+                            relationshipType: 'provenance',
+                            evidence: `Extracted from ${sourceInfo?.sourceFilename}`,
+                            sheet: row?.sheet,
+                            rowNumber: row?.rowNumber,
+                            extractedAt: Date.now(),
+                            format: sourceInfo?.format,
+                            ...(trackingContext && {
+                                importIds: [trackingContext.importId],
+                                projectId: trackingContext.projectId,
+                                createdAt: Date.now(),
+                                importFormat: trackingContext.importFormat,
+                                ...trackingContext.customMetadata
+                            })
+                        }
+                    };
+                });
+                await this.brain.relateMany({
+                    items: provenanceParams,
+                    continueOnError: true
+                });
+                provenanceCount = provenanceParams.length;
+            }
+        }
+        else {
+            // SLOW PATH: Entity-by-entity with deduplication (only for small imports < 100 entities)
+            for (const row of rows) {
+                const entity = row.entity || row;
+                const vfsFile = vfsResult.files.find((f) => f.entityId === entity.id);
+                try {
+                    const importSource = vfsResult.rootPath;
+                    let entityId;
+                    let wasMerged = false;
                     // Use deduplicator to check for existing entities
                     const mergeResult = await this.deduplicator.createOrMerge({
                         id: entity.id,
@@ -546,181 +643,154 @@ export class ImportCoordinator {
                     else {
                         newCount++;
                     }
-                }
-                else {
-                    // Direct creation without deduplication
-                    entityId = await this.brain.add({
-                        data: entity.description || entity.name,
+                    // Update entity ID in extraction result
+                    entity.id = entityId;
+                    entities.push({
+                        id: entityId,
+                        name: entity.name,
                         type: entity.type,
-                        metadata: {
-                            ...entity.metadata,
-                            name: entity.name,
-                            confidence: entity.confidence,
-                            vfsPath: vfsFile?.path,
-                            importedFrom: 'import-coordinator',
-                            imports: [importSource],
-                            // v4.10.0: Import tracking metadata
-                            ...(trackingContext && {
-                                importIds: [trackingContext.importId],
-                                projectId: trackingContext.projectId,
-                                importedAt: trackingContext.importedAt,
-                                importFormat: trackingContext.importFormat,
-                                importSource: trackingContext.importSource,
-                                sourceRow: row.rowNumber,
-                                sourceSheet: row.sheet,
-                                ...trackingContext.customMetadata
-                            })
-                        }
+                        vfsPath: vfsFile?.path
                     });
-                    newCount++;
-                }
-                // Update entity ID in extraction result
-                entity.id = entityId;
-                entities.push({
-                    id: entityId,
-                    name: entity.name,
-                    type: entity.type,
-                    vfsPath: vfsFile?.path
-                });
-                // ============================================
-                // v4.9.0: Create provenance relationship (document → entity)
-                // ============================================
-                if (documentEntityId && options.createProvenanceLinks !== false) {
-                    await this.brain.relate({
-                        from: documentEntityId,
-                        to: entityId,
-                        type: VerbType.Contains,
-                        metadata: {
-                            relationshipType: 'provenance',
-                            evidence: `Extracted from ${sourceInfo?.sourceFilename}`,
-                            sheet: row.sheet,
-                            rowNumber: row.rowNumber,
-                            extractedAt: Date.now(),
-                            format: sourceInfo?.format,
-                            // v4.10.0: Import tracking metadata
-                            ...(trackingContext && {
-                                importIds: [trackingContext.importId],
-                                projectId: trackingContext.projectId,
-                                createdAt: Date.now(),
-                                importFormat: trackingContext.importFormat,
-                                ...trackingContext.customMetadata
-                            })
-                        }
-                    });
-                    provenanceCount++;
-                }
-                // Collect relationships for batch creation
-                if (options.createRelationships && row.relationships) {
-                    for (const rel of row.relationships) {
-                        try {
-                            // CRITICAL FIX (v3.43.2): Prevent infinite placeholder creation loop
-                            // Find or create target entity using EXACT matching only
-                            let targetEntityId;
-                            // STEP 1: Check if target already exists in entities list (includes placeholders)
-                            // This prevents creating duplicate placeholders - the root cause of Bug #1
-                            const existingTarget = entities.find(e => e.name.toLowerCase() === rel.to.toLowerCase());
-                            if (existingTarget) {
-                                targetEntityId = existingTarget.id;
+                    // ============================================
+                    // v4.9.0: Create provenance relationship (document → entity)
+                    // ============================================
+                    if (documentEntityId && options.createProvenanceLinks !== false) {
+                        await this.brain.relate({
+                            from: documentEntityId,
+                            to: entityId,
+                            type: VerbType.Contains,
+                            metadata: {
+                                relationshipType: 'provenance',
+                                evidence: `Extracted from ${sourceInfo?.sourceFilename}`,
+                                sheet: row.sheet,
+                                rowNumber: row.rowNumber,
+                                extractedAt: Date.now(),
+                                format: sourceInfo?.format,
+                                // v4.10.0: Import tracking metadata
+                                ...(trackingContext && {
+                                    importIds: [trackingContext.importId],
+                                    projectId: trackingContext.projectId,
+                                    createdAt: Date.now(),
+                                    importFormat: trackingContext.importFormat,
+                                    ...trackingContext.customMetadata
+                                })
                             }
-                            else {
-                                // STEP 2: Try to find in extraction results (rows)
-                                // FIX: Use EXACT matching instead of fuzzy .includes()
-                                // Fuzzy matching caused false matches (e.g., "Entity_29" matching "Entity_297")
-                                for (const otherRow of rows) {
-                                    const otherEntity = otherRow.entity || otherRow;
-                                    if (otherEntity.name.toLowerCase() === rel.to.toLowerCase()) {
-                                        targetEntityId = otherEntity.id;
-                                        break;
-                                    }
+                        });
+                        provenanceCount++;
+                    }
+                    // Collect relationships for batch creation
+                    if (options.createRelationships && row.relationships) {
+                        for (const rel of row.relationships) {
+                            try {
+                                // CRITICAL FIX (v3.43.2): Prevent infinite placeholder creation loop
+                                // Find or create target entity using EXACT matching only
+                                let targetEntityId;
+                                // STEP 1: Check if target already exists in entities list (includes placeholders)
+                                // This prevents creating duplicate placeholders - the root cause of Bug #1
+                                const existingTarget = entities.find(e => e.name.toLowerCase() === rel.to.toLowerCase());
+                                if (existingTarget) {
+                                    targetEntityId = existingTarget.id;
                                 }
-                                // STEP 3: If still not found, create placeholder entity ONCE
-                                // The placeholder is added to entities array, so future searches will find it
-                                if (!targetEntityId) {
-                                    targetEntityId = await this.brain.add({
-                                        data: rel.to,
-                                        type: NounType.Thing,
-                                        metadata: {
-                                            name: rel.to,
-                                            placeholder: true,
-                                            inferredFrom: entity.name,
-                                            // v4.10.0: Import tracking metadata
-                                            ...(trackingContext && {
-                                                importIds: [trackingContext.importId],
-                                                projectId: trackingContext.projectId,
-                                                importedAt: trackingContext.importedAt,
-                                                importFormat: trackingContext.importFormat,
-                                                ...trackingContext.customMetadata
-                                            })
+                                else {
+                                    // STEP 2: Try to find in extraction results (rows)
+                                    // FIX: Use EXACT matching instead of fuzzy .includes()
+                                    // Fuzzy matching caused false matches (e.g., "Entity_29" matching "Entity_297")
+                                    for (const otherRow of rows) {
+                                        const otherEntity = otherRow.entity || otherRow;
+                                        if (otherEntity.name.toLowerCase() === rel.to.toLowerCase()) {
+                                            targetEntityId = otherEntity.id;
+                                            break;
                                         }
-                                    });
-                                    // CRITICAL: Add to entities array so future searches find it
-                                    entities.push({
-                                        id: targetEntityId,
-                                        name: rel.to,
-                                        type: NounType.Thing
-                                    });
+                                    }
+                                    // STEP 3: If still not found, create placeholder entity ONCE
+                                    // The placeholder is added to entities array, so future searches will find it
+                                    if (!targetEntityId) {
+                                        targetEntityId = await this.brain.add({
+                                            data: rel.to,
+                                            type: NounType.Thing,
+                                            metadata: {
+                                                name: rel.to,
+                                                placeholder: true,
+                                                inferredFrom: entity.name,
+                                                // v4.10.0: Import tracking metadata
+                                                ...(trackingContext && {
+                                                    importIds: [trackingContext.importId],
+                                                    projectId: trackingContext.projectId,
+                                                    importedAt: trackingContext.importedAt,
+                                                    importFormat: trackingContext.importFormat,
+                                                    ...trackingContext.customMetadata
+                                                })
+                                            }
+                                        });
+                                        // CRITICAL: Add to entities array so future searches find it
+                                        entities.push({
+                                            id: targetEntityId,
+                                            name: rel.to,
+                                            type: NounType.Thing
+                                        });
+                                    }
                                 }
+                                // Add to relationships array with target ID for batch processing
+                                relationships.push({
+                                    id: '', // Will be assigned after batch creation
+                                    from: entityId,
+                                    to: targetEntityId,
+                                    type: rel.type,
+                                    confidence: rel.confidence, // v4.2.0: Top-level field
+                                    weight: rel.weight || 1.0, // v4.2.0: Top-level field
+                                    metadata: {
+                                        evidence: rel.evidence,
+                                        // v4.10.0: Import tracking metadata (will be merged in batch creation)
+                                        ...(trackingContext && {
+                                            importIds: [trackingContext.importId],
+                                            projectId: trackingContext.projectId,
+                                            importedAt: trackingContext.importedAt,
+                                            importFormat: trackingContext.importFormat,
+                                            ...trackingContext.customMetadata
+                                        })
+                                    }
+                                });
+                            }
+                            catch (error) {
+                                // Skip relationship collection errors (entity might not exist, etc.)
+                                continue;
                             }
-                            // Add to relationships array with target ID for batch processing
-                            relationships.push({
-                                id: '', // Will be assigned after batch creation
-                                from: entityId,
-                                to: targetEntityId,
-                                type: rel.type,
-                                confidence: rel.confidence, // v4.2.0: Top-level field
-                                weight: rel.weight || 1.0, // v4.2.0: Top-level field
-                                metadata: {
-                                    evidence: rel.evidence,
-                                    // v4.10.0: Import tracking metadata (will be merged in batch creation)
-                                    ...(trackingContext && {
-                                        importIds: [trackingContext.importId],
-                                        projectId: trackingContext.projectId,
-                                        importedAt: trackingContext.importedAt,
-                                        importFormat: trackingContext.importFormat,
-                                        ...trackingContext.customMetadata
-                                    })
-                                }
-                            });
                         }
-                        catch (error) {
-                            // Skip relationship collection errors (entity might not exist, etc.)
-                            continue;
+                    }
+                    // Streaming import: Progressive flush with dynamic interval adjustment (v4.2.0+)
+                    entitiesSinceFlush++;
+                    if (entitiesSinceFlush >= currentFlushInterval) {
+                        const flushStart = Date.now();
+                        await this.brain.flush();
+                        const flushDuration = Date.now() - flushStart;
+                        totalFlushes++;
+                        // Reset counter
+                        entitiesSinceFlush = 0;
+                        // Recalculate flush interval based on current entity count
+                        const newInterval = this.getProgressiveFlushInterval(entities.length);
+                        if (newInterval !== currentFlushInterval) {
+                            console.log(`📊 Flush interval adjusted: ${currentFlushInterval} → ${newInterval}\n` +
+                                `   Reason: Reached ${entities.length} entities (threshold for next tier)\n` +
+                                `   Impact: ${newInterval > currentFlushInterval ? 'Fewer' : 'More'} flushes = ${newInterval > currentFlushInterval ? 'Better performance' : 'More frequent updates'}`);
+                            currentFlushInterval = newInterval;
                         }
+                        // Notify progress callback that data is now queryable
+                        await options.onProgress?.({
+                            stage: 'storing-graph',
+                            message: `Flushed indexes (${entities.length}/${rows.length} entities, ${flushDuration}ms)`,
+                            processed: entities.length,
+                            total: rows.length,
+                            entities: entities.length,
+                            queryable: true // ← Indexes are flushed, data is queryable!
+                        });
                     }
                 }
-                // Streaming import: Progressive flush with dynamic interval adjustment (v4.2.0+)
-                entitiesSinceFlush++;
-                if (entitiesSinceFlush >= currentFlushInterval) {
-                    const flushStart = Date.now();
-                    await this.brain.flush();
-                    const flushDuration = Date.now() - flushStart;
-                    totalFlushes++;
-                    // Reset counter
-                    entitiesSinceFlush = 0;
-                    // Recalculate flush interval based on current entity count
-                    const newInterval = this.getProgressiveFlushInterval(entities.length);
-                    if (newInterval !== currentFlushInterval) {
-                        console.log(`📊 Flush interval adjusted: ${currentFlushInterval} → ${newInterval}\n` +
-                            `   Reason: Reached ${entities.length} entities (threshold for next tier)\n` +
-                            `   Impact: ${newInterval > currentFlushInterval ? 'Fewer' : 'More'} flushes = ${newInterval > currentFlushInterval ? 'Better performance' : 'More frequent updates'}`);
-                        currentFlushInterval = newInterval;
-                    }
-                    // Notify progress callback that data is now queryable
-                    await options.onProgress?.({
-                        stage: 'storing-graph',
-                        message: `Flushed indexes (${entities.length}/${rows.length} entities, ${flushDuration}ms)`,
-                        processed: entities.length,
-                        total: rows.length,
-                        entities: entities.length,
-                        queryable: true // ← Indexes are flushed, data is queryable!
-                    });
+                catch (error) {
+                    // Skip entity creation errors (might already exist, etc.)
+                    continue;
                 }
             }
-            catch (error) {
-                // Skip entity creation errors (might already exist, etc.)
-                continue;
-            }
-        }
+        } // End of deduplication else block
         // Final flush for any remaining entities
         if (entitiesSinceFlush > 0) {
             const flushStart = Date.now();

package/dist/storage/adapters/azureBlobStorage.d.ts

@@ -12,7 +12,7 @@
  * v4.0.0: Fully compatible with metadata/vector separation architecture
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
 /**
@@ -70,6 +70,20 @@ export declare class AzureBlobStorage extends BaseStorage {
         };
         readOnly?: boolean;
     });
+    /**
+     * Get Azure Blob-optimized batch configuration
+     *
+     * Azure Blob Storage has moderate rate limits between GCS and S3:
+     * - Medium batch sizes (75 items)
+     * - Parallel processing supported
+     * - Moderate delays (75ms)
+     *
+     * Azure can handle ~2000 operations/second with good performance
+     *
+     * @returns Azure Blob-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/azureBlobStorage.js

@@ -83,6 +83,31 @@ export class AzureBlobStorage extends BaseStorage {
             prodLog.info('🚀 High-volume mode FORCED via BRAINY_FORCE_HIGH_VOLUME environment variable');
         }
     }
+    /**
+     * Get Azure Blob-optimized batch configuration
+     *
+     * Azure Blob Storage has moderate rate limits between GCS and S3:
+     * - Medium batch sizes (75 items)
+     * - Parallel processing supported
+     * - Moderate delays (75ms)
+     *
+     * Azure can handle ~2000 operations/second with good performance
+     *
+     * @returns Azure Blob-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 75,
+            batchDelayMs: 75,
+            maxConcurrent: 75,
+            supportsParallelWrites: true, // Azure handles parallel reasonably
+            rateLimit: {
+                operationsPerSecond: 2000, // Moderate limits
+                burstCapacity: 500
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/baseStorageAdapter.d.ts

@@ -3,6 +3,7 @@
  * Provides common functionality for all storage adapters, including statistics tracking
  */
 import { StatisticsData, StorageAdapter, HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, NounMetadata, VerbMetadata } from '../../coreTypes.js';
+import { StorageBatchConfig } from '../baseStorage.js';
 /**
  * Base class for storage adapters that implements statistics tracking
  */
@@ -50,6 +51,18 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
         quota: number | null;
         details?: Record<string, any>;
     }>;
+    /**
+     * Get optimal batch configuration for this storage adapter
+     * Override in subclasses to provide storage-specific optimization
+     *
+     * This method allows each storage adapter to declare its optimal batch behavior
+     * for rate limiting and performance. The configuration is used by addMany(),
+     * relateMany(), and import operations to automatically adapt to storage capabilities.
+     *
+     * @returns Batch configuration optimized for this storage type
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Get nouns with pagination and filtering
      * @param options Pagination and filtering options

package/dist/storage/adapters/baseStorageAdapter.js

@@ -60,6 +60,32 @@ export class BaseStorageAdapter {
         this.countPersistBatchSize = 10; // Operations before forcing persist (cloud storage)
         this.countPersistInterval = 5000; // Milliseconds before forcing persist (cloud storage)
     }
+    /**
+     * Get optimal batch configuration for this storage adapter
+     * Override in subclasses to provide storage-specific optimization
+     *
+     * This method allows each storage adapter to declare its optimal batch behavior
+     * for rate limiting and performance. The configuration is used by addMany(),
+     * relateMany(), and import operations to automatically adapt to storage capabilities.
+     *
+     * @returns Batch configuration optimized for this storage type
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        // Conservative defaults that work safely across all storage types
+        // Cloud storage adapters should override with higher throughput values
+        // Local storage adapters should override with no delays
+        return {
+            maxBatchSize: 50,
+            batchDelayMs: 100,
+            maxConcurrent: 50,
+            supportsParallelWrites: false,
+            rateLimit: {
+                operationsPerSecond: 100,
+                burstCapacity: 200
+            }
+        };
+    }
     /**
      * Save statistics data
      * @param statistics The statistics data to save

package/dist/storage/adapters/fileSystemStorage.d.ts

@@ -3,7 +3,7 @@
  * File system storage adapter for Node.js environments
  */
 import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
-import { BaseStorage } from '../baseStorage.js';
+import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
 type HNSWNode = HNSWNoun;
 type Edge = HNSWVerb;
 /**
@@ -39,6 +39,19 @@ export declare class FileSystemStorage extends BaseStorage {
         compression?: boolean;
         compressionLevel?: number;
     });
+    /**
+     * Get FileSystem-optimized batch configuration
+     *
+     * File system storage is I/O bound but not rate limited:
+     * - Large batch sizes (500 items)
+     * - No delays needed (0ms)
+     * - Moderate concurrency (100 operations) - limited by I/O threads
+     * - Parallel processing supported
+     *
+     * @returns FileSystem-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig(): StorageBatchConfig;
     /**
      * Initialize the storage adapter
      */

package/dist/storage/adapters/fileSystemStorage.js

@@ -70,6 +70,30 @@ export class FileSystemStorage extends BaseStorage {
         }
         // Defer path operations until init() when path module is guaranteed to be loaded
     }
+    /**
+     * Get FileSystem-optimized batch configuration
+     *
+     * File system storage is I/O bound but not rate limited:
+     * - Large batch sizes (500 items)
+     * - No delays needed (0ms)
+     * - Moderate concurrency (100 operations) - limited by I/O threads
+     * - Parallel processing supported
+     *
+     * @returns FileSystem-optimized batch configuration
+     * @since v4.11.0
+     */
+    getBatchConfig() {
+        return {
+            maxBatchSize: 500,
+            batchDelayMs: 0,
+            maxConcurrent: 100,
+            supportsParallelWrites: true, // Filesystem handles parallel I/O
+            rateLimit: {
+                operationsPerSecond: 5000, // Depends on disk speed
+                burstCapacity: 2000
+            }
+        };
+    }
     /**
      * Initialize the storage adapter
      */