@soulcraft/brainy 3.48.0 → 3.49.0

package/dist/api/UniversalImportAPI.d.ts

@@ -46,6 +46,14 @@ export interface NeuralImportResult {
         processingTimeMs: number;
     };
 }
+export interface NeuralImportProgress {
+    phase: 'extracting' | 'storing-entities' | 'storing-relationships' | 'complete';
+    message: string;
+    current: number;
+    total: number;
+    entities?: number;
+    relationships?: number;
+}
 export declare class UniversalImportAPI {
     private brain;
     private typeMatcher;
@@ -60,7 +68,9 @@ export declare class UniversalImportAPI {
      * Universal import - handles ANY data source
      * ALWAYS uses neural matching, NEVER falls back
      */
-    import(source: ImportSource | string | any): Promise<NeuralImportResult>;
+    import(source: ImportSource | string | any, options?: {
+        onProgress?: (progress: NeuralImportProgress) => void;
+    }): Promise<NeuralImportResult>;
     /**
      * Import from URL - fetches and processes
      */

package/dist/api/UniversalImportAPI.js

@@ -34,17 +34,31 @@ export class UniversalImportAPI {
      * Universal import - handles ANY data source
      * ALWAYS uses neural matching, NEVER falls back
      */
-    async import(source) {
+    async import(source, options) {
         const startTime = Date.now();
         // Normalize source
         const normalizedSource = this.normalizeSource(source);
+        options?.onProgress?.({
+            phase: 'extracting',
+            message: 'Extracting data from source...',
+            current: 0,
+            total: 0
+        });
         // Extract data based on source type
         const extractedData = await this.extractData(normalizedSource);
         // Neural processing - MANDATORY
         const neuralResults = await this.neuralProcess(extractedData);
         // Store in brain
-        const result = await this.storeInBrain(neuralResults);
+        const result = await this.storeInBrain(neuralResults, options?.onProgress);
         result.stats.processingTimeMs = Date.now() - startTime;
+        options?.onProgress?.({
+            phase: 'complete',
+            message: 'Import complete',
+            current: result.stats.entitiesCreated + result.stats.relationshipsCreated,
+            total: result.stats.totalProcessed,
+            entities: result.stats.entitiesCreated,
+            relationships: result.stats.relationshipsCreated
+        });
         return result;
     }
     /**
@@ -399,7 +413,7 @@ export class UniversalImportAPI {
     /**
      * Store processed data in brain
      */
-    async storeInBrain(neuralResults) {
+    async storeInBrain(neuralResults, onProgress) {
         const result = {
             entities: [],
             relationships: [],
@@ -413,6 +427,13 @@ export class UniversalImportAPI {
         };
         let totalConfidence = 0;
         // Store entities
+        onProgress?.({
+            phase: 'storing-entities',
+            message: 'Storing entities...',
+            current: 0,
+            total: neuralResults.entities.size
+        });
+        let entitiesProcessed = 0;
         for (const entity of neuralResults.entities.values()) {
             const id = await this.brain.add({
                 data: entity.data,
@@ -428,30 +449,78 @@ export class UniversalImportAPI {
             });
             result.stats.entitiesCreated++;
             totalConfidence += entity.confidence;
+            entitiesProcessed++;
+            // Report progress periodically
+            if (entitiesProcessed % 10 === 0 || entitiesProcessed === neuralResults.entities.size) {
+                onProgress?.({
+                    phase: 'storing-entities',
+                    message: `Storing entities: ${entitiesProcessed}/${neuralResults.entities.size}`,
+                    current: entitiesProcessed,
+                    total: neuralResults.entities.size,
+                    entities: entitiesProcessed
+                });
+            }
         }
-        // Store relationships
-        for (const relation of neuralResults.relationships.values()) {
-            // Map to actual entity IDs
-            const sourceEntity = Array.from(neuralResults.entities.values())
-                .find(e => e.id === relation.from);
-            const targetEntity = Array.from(neuralResults.entities.values())
-                .find(e => e.id === relation.to);
-            if (sourceEntity && targetEntity) {
-                const id = await this.brain.relate({
-                    from: sourceEntity.id,
-                    to: targetEntity.id,
-                    type: relation.type,
-                    weight: relation.weight,
-                    metadata: relation.metadata
+        // Store relationships using batch processing
+        if (neuralResults.relationships.size > 0) {
+            onProgress?.({
+                phase: 'storing-relationships',
+                message: 'Preparing relationships...',
+                current: 0,
+                total: neuralResults.relationships.size
+            });
+            // Collect all relationship parameters
+            const relationshipParams = [];
+            for (const relation of neuralResults.relationships.values()) {
+                // Map to actual entity IDs
+                const sourceEntity = Array.from(neuralResults.entities.values())
+                    .find(e => e.id === relation.from);
+                const targetEntity = Array.from(neuralResults.entities.values())
+                    .find(e => e.id === relation.to);
+                if (sourceEntity && targetEntity) {
+                    relationshipParams.push({
+                        from: sourceEntity.id,
+                        to: targetEntity.id,
+                        type: relation.type,
+                        weight: relation.weight,
+                        metadata: relation.metadata
+                    });
+                    totalConfidence += relation.confidence;
+                }
+            }
+            // Batch create relationships with progress
+            if (relationshipParams.length > 0) {
+                const relationshipIds = await this.brain.relateMany({
+                    items: relationshipParams,
+                    parallel: true,
+                    chunkSize: 100,
+                    continueOnError: true,
+                    onProgress: (done, total) => {
+                        onProgress?.({
+                            phase: 'storing-relationships',
+                            message: `Building relationships: ${done}/${total}`,
+                            current: done,
+                            total: total,
+                            entities: result.stats.entitiesCreated,
+                            relationships: done
+                        });
+                    }
                 });
-                result.relationships.push({
-                    ...relation,
-                    id,
-                    from: sourceEntity.id,
-                    to: targetEntity.id
+                // Map results back
+                relationshipIds.forEach((id, index) => {
+                    if (id && relationshipParams[index]) {
+                        result.relationships.push({
+                            id,
+                            from: relationshipParams[index].from,
+                            to: relationshipParams[index].to,
+                            type: relationshipParams[index].type,
+                            weight: relationshipParams[index].weight || 1,
+                            confidence: 0.5, // Default confidence
+                            metadata: relationshipParams[index].metadata
+                        });
+                    }
                 });
-                result.stats.relationshipsCreated++;
-                totalConfidence += relation.confidence;
+                result.stats.relationshipsCreated = relationshipIds.length;
             }
         }
         // Calculate average confidence

package/dist/brainy.d.ts

@@ -698,12 +698,16 @@ export declare class Brainy<T = any> implements BrainyInterface<T> {
         enableConceptExtraction?: boolean;
         confidenceThreshold?: number;
         onProgress?: (progress: {
-            stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete';
+            stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete';
+            phase?: 'extraction' | 'relationships';
             message: string;
             processed?: number;
+            current?: number;
             total?: number;
             entities?: number;
             relationships?: number;
+            throughput?: number;
+            eta?: number;
         }) => void;
     }): Promise<import("./import/ImportCoordinator.js").ImportResult>;
     /**

package/dist/import/ImportCoordinator.d.ts

@@ -56,9 +56,13 @@ export interface ImportOptions {
     onProgress?: (progress: ImportProgress) => void;
 }
 export interface ImportProgress {
-    stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete';
+    stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete';
+    /** Phase of import - extraction or relationship building (v3.49.0) */
+    phase?: 'extraction' | 'relationships';
     message: string;
     processed?: number;
+    /** Alias for processed, used in relationship phase (v3.49.0) */
+    current?: number;
     total?: number;
     entities?: number;
     relationships?: number;

package/dist/import/ImportCoordinator.js

@@ -460,7 +460,19 @@ export class ImportCoordinator {
                     items: relationshipParams,
                     parallel: true,
                     chunkSize: 100,
-                    continueOnError: true
+                    continueOnError: true,
+                    onProgress: (done, total) => {
+                        options.onProgress?.({
+                            stage: 'storing-graph',
+                            phase: 'relationships',
+                            message: `Building relationships: ${done}/${total}`,
+                            current: done,
+                            processed: done,
+                            total: total,
+                            entities: entities.length,
+                            relationships: done
+                        });
+                    }
                 });
                 // Update relationship IDs
                 relationshipIds.forEach((id, index) => {

package/dist/importers/SmartImportOrchestrator.d.ts

@@ -29,7 +29,7 @@ export interface SmartImportOptions extends SmartExcelOptions {
     filename?: string;
 }
 export interface SmartImportProgress {
-    phase: 'parsing' | 'extracting' | 'creating' | 'organizing' | 'complete';
+    phase: 'parsing' | 'extracting' | 'creating' | 'relationships' | 'organizing' | 'complete';
     message: string;
     processed: number;
     total: number;

package/dist/importers/SmartImportOrchestrator.js

@@ -129,7 +129,7 @@ export class SmartImportOrchestrator {
             if (options.createRelationships !== false && options.createEntities !== false) {
                 onProgress?.({
                     phase: 'creating',
-                    message: 'Creating relationships...',
+                    message: 'Preparing relationships...',
                     processed: 0,
                     total: result.extraction.rows.length,
                     entities: result.entityIds.length,
@@ -140,7 +140,8 @@ export class SmartImportOrchestrator {
                 for (const extracted of result.extraction.rows) {
                     entityMap.set(extracted.entity.name.toLowerCase(), extracted.entity.id);
                 }
-                // Create relationships
+                // Collect all relationship parameters
+                const relationshipParams = [];
                 for (const extracted of result.extraction.rows) {
                     for (const rel of extracted.relationships) {
                         try {
@@ -167,8 +168,8 @@ export class SmartImportOrchestrator {
                                 });
                                 result.entityIds.push(toEntityId);
                             }
-                            // Create relationship
-                            const relId = await this.brain.relate({
+                            // Collect relationship parameter
+                            relationshipParams.push({
                                 from: extracted.entity.id,
                                 to: toEntityId,
                                 type: rel.type,
@@ -177,14 +178,46 @@ export class SmartImportOrchestrator {
                                     evidence: rel.evidence
                                 }
                             });
-                            result.relationshipIds.push(relId);
-                            result.stats.relationshipsCreated++;
                         }
                         catch (error) {
-                            result.errors.push(`Failed to create relationship: ${error.message}`);
+                            result.errors.push(`Failed to prepare relationship: ${error.message}`);
                         }
                     }
                 }
+                // Batch create all relationships with progress
+                if (relationshipParams.length > 0) {
+                    onProgress?.({
+                        phase: 'relationships',
+                        message: 'Building relationships...',
+                        processed: 0,
+                        total: relationshipParams.length,
+                        entities: result.entityIds.length,
+                        relationships: 0
+                    });
+                    try {
+                        const relationshipIds = await this.brain.relateMany({
+                            items: relationshipParams,
+                            parallel: true,
+                            chunkSize: 100,
+                            continueOnError: true,
+                            onProgress: (done, total) => {
+                                onProgress?.({
+                                    phase: 'relationships',
+                                    message: `Building relationships: ${done}/${total}`,
+                                    processed: done,
+                                    total: total,
+                                    entities: result.entityIds.length,
+                                    relationships: done
+                                });
+                            }
+                        });
+                        result.relationshipIds = relationshipIds;
+                        result.stats.relationshipsCreated = relationshipIds.length;
+                    }
+                    catch (error) {
+                        result.errors.push(`Failed to create relationships: ${error.message}`);
+                    }
+                }
             }
             // Phase 4: Create VFS structure
             if (options.createVFSStructure !== false) {
@@ -415,7 +448,9 @@ export class SmartImportOrchestrator {
             }
         }
         if (options.createRelationships !== false && options.createEntities !== false) {
-            onProgress?.({ phase: 'creating', message: 'Creating relationships...', processed: 0, total: result.extraction.rows.length, entities: result.entityIds.length, relationships: 0 });
+            onProgress?.({ phase: 'creating', message: 'Preparing relationships...', processed: 0, total: result.extraction.rows.length, entities: result.entityIds.length, relationships: 0 });
+            // Collect all relationship parameters
+            const relationshipParams = [];
             for (const extracted of result.extraction.rows) {
                 for (const rel of extracted.relationships) {
                     try {
@@ -430,15 +465,33 @@ export class SmartImportOrchestrator {
                             toEntityId = await this.brain.add({ data: rel.to, type: NounType.Thing, metadata: { name: rel.to, placeholder: true, extractedFrom: extracted.entity.name } });
                             result.entityIds.push(toEntityId);
                         }
-                        const relId = await this.brain.relate({ from: extracted.entity.id, to: toEntityId, type: rel.type, metadata: { confidence: rel.confidence, evidence: rel.evidence } });
-                        result.relationshipIds.push(relId);
-                        result.stats.relationshipsCreated++;
+                        relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, metadata: { confidence: rel.confidence, evidence: rel.evidence } });
                     }
                     catch (error) {
-                        result.errors.push(`Failed to create relationship: ${error.message}`);
+                        result.errors.push(`Failed to prepare relationship: ${error.message}`);
                     }
                 }
             }
+            // Batch create all relationships with progress
+            if (relationshipParams.length > 0) {
+                onProgress?.({ phase: 'relationships', message: 'Building relationships...', processed: 0, total: relationshipParams.length, entities: result.entityIds.length, relationships: 0 });
+                try {
+                    const relationshipIds = await this.brain.relateMany({
+                        items: relationshipParams,
+                        parallel: true,
+                        chunkSize: 100,
+                        continueOnError: true,
+                        onProgress: (done, total) => {
+                            onProgress?.({ phase: 'relationships', message: `Building relationships: ${done}/${total}`, processed: done, total: total, entities: result.entityIds.length, relationships: done });
+                        }
+                    });
+                    result.relationshipIds = relationshipIds;
+                    result.stats.relationshipsCreated = relationshipIds.length;
+                }
+                catch (error) {
+                    result.errors.push(`Failed to create relationships: ${error.message}`);
+                }
+            }
         }
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.48.0",
+  "version": "3.49.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",