@soulcraft/brainy 3.38.0 → 3.39.0

package/dist/import/ImportCoordinator.d.ts

@@ -62,6 +62,10 @@ export interface ImportProgress {
     total?: number;
     entities?: number;
     relationships?: number;
+    /** Rows per second (v3.38.0) */
+    throughput?: number;
+    /** Estimated time remaining in ms (v3.38.0) */
+    eta?: number;
 }
 export interface ImportResult {
     /** Import ID for history tracking */

package/dist/import/ImportCoordinator.js

@@ -237,13 +237,20 @@ export class ImportCoordinator {
             enableConceptExtraction: options.enableConceptExtraction !== false,
             confidenceThreshold: options.confidenceThreshold || 0.6,
             onProgress: (stats) => {
+                // Enhanced progress reporting (v3.38.0) with throughput and ETA
+                const message = stats.throughput
+                    ? `Extracting entities from ${format} (${stats.throughput} rows/sec, ETA: ${Math.round(stats.eta / 1000)}s)...`
+                    : `Extracting entities from ${format}...`;
                 options.onProgress?.({
                     stage: 'extracting',
-                    message: `Extracting entities from ${format}...`,
+                    message,
                     processed: stats.processed,
                     total: stats.total,
                     entities: stats.entities,
-                    relationships: stats.relationships
+                    relationships: stats.relationships,
+                    // Pass through enhanced metrics if available
+                    throughput: stats.throughput,
+                    eta: stats.eta
                 });
             }
         };

package/dist/importers/SmartExcelImporter.d.ts

@@ -25,12 +25,18 @@ export interface SmartExcelOptions extends FormatHandlerOptions {
     definitionColumn?: string;
     typeColumn?: string;
     relatedColumn?: string;
-    /** Progress callback */
+    /** Progress callback (v3.38.0: Enhanced with performance metrics) */
     onProgress?: (stats: {
         processed: number;
         total: number;
         entities: number;
         relationships: number;
+        /** Rows per second (v3.38.0) */
+        throughput?: number;
+        /** Estimated time remaining in ms (v3.38.0) */
+        eta?: number;
+        /** Current phase (v3.38.0) */
+        phase?: string;
     }) => void;
 }
 export interface ExtractedRow {

package/dist/importers/SmartExcelImporter.js

@@ -66,114 +66,141 @@ export class SmartExcelImporter {
         }
         // Detect column names
         const columns = this.detectColumns(rows[0], opts);
-        // Process each row
+        // Process each row with BATCHED PARALLEL PROCESSING (v3.38.0)
         const extractedRows = [];
         const entityMap = new Map();
         const stats = {
             byType: {},
             byConfidence: { high: 0, medium: 0, low: 0 }
         };
-        for (let i = 0; i < rows.length; i++) {
-            const row = rows[i];
-            // Extract data from row
-            const term = this.getColumnValue(row, columns.term) || `Entity_${i}`;
-            const definition = this.getColumnValue(row, columns.definition) || '';
-            const type = this.getColumnValue(row, columns.type);
-            const relatedTerms = this.getColumnValue(row, columns.related);
-            // Extract entities from definition
-            let relatedEntities = [];
-            if (opts.enableNeuralExtraction && definition) {
-                relatedEntities = await this.extractor.extract(definition, {
-                    confidence: opts.confidenceThreshold * 0.8, // Lower threshold for related entities
-                    neuralMatching: true,
-                    cache: { enabled: true }
-                });
-                // Filter out the main term from related entities
-                relatedEntities = relatedEntities.filter(e => e.text.toLowerCase() !== term.toLowerCase());
-            }
-            // Determine main entity type
-            const mainEntityType = type ?
-                this.mapTypeString(type) :
-                (relatedEntities.length > 0 ? relatedEntities[0].type : NounType.Thing);
-            // Generate entity ID
-            const entityId = this.generateEntityId(term);
-            entityMap.set(term.toLowerCase(), entityId);
-            // Extract concepts
-            let concepts = [];
-            if (opts.enableConceptExtraction && definition) {
-                try {
-                    concepts = await this.brain.extractConcepts(definition, { limit: 10 });
-                }
-                catch (error) {
-                    // Concept extraction is optional
-                    concepts = [];
-                }
-            }
-            // Create main entity
-            const mainEntity = {
-                id: entityId,
-                name: term,
-                type: mainEntityType,
-                description: definition,
-                confidence: 0.95, // Main entity from row has high confidence
-                metadata: {
-                    source: 'excel',
-                    row: i + 1,
-                    originalData: row,
-                    concepts,
-                    extractedAt: Date.now()
-                }
-            };
-            // Track statistics
-            this.updateStats(stats, mainEntityType, mainEntity.confidence);
-            // Infer relationships
-            const relationships = [];
-            if (opts.enableRelationshipInference) {
-                // Extract relationships from definition text
-                for (const relEntity of relatedEntities) {
-                    const verbType = await this.inferRelationship(term, relEntity.text, definition);
-                    relationships.push({
-                        from: entityId,
-                        to: relEntity.text, // Use entity name directly, will be resolved later
-                        type: verbType,
-                        confidence: relEntity.confidence,
-                        evidence: `Extracted from: "${definition.substring(0, 100)}..."`
-                    });
-                }
-                // Parse explicit "Related Terms" column
-                if (relatedTerms) {
-                    const terms = relatedTerms.split(/[,;]/).map(t => t.trim()).filter(Boolean);
-                    for (const relTerm of terms) {
-                        // Ensure we don't create self-relationships
-                        if (relTerm.toLowerCase() !== term.toLowerCase()) {
-                            relationships.push({
-                                from: entityId,
-                                to: relTerm, // Use term name directly
-                                type: VerbType.RelatedTo,
-                                confidence: 0.9, // Explicit relationships have high confidence
-                                evidence: `Explicitly listed in "Related" column`
-                            });
+        // Batch processing configuration
+        const CHUNK_SIZE = 10; // Process 10 rows at a time for optimal performance
+        let totalProcessed = 0;
+        const performanceStartTime = Date.now();
+        // Process rows in chunks
+        for (let chunkStart = 0; chunkStart < rows.length; chunkStart += CHUNK_SIZE) {
+            const chunk = rows.slice(chunkStart, Math.min(chunkStart + CHUNK_SIZE, rows.length));
+            // Process chunk in parallel for massive speedup
+            const chunkResults = await Promise.all(chunk.map(async (row, chunkIndex) => {
+                const i = chunkStart + chunkIndex;
+                // Extract data from row
+                const term = this.getColumnValue(row, columns.term) || `Entity_${i}`;
+                const definition = this.getColumnValue(row, columns.definition) || '';
+                const type = this.getColumnValue(row, columns.type);
+                const relatedTerms = this.getColumnValue(row, columns.related);
+                // Parallel extraction: entities AND concepts at the same time
+                const [relatedEntities, concepts] = await Promise.all([
+                    // Extract entities from definition
+                    opts.enableNeuralExtraction && definition
+                        ? this.extractor.extract(definition, {
+                            confidence: opts.confidenceThreshold * 0.8,
+                            neuralMatching: true,
+                            cache: { enabled: true }
+                        }).then(entities => 
+                        // Filter out the main term from related entities
+                        entities.filter(e => e.text.toLowerCase() !== term.toLowerCase()))
+                        : Promise.resolve([]),
+                    // Extract concepts (in parallel with entity extraction)
+                    opts.enableConceptExtraction && definition
+                        ? this.brain.extractConcepts(definition, { limit: 10 }).catch(() => [])
+                        : Promise.resolve([])
+                ]);
+                // Determine main entity type
+                const mainEntityType = type ?
+                    this.mapTypeString(type) :
+                    (relatedEntities.length > 0 ? relatedEntities[0].type : NounType.Thing);
+                // Generate entity ID
+                const entityId = this.generateEntityId(term);
+                // Create main entity
+                const mainEntity = {
+                    id: entityId,
+                    name: term,
+                    type: mainEntityType,
+                    description: definition,
+                    confidence: 0.95,
+                    metadata: {
+                        source: 'excel',
+                        row: i + 1,
+                        originalData: row,
+                        concepts,
+                        extractedAt: Date.now()
+                    }
+                };
+                // Infer relationships
+                const relationships = [];
+                if (opts.enableRelationshipInference) {
+                    // Extract relationships from definition text
+                    for (const relEntity of relatedEntities) {
+                        const verbType = await this.inferRelationship(term, relEntity.text, definition);
+                        relationships.push({
+                            from: entityId,
+                            to: relEntity.text,
+                            type: verbType,
+                            confidence: relEntity.confidence,
+                            evidence: `Extracted from: "${definition.substring(0, 100)}..."`
+                        });
+                    }
+                    // Parse explicit "Related Terms" column
+                    if (relatedTerms) {
+                        const terms = relatedTerms.split(/[,;]/).map(t => t.trim()).filter(Boolean);
+                        for (const relTerm of terms) {
+                            if (relTerm.toLowerCase() !== term.toLowerCase()) {
+                                relationships.push({
+                                    from: entityId,
+                                    to: relTerm,
+                                    type: VerbType.RelatedTo,
+                                    confidence: 0.9,
+                                    evidence: `Explicitly listed in "Related" column`
+                                });
+                            }
                         }
                     }
                 }
+                return {
+                    term,
+                    entityId,
+                    mainEntity,
+                    mainEntityType,
+                    relatedEntities,
+                    relationships,
+                    concepts
+                };
+            }));
+            // Process chunk results sequentially to maintain order
+            for (const result of chunkResults) {
+                // Store entity ID mapping
+                entityMap.set(result.term.toLowerCase(), result.entityId);
+                // Track statistics
+                this.updateStats(stats, result.mainEntityType, result.mainEntity.confidence);
+                // Add extracted row
+                extractedRows.push({
+                    entity: result.mainEntity,
+                    relatedEntities: result.relatedEntities.map(e => ({
+                        name: e.text,
+                        type: e.type,
+                        confidence: e.confidence
+                    })),
+                    relationships: result.relationships,
+                    concepts: result.concepts
+                });
             }
-            // Add extracted row
-            extractedRows.push({
-                entity: mainEntity,
-                relatedEntities: relatedEntities.map(e => ({
-                    name: e.text,
-                    type: e.type,
-                    confidence: e.confidence
-                })),
-                relationships,
-                concepts
-            });
-            // Report progress
+            // Update progress tracking
+            totalProcessed += chunk.length;
+            // Calculate performance metrics
+            const elapsed = Date.now() - performanceStartTime;
+            const rowsPerSecond = totalProcessed / (elapsed / 1000);
+            const remainingRows = rows.length - totalProcessed;
+            const estimatedTimeRemaining = remainingRows / rowsPerSecond;
+            // Report progress with enhanced metrics
             opts.onProgress({
-                processed: i + 1,
+                processed: totalProcessed,
                 total: rows.length,
-                entities: extractedRows.length + relatedEntities.length,
-                relationships: relationships.length
+                entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
+                relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
+                // Additional performance metrics (v3.38.0)
+                throughput: Math.round(rowsPerSecond * 10) / 10,
+                eta: Math.round(estimatedTimeRemaining),
+                phase: 'extracting'
             });
         }
         return {

package/dist/neural/entityExtractor.d.ts

@@ -24,6 +24,8 @@ export declare class NeuralEntityExtractor {
     private typeEmbeddings;
     private initialized;
     private cache;
+    private embeddingCache;
+    private embeddingCacheStats;
     constructor(brain: Brainy | Brainy<any>, cacheOptions?: EntityCacheOptions);
     /**
      * Initialize type embeddings for neural matching
@@ -61,7 +63,10 @@ export declare class NeuralEntityExtractor {
      */
     private classifyByRules;
     /**
-     * Get embedding for text
+     * Get embedding for text with caching (v3.38.0)
+     *
+     * PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session
+     * to avoid redundant model calls for repeated text (common in large imports)
      */
     private getEmbedding;
     /**
@@ -96,4 +101,27 @@ export declare class NeuralEntityExtractor {
      * Cleanup expired cache entries
      */
     cleanupCache(): number;
+    /**
+     * Clear embedding cache (v3.38.0)
+     *
+     * Clears the runtime embedding cache. Useful for:
+     * - Freeing memory after large imports
+     * - Testing with fresh cache state
+     */
+    clearEmbeddingCache(): void;
+    /**
+     * Get embedding cache statistics (v3.38.0)
+     *
+     * Returns performance metrics for the embedding cache:
+     * - hits: Number of cache hits (avoided model calls)
+     * - misses: Number of cache misses (required model calls)
+     * - size: Current cache size
+     * - hitRate: Percentage of requests served from cache
+     */
+    getEmbeddingCacheStats(): {
+        hitRate: number;
+        hits: number;
+        misses: number;
+        size: number;
+    };
 }

package/dist/neural/entityExtractor.js

@@ -12,6 +12,14 @@ export class NeuralEntityExtractor {
         // Type embeddings for similarity matching
         this.typeEmbeddings = new Map();
         this.initialized = false;
+        // Runtime embedding cache for performance (v3.38.0)
+        // Caches candidate embeddings during an extraction session to avoid redundant model calls
+        this.embeddingCache = new Map();
+        this.embeddingCacheStats = {
+            hits: 0,
+            misses: 0,
+            size: 0
+        };
         this.brain = brain;
         this.cache = new EntityExtractionCache(cacheOptions);
     }
@@ -253,20 +261,46 @@ export class NeuralEntityExtractor {
         return { type: NounType.Thing, confidence: 0.3 };
     }
     /**
-     * Get embedding for text
+     * Get embedding for text with caching (v3.38.0)
+     *
+     * PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session
+     * to avoid redundant model calls for repeated text (common in large imports)
      */
     async getEmbedding(text) {
+        // Normalize text for cache key
+        const normalizedText = text.trim().toLowerCase();
+        // Check cache first
+        const cached = this.embeddingCache.get(normalizedText);
+        if (cached) {
+            this.embeddingCacheStats.hits++;
+            return cached;
+        }
+        // Cache miss - generate embedding
+        this.embeddingCacheStats.misses++;
+        let vector;
         if ('embed' in this.brain && typeof this.brain.embed === 'function') {
-            return await this.brain.embed(text);
+            vector = await this.brain.embed(text);
         }
         else {
             // Fallback - create simple hash-based vector
-            const vector = new Array(384).fill(0);
+            vector = new Array(384).fill(0);
             for (let i = 0; i < text.length; i++) {
                 vector[i % 384] += text.charCodeAt(i) / 255;
             }
-            return vector.map(v => v / text.length);
+            vector = vector.map(v => v / text.length);
+        }
+        // Store in cache
+        this.embeddingCache.set(normalizedText, vector);
+        this.embeddingCacheStats.size = this.embeddingCache.size;
+        // Memory management: Clear cache if it grows too large (>10000 entries)
+        if (this.embeddingCache.size > 10000) {
+            // Keep most recent 5000 entries (simple LRU approximation)
+            const entries = Array.from(this.embeddingCache.entries());
+            this.embeddingCache.clear();
+            entries.slice(-5000).forEach(([k, v]) => this.embeddingCache.set(k, v));
+            this.embeddingCacheStats.size = this.embeddingCache.size;
         }
+        return vector;
     }
     /**
      * Calculate cosine similarity between vectors
@@ -355,5 +389,36 @@ export class NeuralEntityExtractor {
     cleanupCache() {
         return this.cache.cleanup();
     }
+    /**
+     * Clear embedding cache (v3.38.0)
+     *
+     * Clears the runtime embedding cache. Useful for:
+     * - Freeing memory after large imports
+     * - Testing with fresh cache state
+     */
+    clearEmbeddingCache() {
+        this.embeddingCache.clear();
+        this.embeddingCacheStats = {
+            hits: 0,
+            misses: 0,
+            size: 0
+        };
+    }
+    /**
+     * Get embedding cache statistics (v3.38.0)
+     *
+     * Returns performance metrics for the embedding cache:
+     * - hits: Number of cache hits (avoided model calls)
+     * - misses: Number of cache misses (required model calls)
+     * - size: Current cache size
+     * - hitRate: Percentage of requests served from cache
+     */
+    getEmbeddingCacheStats() {
+        const total = this.embeddingCacheStats.hits + this.embeddingCacheStats.misses;
+        return {
+            ...this.embeddingCacheStats,
+            hitRate: total > 0 ? this.embeddingCacheStats.hits / total : 0
+        };
+    }
 }
 //# sourceMappingURL=entityExtractor.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.38.0",
+  "version": "3.39.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",