@soulcraft/brainy 3.37.8 → 3.39.0

package/dist/import/ImportCoordinator.d.ts

@@ -62,6 +62,10 @@ export interface ImportProgress {
     total?: number;
     entities?: number;
     relationships?: number;
+    /** Rows per second (v3.38.0) */
+    throughput?: number;
+    /** Estimated time remaining in ms (v3.38.0) */
+    eta?: number;
 }
 export interface ImportResult {
     /** Import ID for history tracking */

package/dist/import/ImportCoordinator.js

@@ -237,13 +237,20 @@ export class ImportCoordinator {
             enableConceptExtraction: options.enableConceptExtraction !== false,
             confidenceThreshold: options.confidenceThreshold || 0.6,
             onProgress: (stats) => {
+                // Enhanced progress reporting (v3.38.0) with throughput and ETA
+                const message = stats.throughput
+                    ? `Extracting entities from ${format} (${stats.throughput} rows/sec, ETA: ${Math.round(stats.eta / 1000)}s)...`
+                    : `Extracting entities from ${format}...`;
                 options.onProgress?.({
                     stage: 'extracting',
-                    message: `Extracting entities from ${format}...`,
+                    message,
                     processed: stats.processed,
                     total: stats.total,
                     entities: stats.entities,
-                    relationships: stats.relationships
+                    relationships: stats.relationships,
+                    // Pass through enhanced metrics if available
+                    throughput: stats.throughput,
+                    eta: stats.eta
                 });
             }
         };

package/dist/importers/SmartExcelImporter.d.ts

@@ -25,12 +25,18 @@ export interface SmartExcelOptions extends FormatHandlerOptions {
     definitionColumn?: string;
     typeColumn?: string;
     relatedColumn?: string;
-    /** Progress callback */
+    /** Progress callback (v3.38.0: Enhanced with performance metrics) */
     onProgress?: (stats: {
         processed: number;
         total: number;
         entities: number;
         relationships: number;
+        /** Rows per second (v3.38.0) */
+        throughput?: number;
+        /** Estimated time remaining in ms (v3.38.0) */
+        eta?: number;
+        /** Current phase (v3.38.0) */
+        phase?: string;
     }) => void;
 }
 export interface ExtractedRow {

package/dist/importers/SmartExcelImporter.js

@@ -66,114 +66,141 @@ export class SmartExcelImporter {
         }
         // Detect column names
         const columns = this.detectColumns(rows[0], opts);
-        // Process each row
+        // Process each row with BATCHED PARALLEL PROCESSING (v3.38.0)
         const extractedRows = [];
         const entityMap = new Map();
         const stats = {
             byType: {},
             byConfidence: { high: 0, medium: 0, low: 0 }
         };
-        for (let i = 0; i < rows.length; i++) {
-            const row = rows[i];
-            // Extract data from row
-            const term = this.getColumnValue(row, columns.term) || `Entity_${i}`;
-            const definition = this.getColumnValue(row, columns.definition) || '';
-            const type = this.getColumnValue(row, columns.type);
-            const relatedTerms = this.getColumnValue(row, columns.related);
-            // Extract entities from definition
-            let relatedEntities = [];
-            if (opts.enableNeuralExtraction && definition) {
-                relatedEntities = await this.extractor.extract(definition, {
-                    confidence: opts.confidenceThreshold * 0.8, // Lower threshold for related entities
-                    neuralMatching: true,
-                    cache: { enabled: true }
-                });
-                // Filter out the main term from related entities
-                relatedEntities = relatedEntities.filter(e => e.text.toLowerCase() !== term.toLowerCase());
-            }
-            // Determine main entity type
-            const mainEntityType = type ?
-                this.mapTypeString(type) :
-                (relatedEntities.length > 0 ? relatedEntities[0].type : NounType.Thing);
-            // Generate entity ID
-            const entityId = this.generateEntityId(term);
-            entityMap.set(term.toLowerCase(), entityId);
-            // Extract concepts
-            let concepts = [];
-            if (opts.enableConceptExtraction && definition) {
-                try {
-                    concepts = await this.brain.extractConcepts(definition, { limit: 10 });
-                }
-                catch (error) {
-                    // Concept extraction is optional
-                    concepts = [];
-                }
-            }
-            // Create main entity
-            const mainEntity = {
-                id: entityId,
-                name: term,
-                type: mainEntityType,
-                description: definition,
-                confidence: 0.95, // Main entity from row has high confidence
-                metadata: {
-                    source: 'excel',
-                    row: i + 1,
-                    originalData: row,
-                    concepts,
-                    extractedAt: Date.now()
-                }
-            };
-            // Track statistics
-            this.updateStats(stats, mainEntityType, mainEntity.confidence);
-            // Infer relationships
-            const relationships = [];
-            if (opts.enableRelationshipInference) {
-                // Extract relationships from definition text
-                for (const relEntity of relatedEntities) {
-                    const verbType = await this.inferRelationship(term, relEntity.text, definition);
-                    relationships.push({
-                        from: entityId,
-                        to: relEntity.text, // Use entity name directly, will be resolved later
-                        type: verbType,
-                        confidence: relEntity.confidence,
-                        evidence: `Extracted from: "${definition.substring(0, 100)}..."`
-                    });
-                }
-                // Parse explicit "Related Terms" column
-                if (relatedTerms) {
-                    const terms = relatedTerms.split(/[,;]/).map(t => t.trim()).filter(Boolean);
-                    for (const relTerm of terms) {
-                        // Ensure we don't create self-relationships
-                        if (relTerm.toLowerCase() !== term.toLowerCase()) {
-                            relationships.push({
-                                from: entityId,
-                                to: relTerm, // Use term name directly
-                                type: VerbType.RelatedTo,
-                                confidence: 0.9, // Explicit relationships have high confidence
-                                evidence: `Explicitly listed in "Related" column`
-                            });
+        // Batch processing configuration
+        const CHUNK_SIZE = 10; // Process 10 rows at a time for optimal performance
+        let totalProcessed = 0;
+        const performanceStartTime = Date.now();
+        // Process rows in chunks
+        for (let chunkStart = 0; chunkStart < rows.length; chunkStart += CHUNK_SIZE) {
+            const chunk = rows.slice(chunkStart, Math.min(chunkStart + CHUNK_SIZE, rows.length));
+            // Process chunk in parallel for massive speedup
+            const chunkResults = await Promise.all(chunk.map(async (row, chunkIndex) => {
+                const i = chunkStart + chunkIndex;
+                // Extract data from row
+                const term = this.getColumnValue(row, columns.term) || `Entity_${i}`;
+                const definition = this.getColumnValue(row, columns.definition) || '';
+                const type = this.getColumnValue(row, columns.type);
+                const relatedTerms = this.getColumnValue(row, columns.related);
+                // Parallel extraction: entities AND concepts at the same time
+                const [relatedEntities, concepts] = await Promise.all([
+                    // Extract entities from definition
+                    opts.enableNeuralExtraction && definition
+                        ? this.extractor.extract(definition, {
+                            confidence: opts.confidenceThreshold * 0.8,
+                            neuralMatching: true,
+                            cache: { enabled: true }
+                        }).then(entities => 
+                        // Filter out the main term from related entities
+                        entities.filter(e => e.text.toLowerCase() !== term.toLowerCase()))
+                        : Promise.resolve([]),
+                    // Extract concepts (in parallel with entity extraction)
+                    opts.enableConceptExtraction && definition
+                        ? this.brain.extractConcepts(definition, { limit: 10 }).catch(() => [])
+                        : Promise.resolve([])
+                ]);
+                // Determine main entity type
+                const mainEntityType = type ?
+                    this.mapTypeString(type) :
+                    (relatedEntities.length > 0 ? relatedEntities[0].type : NounType.Thing);
+                // Generate entity ID
+                const entityId = this.generateEntityId(term);
+                // Create main entity
+                const mainEntity = {
+                    id: entityId,
+                    name: term,
+                    type: mainEntityType,
+                    description: definition,
+                    confidence: 0.95,
+                    metadata: {
+                        source: 'excel',
+                        row: i + 1,
+                        originalData: row,
+                        concepts,
+                        extractedAt: Date.now()
+                    }
+                };
+                // Infer relationships
+                const relationships = [];
+                if (opts.enableRelationshipInference) {
+                    // Extract relationships from definition text
+                    for (const relEntity of relatedEntities) {
+                        const verbType = await this.inferRelationship(term, relEntity.text, definition);
+                        relationships.push({
+                            from: entityId,
+                            to: relEntity.text,
+                            type: verbType,
+                            confidence: relEntity.confidence,
+                            evidence: `Extracted from: "${definition.substring(0, 100)}..."`
+                        });
+                    }
+                    // Parse explicit "Related Terms" column
+                    if (relatedTerms) {
+                        const terms = relatedTerms.split(/[,;]/).map(t => t.trim()).filter(Boolean);
+                        for (const relTerm of terms) {
+                            if (relTerm.toLowerCase() !== term.toLowerCase()) {
+                                relationships.push({
+                                    from: entityId,
+                                    to: relTerm,
+                                    type: VerbType.RelatedTo,
+                                    confidence: 0.9,
+                                    evidence: `Explicitly listed in "Related" column`
+                                });
+                            }
                         }
                     }
                 }
+                return {
+                    term,
+                    entityId,
+                    mainEntity,
+                    mainEntityType,
+                    relatedEntities,
+                    relationships,
+                    concepts
+                };
+            }));
+            // Process chunk results sequentially to maintain order
+            for (const result of chunkResults) {
+                // Store entity ID mapping
+                entityMap.set(result.term.toLowerCase(), result.entityId);
+                // Track statistics
+                this.updateStats(stats, result.mainEntityType, result.mainEntity.confidence);
+                // Add extracted row
+                extractedRows.push({
+                    entity: result.mainEntity,
+                    relatedEntities: result.relatedEntities.map(e => ({
+                        name: e.text,
+                        type: e.type,
+                        confidence: e.confidence
+                    })),
+                    relationships: result.relationships,
+                    concepts: result.concepts
+                });
             }
-            // Add extracted row
-            extractedRows.push({
-                entity: mainEntity,
-                relatedEntities: relatedEntities.map(e => ({
-                    name: e.text,
-                    type: e.type,
-                    confidence: e.confidence
-                })),
-                relationships,
-                concepts
-            });
-            // Report progress
+            // Update progress tracking
+            totalProcessed += chunk.length;
+            // Calculate performance metrics
+            const elapsed = Date.now() - performanceStartTime;
+            const rowsPerSecond = totalProcessed / (elapsed / 1000);
+            const remainingRows = rows.length - totalProcessed;
+            const estimatedTimeRemaining = remainingRows / rowsPerSecond;
+            // Report progress with enhanced metrics
             opts.onProgress({
-                processed: i + 1,
+                processed: totalProcessed,
                 total: rows.length,
-                entities: extractedRows.length + relatedEntities.length,
-                relationships: relationships.length
+                entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0),
+                relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0),
+                // Additional performance metrics (v3.38.0)
+                throughput: Math.round(rowsPerSecond * 10) / 10,
+                eta: Math.round(estimatedTimeRemaining),
+                phase: 'extracting'
             });
         }
         return {

package/dist/neural/entityExtractor.d.ts

@@ -24,6 +24,8 @@ export declare class NeuralEntityExtractor {
     private typeEmbeddings;
     private initialized;
     private cache;
+    private embeddingCache;
+    private embeddingCacheStats;
     constructor(brain: Brainy | Brainy<any>, cacheOptions?: EntityCacheOptions);
     /**
      * Initialize type embeddings for neural matching
@@ -61,7 +63,10 @@ export declare class NeuralEntityExtractor {
      */
     private classifyByRules;
     /**
-     * Get embedding for text
+     * Get embedding for text with caching (v3.38.0)
+     *
+     * PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session
+     * to avoid redundant model calls for repeated text (common in large imports)
      */
     private getEmbedding;
     /**
@@ -96,4 +101,27 @@ export declare class NeuralEntityExtractor {
      * Cleanup expired cache entries
      */
     cleanupCache(): number;
+    /**
+     * Clear embedding cache (v3.38.0)
+     *
+     * Clears the runtime embedding cache. Useful for:
+     * - Freeing memory after large imports
+     * - Testing with fresh cache state
+     */
+    clearEmbeddingCache(): void;
+    /**
+     * Get embedding cache statistics (v3.38.0)
+     *
+     * Returns performance metrics for the embedding cache:
+     * - hits: Number of cache hits (avoided model calls)
+     * - misses: Number of cache misses (required model calls)
+     * - size: Current cache size
+     * - hitRate: Percentage of requests served from cache
+     */
+    getEmbeddingCacheStats(): {
+        hitRate: number;
+        hits: number;
+        misses: number;
+        size: number;
+    };
 }

package/dist/neural/entityExtractor.js

@@ -12,6 +12,14 @@ export class NeuralEntityExtractor {
         // Type embeddings for similarity matching
         this.typeEmbeddings = new Map();
         this.initialized = false;
+        // Runtime embedding cache for performance (v3.38.0)
+        // Caches candidate embeddings during an extraction session to avoid redundant model calls
+        this.embeddingCache = new Map();
+        this.embeddingCacheStats = {
+            hits: 0,
+            misses: 0,
+            size: 0
+        };
         this.brain = brain;
         this.cache = new EntityExtractionCache(cacheOptions);
     }
@@ -253,20 +261,46 @@ export class NeuralEntityExtractor {
         return { type: NounType.Thing, confidence: 0.3 };
     }
     /**
-     * Get embedding for text
+     * Get embedding for text with caching (v3.38.0)
+     *
+     * PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session
+     * to avoid redundant model calls for repeated text (common in large imports)
      */
     async getEmbedding(text) {
+        // Normalize text for cache key
+        const normalizedText = text.trim().toLowerCase();
+        // Check cache first
+        const cached = this.embeddingCache.get(normalizedText);
+        if (cached) {
+            this.embeddingCacheStats.hits++;
+            return cached;
+        }
+        // Cache miss - generate embedding
+        this.embeddingCacheStats.misses++;
+        let vector;
         if ('embed' in this.brain && typeof this.brain.embed === 'function') {
-            return await this.brain.embed(text);
+            vector = await this.brain.embed(text);
         }
         else {
             // Fallback - create simple hash-based vector
-            const vector = new Array(384).fill(0);
+            vector = new Array(384).fill(0);
             for (let i = 0; i < text.length; i++) {
                 vector[i % 384] += text.charCodeAt(i) / 255;
             }
-            return vector.map(v => v / text.length);
+            vector = vector.map(v => v / text.length);
+        }
+        // Store in cache
+        this.embeddingCache.set(normalizedText, vector);
+        this.embeddingCacheStats.size = this.embeddingCache.size;
+        // Memory management: Clear cache if it grows too large (>10000 entries)
+        if (this.embeddingCache.size > 10000) {
+            // Keep most recent 5000 entries (simple LRU approximation)
+            const entries = Array.from(this.embeddingCache.entries());
+            this.embeddingCache.clear();
+            entries.slice(-5000).forEach(([k, v]) => this.embeddingCache.set(k, v));
+            this.embeddingCacheStats.size = this.embeddingCache.size;
         }
+        return vector;
     }
     /**
      * Calculate cosine similarity between vectors
@@ -355,5 +389,36 @@ export class NeuralEntityExtractor {
     cleanupCache() {
         return this.cache.cleanup();
     }
+    /**
+     * Clear embedding cache (v3.38.0)
+     *
+     * Clears the runtime embedding cache. Useful for:
+     * - Freeing memory after large imports
+     * - Testing with fresh cache state
+     */
+    clearEmbeddingCache() {
+        this.embeddingCache.clear();
+        this.embeddingCacheStats = {
+            hits: 0,
+            misses: 0,
+            size: 0
+        };
+    }
+    /**
+     * Get embedding cache statistics (v3.38.0)
+     *
+     * Returns performance metrics for the embedding cache:
+     * - hits: Number of cache hits (avoided model calls)
+     * - misses: Number of cache misses (required model calls)
+     * - size: Current cache size
+     * - hitRate: Percentage of requests served from cache
+     */
+    getEmbeddingCacheStats() {
+        const total = this.embeddingCacheStats.hits + this.embeddingCacheStats.misses;
+        return {
+            ...this.embeddingCacheStats,
+            hitRate: total > 0 ? this.embeddingCacheStats.hits / total : 0
+        };
+    }
 }
 //# sourceMappingURL=entityExtractor.js.map

package/dist/storage/adapters/gcsStorage.js

@@ -347,9 +347,7 @@ export class GcsStorage extends BaseStorage {
             if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
                 this.nounCacheManager.set(node.id, node);
             }
-            else {
-                prodLog.warn(`[saveNode] Not caching node ${node.id.substring(0, 8)}... with empty vector (HNSW lazy mode)`);
-            }
+            // Note: Empty vectors are intentional during HNSW lazy mode - not logged
             // Increment noun count
             const metadata = await this.getNounMetadata(node.id);
             if (metadata && metadata.type) {
@@ -392,53 +390,28 @@ export class GcsStorage extends BaseStorage {
      */
     async getNode(id) {
         await this.ensureInitialized();
-        // Check cache first WITH LOGGING
+        // Check cache first
         const cached = await this.nounCacheManager.get(id);
-        // DIAGNOSTIC LOGGING: Reveal cache poisoning
-        prodLog.info(`[getNode] 🔍 Cache check for ${id.substring(0, 8)}...:`, {
-            hasCached: cached !== undefined,
-            isNull: cached === null,
-            isObject: cached !== null && typeof cached === 'object',
-            type: typeof cached
-        });
-        // CRITICAL FIX (v3.37.8): Validate cached object before returning
+        // Validate cached object before returning (v3.37.8+)
         if (cached !== undefined && cached !== null) {
-            // Log cached object structure to diagnose incomplete objects
-            prodLog.info(`[getNode] Cached object structure:`, {
-                hasId: !!cached.id,
-                idMatches: cached.id === id,
-                hasVector: !!cached.vector,
-                vectorLength: cached.vector?.length,
-                hasConnections: !!cached.connections,
-                connectionsType: typeof cached.connections,
-                hasLevel: cached.level !== undefined,
-                level: cached.level,
-                objectKeys: Object.keys(cached || {})
-            });
             // Validate cached object has required fields (including non-empty vector!)
             if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
-                prodLog.error(`[getNode] ❌ INVALID cached object for ${id.substring(0, 8)}...:`, {
-                    reason: !cached.id ? 'missing id' :
-                        !cached.vector ? 'missing vector' :
-                            !Array.isArray(cached.vector) ? 'vector not array' :
-                                cached.vector.length === 0 ? 'vector is empty array' :
-                                    'unknown'
-                });
-                prodLog.error(`[getNode] Removing invalid object from cache and loading from GCS`);
+                // Invalid cache detected - log and auto-recover
+                prodLog.warn(`[GCS] Invalid cached object for ${id.substring(0, 8)} (${!cached.id ? 'missing id' :
+                    !cached.vector ? 'missing vector' :
+                        !Array.isArray(cached.vector) ? 'vector not array' :
+                            'empty vector'}) - removing from cache and reloading`);
                 this.nounCacheManager.delete(id);
                 // Fall through to load from GCS
             }
             else {
-                prodLog.info(`[getNode] ✅ Valid cached object - returning`);
+                // Valid cache hit
                 this.logger.trace(`Cache hit for noun ${id}`);
                 return cached;
             }
         }
         else if (cached === null) {
-            prodLog.warn(`[getNode] ⚠️ Cache contains NULL for ${id.substring(0, 8)}... - ignoring and loading from GCS`);
-        }
-        else {
-            prodLog.info(`[getNode] ❌ Cache MISS - loading from GCS for ${id.substring(0, 8)}...`);
+            prodLog.warn(`[GCS] Cache contains null for ${id.substring(0, 8)} - reloading from storage`);
         }
         // Apply backpressure
         const requestId = await this.applyBackpressure();
@@ -446,20 +419,11 @@ export class GcsStorage extends BaseStorage {
             this.logger.trace(`Getting node ${id}`);
             // Get the GCS key with UUID-based sharding
             const key = this.getNounKey(id);
-            // DIAGNOSTIC LOGGING: Show exact path being accessed
-            prodLog.info(`[getNode] 🔍 Attempting to load:`);
-            prodLog.info(`[getNode]   UUID: ${id}`);
-            prodLog.info(`[getNode]   Path: ${key}`);
-            prodLog.info(`[getNode]   Bucket: ${this.bucketName}`);
             // Download from GCS
             const file = this.bucket.file(key);
-            prodLog.info(`[getNode] 📥 Downloading file...`);
             const [contents] = await file.download();
-            prodLog.info(`[getNode] ✅ Download successful: ${contents.length} bytes`);
             // Parse JSON
-            prodLog.info(`[getNode] 🔧 Parsing JSON...`);
             const data = JSON.parse(contents.toString());
-            prodLog.info(`[getNode] ✅ JSON parsed successfully, id: ${data.id}`);
             // Convert serialized connections back to Map<number, Set<string>>
             const connections = new Map();
             for (const [level, nounIds] of Object.entries(data.connections || {})) {
@@ -477,10 +441,9 @@ export class GcsStorage extends BaseStorage {
             // CRITICAL FIX: Only cache valid nodes with non-empty vectors (never cache null or empty)
             if (node && node.id && node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
                 this.nounCacheManager.set(id, node);
-                prodLog.info(`[getNode] 💾 Cached node ${id.substring(0, 8)}... successfully`);
             }
             else {
-                prodLog.warn(`[getNode] ⚠️ NOT caching invalid node for ${id.substring(0, 8)}... (missing id/vector or empty vector)`);
+                prodLog.warn(`[GCS] Not caching invalid node ${id.substring(0, 8)} (missing id/vector or empty vector)`);
             }
             this.logger.trace(`Successfully retrieved node ${id}`);
             this.releaseBackpressure(true, requestId);
@@ -868,13 +831,6 @@ export class GcsStorage extends BaseStorage {
         await this.ensureInitialized(); // CRITICAL: Must initialize before using this.bucket
         const limit = options.limit || 100;
         const useCache = options.useCache !== false;
-        // DIAGNOSTIC LOGGING: Track pagination performance
-        prodLog.info(`[getNodesWithPagination] Starting pagination: limit=${limit}, cursor=${options.cursor || 'none'}`);
-        const startTime = Date.now();
-        let shardsChecked = 0;
-        let filesFound = 0;
-        let nodesLoaded = 0;
-        let nodesFailed = 0;
         try {
             const nodes = [];
             // Parse cursor (format: "shardIndex:gcsPageToken")
@@ -889,7 +845,6 @@ export class GcsStorage extends BaseStorage {
             for (let shardIndex = startShardIndex; shardIndex < TOTAL_SHARDS; shardIndex++) {
                 const shardId = getShardIdByIndex(shardIndex);
                 const shardPrefix = `${this.nounPrefix}${shardId}/`;
-                shardsChecked++;
                 // List objects in this shard
                 // Cap maxResults to GCS API limit to prevent "Invalid unsigned integer" errors
                 const requestedPageSize = limit - nodes.length;
@@ -899,12 +854,6 @@ export class GcsStorage extends BaseStorage {
                     maxResults: cappedPageSize,
                     pageToken: shardIndex === startShardIndex ? gcsPageToken : undefined
                 });
-                // DIAGNOSTIC LOGGING: Show files found per shard (only log non-empty shards)
-                if (files && files.length > 0) {
-                    filesFound += files.length;
-                    prodLog.info(`[Shard ${shardId}] Found ${files.length} files in "${shardPrefix}"`);
-                    prodLog.info(`[Shard ${shardId}] Sample file names: ${files.slice(0, 3).map((f) => f.name).join(', ')}`);
-                }
                 // Extract node IDs from file names
                 if (files && files.length > 0) {
                     const nodeIds = files
@@ -921,21 +870,11 @@ export class GcsStorage extends BaseStorage {
                         return name;
                     })
                         .filter((id) => id && id.length > 0);
-                    // DIAGNOSTIC LOGGING: Show extracted UUIDs
-                    prodLog.info(`[Shard ${shardId}] Extracted ${nodeIds.length} UUIDs: ${nodeIds.slice(0, 3).join(', ')}...`);
                     // Load nodes
                     for (const id of nodeIds) {
-                        // DIAGNOSTIC LOGGING: Show each getNode() attempt
-                        prodLog.info(`[Shard ${shardId}] Calling getNode("${id}")...`);
                         const node = await this.getNode(id);
                         if (node) {
                             nodes.push(node);
-                            nodesLoaded++;
-                            prodLog.info(`[Shard ${shardId}] ✅ Successfully loaded node ${id}`);
-                        }
-                        else {
-                            nodesFailed++;
-                            prodLog.warn(`[Shard ${shardId}] ❌ getNode("${id}") returned null!`);
                         }
                         if (nodes.length >= limit) {
                             break;
@@ -968,14 +907,6 @@ export class GcsStorage extends BaseStorage {
                 // Continue to next shard
             }
             // No more shards or nodes
-            // DIAGNOSTIC LOGGING: Final summary
-            const elapsedTime = Date.now() - startTime;
-            prodLog.info(`[getNodesWithPagination] COMPLETED in ${elapsedTime}ms:`);
-            prodLog.info(`  - Shards checked: ${shardsChecked}/${TOTAL_SHARDS}`);
-            prodLog.info(`  - Files found: ${filesFound}`);
-            prodLog.info(`  - Nodes loaded: ${nodesLoaded}`);
-            prodLog.info(`  - Nodes failed: ${nodesFailed}`);
-            prodLog.info(`  - Success rate: ${filesFound > 0 ? ((nodesLoaded / filesFound) * 100).toFixed(1) : 'N/A'}%`);
             return {
                 nodes,
                 totalCount: this.totalNounCount,

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -818,80 +818,47 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async getNode(id) {
         await this.ensureInitialized();
-        // Check cache first WITH LOGGING
+        // Check cache first
         const cached = this.nodeCache.get(id);
-        // DIAGNOSTIC LOGGING: Reveal cache poisoning
-        prodLog.info(`[getNode] 🔍 Cache check for ${id.substring(0, 8)}...:`, {
-            hasCached: cached !== undefined,
-            isNull: cached === null,
-            isObject: cached !== null && typeof cached === 'object',
-            type: typeof cached
-        });
-        // CRITICAL FIX (v3.37.8): Validate cached object before returning
+        // Validate cached object before returning (v3.37.8+)
         if (cached !== undefined && cached !== null) {
-            // Log cached object structure to diagnose incomplete objects
-            prodLog.info(`[getNode] Cached object structure:`, {
-                hasId: !!cached.id,
-                idMatches: cached.id === id,
-                hasVector: !!cached.vector,
-                vectorLength: cached.vector?.length,
-                hasConnections: !!cached.connections,
-                connectionsType: typeof cached.connections,
-                objectKeys: Object.keys(cached || {})
-            });
             // Validate cached object has required fields (including non-empty vector!)
             if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
-                prodLog.error(`[getNode] ❌ INVALID cached object for ${id.substring(0, 8)}...:`, {
-                    reason: !cached.id ? 'missing id' :
-                        !cached.vector ? 'missing vector' :
-                            !Array.isArray(cached.vector) ? 'vector not array' :
-                                cached.vector.length === 0 ? 'vector is empty array' :
-                                    'unknown'
-                });
-                prodLog.error(`[getNode] Removing invalid object from cache and loading from S3`);
+                // Invalid cache detected - log and auto-recover
+                prodLog.warn(`[S3] Invalid cached object for ${id.substring(0, 8)} (${!cached.id ? 'missing id' :
+                    !cached.vector ? 'missing vector' :
+                        !Array.isArray(cached.vector) ? 'vector not array' :
+                            'empty vector'}) - removing from cache and reloading`);
                 this.nodeCache.delete(id);
                 // Fall through to load from S3
             }
             else {
-                prodLog.info(`[getNode] ✅ Valid cached object - returning`);
+                // Valid cache hit
                 this.logger.trace(`Cache hit for node ${id}`);
                 return cached;
             }
         }
         else if (cached === null) {
-            prodLog.warn(`[getNode] ⚠️ Cache contains NULL for ${id.substring(0, 8)}... - ignoring and loading from S3`);
-        }
-        else {
-            prodLog.info(`[getNode] ❌ Cache MISS - loading from S3 for ${id.substring(0, 8)}...`);
+            prodLog.warn(`[S3] Cache contains null for ${id.substring(0, 8)} - reloading from storage`);
         }
         try {
             // Import the GetObjectCommand only when needed
             const { GetObjectCommand } = await import('@aws-sdk/client-s3');
             // Use getNounKey() to properly handle sharding
             const key = this.getNounKey(id);
-            // DIAGNOSTIC LOGGING: Show exact path being accessed
-            prodLog.info(`[getNode] 🔍 Attempting to load:`);
-            prodLog.info(`[getNode]   UUID: ${id}`);
-            prodLog.info(`[getNode]   Path: ${key}`);
-            prodLog.info(`[getNode]   Bucket: ${this.bucketName}`);
             // Try to get the node from the nouns directory
-            prodLog.info(`[getNode] 📥 Downloading file...`);
             const response = await this.s3Client.send(new GetObjectCommand({
                 Bucket: this.bucketName,
                 Key: key
             }));
             // Check if response is null or undefined
             if (!response || !response.Body) {
-                prodLog.warn(`[getNode] ❌ Response or Body is null/undefined`);
+                prodLog.warn(`[S3] Response or Body is null/undefined for ${id.substring(0, 8)}`);
                 return null;
             }
-            // Convert the response body to a string
+            // Convert the response body to a string and parse JSON
             const bodyContents = await response.Body.transformToString();
-            prodLog.info(`[getNode] ✅ Download successful: ${bodyContents.length} bytes`);
-            // Parse the JSON string
-            prodLog.info(`[getNode] 🔧 Parsing JSON...`);
             const parsedNode = JSON.parse(bodyContents);
-            prodLog.info(`[getNode] ✅ JSON parsed successfully, id: ${parsedNode.id}`);
             // Ensure the parsed node has the expected properties
             if (!parsedNode ||
                 !parsedNode.id ||
@@ -917,41 +884,25 @@ export class S3CompatibleStorage extends BaseStorage {
             // CRITICAL FIX: Only cache valid nodes with non-empty vectors (never cache null or empty)
             if (node && node.id && node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
                 this.nodeCache.set(id, node);
-                prodLog.info(`[getNode] 💾 Cached node ${id.substring(0, 8)}... successfully`);
             }
             else {
-                prodLog.warn(`[getNode] ⚠️ NOT caching invalid node for ${id.substring(0, 8)}... (missing id/vector or empty vector)`);
+                prodLog.warn(`[S3] Not caching invalid node ${id.substring(0, 8)} (missing id/vector or empty vector)`);
             }
             this.logger.trace(`Successfully retrieved node ${id}`);
             return node;
         }
         catch (error) {
-            // DIAGNOSTIC LOGGING: Log EVERY error before any conditional checks
-            const key = this.getNounKey(id);
-            prodLog.error(`[getNode] ❌ EXCEPTION CAUGHT:`);
-            prodLog.error(`[getNode]   UUID: ${id}`);
-            prodLog.error(`[getNode]   Path: ${key}`);
-            prodLog.error(`[getNode]   Bucket: ${this.bucketName}`);
-            prodLog.error(`[getNode]   Error type: ${error?.constructor?.name || typeof error}`);
-            prodLog.error(`[getNode]   Error name: ${error?.name}`);
-            prodLog.error(`[getNode]   Error code: ${JSON.stringify(error?.Code || error?.code)}`);
-            prodLog.error(`[getNode]   Error message: ${error?.message || String(error)}`);
-            prodLog.error(`[getNode]   HTTP status: ${error?.$metadata?.httpStatusCode}`);
-            prodLog.error(`[getNode]   Error object:`, JSON.stringify(error, null, 2));
             // Check if this is a "not found" error (S3 uses "NoSuchKey")
             if (error?.name === 'NoSuchKey' || error?.Code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
-                prodLog.warn(`[getNode] Identified as 404/NoSuchKey error - returning null WITHOUT caching`);
-                // CRITICAL FIX: Do NOT cache null values
+                // File not found - not cached, just return null
                 return null;
             }
             // Handle throttling
             if (this.isThrottlingError(error)) {
-                prodLog.warn(`[getNode] Identified as throttling error - rethrowing`);
                 await this.handleThrottling(error);
                 throw error;
             }
             // All other errors should throw, not return null
-            prodLog.error(`[getNode] Unhandled error - rethrowing`);
             this.logger.error(`Failed to get node ${id}:`, error);
             throw BrainyError.fromError(error, `getNoun(${id})`);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "3.37.8",
+  "version": "3.39.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",