@soulcraft/brainy 3.43.3 → 3.45.0

package/dist/utils/metadataIndex.js

@@ -31,10 +31,6 @@ export class MetadataIndexManager {
         this.activeLocks = new Map();
         this.lockPromises = new Map();
         this.lockTimers = new Map(); // Track timers for cleanup
-        // Adaptive Chunked Sparse Indexing (v3.42.0)
-        // Reduces file count from 560k → 89 files (630x reduction)
-        // ALL fields now use chunking - no more flat files
-        this.sparseIndices = new Map(); // field -> sparse index
         this.storage = storage;
         this.config = {
             maxIndexSize: config.maxIndexSize ?? 10000,
@@ -87,6 +83,30 @@ export class MetadataIndexManager {
     async init() {
         // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
         await this.idMapper.init();
+        // Warm the cache with common fields (v3.44.1 - lazy loading optimization)
+        await this.warmCache();
+    }
+    /**
+     * Warm the cache by preloading common field sparse indices (v3.44.1)
+     * This improves cache hit rates by loading frequently-accessed fields at startup
+     * Target: >80% cache hit rate for typical workloads
+     */
+    async warmCache() {
+        // Common fields used in most queries
+        const commonFields = ['noun', 'type', 'service', 'createdAt'];
+        prodLog.debug(`🔥 Warming metadata cache with common fields: ${commonFields.join(', ')}`);
+        // Preload in parallel for speed
+        await Promise.all(commonFields.map(async (field) => {
+            try {
+                await this.loadSparseIndex(field);
+            }
+            catch (error) {
+                // Silently ignore if field doesn't exist yet
+                // This maintains zero-configuration principle
+                prodLog.debug(`Cache warming: field '${field}' not yet indexed`);
+            }
+        }));
+        prodLog.debug('✅ Metadata cache warmed successfully');
     }
     /**
      * Acquire an in-memory lock for coordinating concurrent metadata index writes
@@ -303,16 +323,13 @@ export class MetadataIndexManager {
     }
     /**
      * Get IDs for a value using chunked sparse index with roaring bitmaps (v3.43.0)
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async getIdsFromChunks(field, value) {
-        // Load sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                return []; // No chunked index exists yet
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            return []; // No chunked index exists yet
         }
         // Find candidate chunks using zone maps and bloom filters
         const normalizedValue = this.normalizeValue(value, field);
@@ -339,16 +356,13 @@ export class MetadataIndexManager {
     }
     /**
      * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps (v3.43.0)
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async getIdsFromChunksForRange(field, min, max, includeMin = true, includeMax = true) {
-        // Load sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                return []; // No chunked index exists yet
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            return []; // No chunked index exists yet
         }
         // Find candidate chunks using zone maps
         const candidateChunkIds = sparseIndex.findChunksForRange(min, max);
@@ -384,17 +398,14 @@ export class MetadataIndexManager {
     /**
      * Get roaring bitmap for a field-value pair without converting to UUIDs (v3.43.0)
      * This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      * @returns RoaringBitmap32 containing integer IDs, or null if no matches
      */
     async getBitmapFromChunks(field, value) {
-        // Load sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                return null; // No chunked index exists yet
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            return null; // No chunked index exists yet
         }
         // Find candidate chunks using zone maps and bloom filters
         const normalizedValue = this.normalizeValue(value, field);
@@ -481,25 +492,22 @@ export class MetadataIndexManager {
     }
     /**
      * Add value-ID mapping to chunked index
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async addToChunkedIndex(field, value, id) {
-        // Load or create sparse index
-        let sparseIndex = this.sparseIndices.get(field);
+        // Load or create sparse index via UnifiedCache (lazy loading)
+        let sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
-            sparseIndex = await this.loadSparseIndex(field);
-            if (!sparseIndex) {
-                // Create new sparse index
-                const stats = this.fieldStats.get(field);
-                const chunkSize = stats
-                    ? this.chunkingStrategy.getOptimalChunkSize({
-                        uniqueValues: stats.cardinality.uniqueValues,
-                        distribution: stats.cardinality.distribution,
-                        avgIdsPerValue: stats.cardinality.totalValues / Math.max(1, stats.cardinality.uniqueValues)
-                    })
-                    : 50;
-                sparseIndex = new SparseIndex(field, chunkSize);
-            }
-            this.sparseIndices.set(field, sparseIndex);
+            // Create new sparse index
+            const stats = this.fieldStats.get(field);
+            const chunkSize = stats
+                ? this.chunkingStrategy.getOptimalChunkSize({
+                    uniqueValues: stats.cardinality.uniqueValues,
+                    distribution: stats.cardinality.distribution,
+                    avgIdsPerValue: stats.cardinality.totalValues / Math.max(1, stats.cardinality.uniqueValues)
+                })
+                : 50;
+            sparseIndex = new SparseIndex(field, chunkSize);
         }
         const normalizedValue = this.normalizeValue(value, field);
         // Find existing chunk for this value (check zone maps)
@@ -571,9 +579,11 @@ export class MetadataIndexManager {
     }
     /**
      * Remove ID from chunked index
+     * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
      */
     async removeFromChunkedIndex(field, value, id) {
-        const sparseIndex = this.sparseIndices.get(field) || await this.loadSparseIndex(field);
+        // Load sparse index via UnifiedCache (lazy loading)
+        const sparseIndex = await this.loadSparseIndex(field);
         if (!sparseIndex) {
             return; // No chunked index exists
         }
@@ -834,21 +844,25 @@ export class MetadataIndexManager {
             }
         }
         else {
-            // Remove from all indexes (slower, requires scanning all chunks)
+            // Remove from all indexes (slower, requires scanning all field indexes)
             // This should be rare - prefer providing metadata when removing
-            prodLog.warn(`Removing ID ${id} without metadata requires scanning all sparse indices (slow)`);
-            // Scan all sparse indices
-            for (const [field, sparseIndex] of this.sparseIndices.entries()) {
-                for (const chunkId of sparseIndex.getAllChunkIds()) {
-                    const chunk = await this.chunkManager.loadChunk(field, chunkId);
-                    if (chunk) {
-                        // Convert UUID to integer for bitmap checking
-                        const intId = this.idMapper.getInt(id);
-                        if (intId !== undefined) {
-                            // Check all values in this chunk
-                            for (const [value, bitmap] of chunk.entries) {
-                                if (bitmap.has(intId)) {
-                                    await this.removeFromChunkedIndex(field, value, id);
+            // v3.44.1: Scan via fieldIndexes, load sparse indices on-demand
+            prodLog.warn(`Removing ID ${id} without metadata requires scanning all fields (slow)`);
+            // Scan all fields via fieldIndexes
+            for (const field of this.fieldIndexes.keys()) {
+                const sparseIndex = await this.loadSparseIndex(field);
+                if (sparseIndex) {
+                    for (const chunkId of sparseIndex.getAllChunkIds()) {
+                        const chunk = await this.chunkManager.loadChunk(field, chunkId);
+                        if (chunk) {
+                            // Convert UUID to integer for bitmap checking
+                            const intId = this.idMapper.getInt(id);
+                            if (intId !== undefined) {
+                                // Check all values in this chunk
+                                for (const [value, bitmap] of chunk.entries) {
+                                    if (bitmap.has(intId)) {
+                                        await this.removeFromChunkedIndex(field, value, id);
+                                    }
                                 }
                             }
                         }
@@ -1087,9 +1101,10 @@ export class MetadataIndexManager {
                         case 'exists':
                             if (operand) {
                                 // Get all IDs that have this field (any value) from chunked sparse index with roaring bitmaps (v3.43.0)
+                                // v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
                                 const allIntIds = new Set();
-                                // Load sparse index for this field
-                                const sparseIndex = this.sparseIndices.get(field) || await this.loadSparseIndex(field);
+                                // Load sparse index via UnifiedCache (lazy loading)
+                                const sparseIndex = await this.loadSparseIndex(field);
                                 if (sparseIndex) {
                                     // Iterate through all chunks for this field
                                     for (const chunkId of sparseIndex.getAllChunkIds()) {
@@ -1343,29 +1358,31 @@ export class MetadataIndexManager {
     }
     /**
      * Get index statistics with enhanced counting information
+     * v3.44.1: Sparse indices now lazy-loaded via UnifiedCache
+     * Note: This method may load sparse indices to calculate stats
      */
     async getStats() {
         const fields = new Set();
         let totalEntries = 0;
         let totalIds = 0;
-        // Collect stats from sparse indices (v3.42.0 - removed indexCache)
-        for (const [field, sparseIndex] of this.sparseIndices.entries()) {
+        // Collect stats from field indexes (lightweight - always in memory)
+        for (const field of this.fieldIndexes.keys()) {
             fields.add(field);
-            // Count entries and IDs from all chunks
-            for (const chunkId of sparseIndex.getAllChunkIds()) {
-                const chunk = await this.chunkManager.loadChunk(field, chunkId);
-                if (chunk) {
-                    totalEntries += chunk.entries.size;
-                    for (const ids of chunk.entries.values()) {
-                        totalIds += ids.size;
+            // Load sparse index to count entries (may trigger lazy load)
+            const sparseIndex = await this.loadSparseIndex(field);
+            if (sparseIndex) {
+                // Count entries and IDs from all chunks
+                for (const chunkId of sparseIndex.getAllChunkIds()) {
+                    const chunk = await this.chunkManager.loadChunk(field, chunkId);
+                    if (chunk) {
+                        totalEntries += chunk.entries.size;
+                        for (const ids of chunk.entries.values()) {
+                            totalIds += ids.size;
+                        }
                     }
                 }
             }
         }
-        // Also include fields from fieldIndexes that might not have sparse indices yet
-        for (const field of this.fieldIndexes.keys()) {
-            fields.add(field);
-        }
         return {
             totalEntries,
             totalIds,
@@ -1377,6 +1394,7 @@ export class MetadataIndexManager {
     /**
      * Rebuild entire index from scratch using pagination
      * Non-blocking version that yields control back to event loop
+     * v3.44.1: Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map)
      */
     async rebuild() {
         if (this.isRebuilding)
@@ -1387,9 +1405,12 @@ export class MetadataIndexManager {
             prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`);
             prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`);
             // Clear existing indexes (v3.42.0 - use sparse indices instead of flat files)
-            this.sparseIndices.clear();
+            // v3.44.1: No sparseIndices Map to clear - UnifiedCache handles eviction
             this.fieldIndexes.clear();
             this.dirtyFields.clear();
+            // Clear all cached sparse indices in UnifiedCache
+            // This ensures rebuild starts fresh (v3.44.1)
+            this.unifiedCache.clear('metadata');
             // Rebuild noun metadata indexes using pagination
             let nounOffset = 0;
             const nounLimit = 25; // Even smaller batches during initialization to prevent socket exhaustion

package/dist/vfs/ConceptSystem.d.ts

@@ -0,0 +1,203 @@
+/**
+ * Universal Concept System for VFS
+ *
+ * Manages concepts that transcend files and exist independently
+ * Ideas that can be linked to multiple manifestations across domains
+ * PRODUCTION-READY: Real implementation using Brainy
+ */
+import { Brainy } from '../brainy.js';
+import { EntityManager, ManagedEntity } from './EntityManager.js';
+/**
+ * Universal concept that exists independently of files
+ */
+export interface UniversalConcept extends ManagedEntity {
+    id: string;
+    name: string;
+    description?: string;
+    domain: string;
+    category: string;
+    keywords: string[];
+    links: ConceptLink[];
+    manifestations: ConceptManifestation[];
+    strength: number;
+    created: number;
+    lastUpdated: number;
+    version: number;
+    metadata: Record<string, any>;
+    conceptType?: string;
+}
+/**
+ * A link between concepts
+ */
+export interface ConceptLink {
+    id: string;
+    targetConceptId: string;
+    relationship: 'extends' | 'implements' | 'uses' | 'opposite' | 'related' | 'contains' | 'part-of';
+    strength: number;
+    context?: string;
+    bidirectional: boolean;
+}
+/**
+ * A manifestation of a concept in a specific location
+ */
+export interface ConceptManifestation extends ManagedEntity {
+    id: string;
+    conceptId: string;
+    filePath: string;
+    context: string;
+    form: 'definition' | 'usage' | 'example' | 'discussion' | 'implementation';
+    position?: {
+        line?: number;
+        column?: number;
+        offset?: number;
+    };
+    confidence: number;
+    timestamp: number;
+    extractedBy: 'manual' | 'auto' | 'ai';
+}
+/**
+ * Configuration for concept system
+ */
+export interface ConceptSystemConfig {
+    autoLink?: boolean;
+    similarityThreshold?: number;
+    maxManifestations?: number;
+    strengthDecay?: number;
+}
+/**
+ * Concept graph structure for visualization
+ */
+export interface ConceptGraph {
+    concepts: Array<{
+        id: string;
+        name: string;
+        domain: string;
+        strength: number;
+        manifestationCount: number;
+    }>;
+    links: Array<{
+        source: string;
+        target: string;
+        relationship: string;
+        strength: number;
+    }>;
+}
+/**
+ * Universal Concept System
+ *
+ * Manages concepts that exist independently of any specific file or context
+ * Examples:
+ * - "Authentication" concept appearing in docs, code, tests
+ * - "Customer Journey" concept in marketing, UX, analytics
+ * - "Dependency Injection" pattern across multiple codebases
+ * - "Sustainability" theme in various research papers
+ */
+export declare class ConceptSystem extends EntityManager {
+    private config;
+    private conceptCache;
+    constructor(brain: Brainy, config?: ConceptSystemConfig);
+    /**
+     * Create a new universal concept
+     */
+    createConcept(concept: Omit<UniversalConcept, 'id' | 'created' | 'lastUpdated' | 'version' | 'links' | 'manifestations'>): Promise<string>;
+    /**
+     * Find concepts by various criteria
+     */
+    findConcepts(query: {
+        name?: string;
+        domain?: string;
+        category?: string;
+        keywords?: string[];
+        similar?: string;
+        manifestedIn?: string;
+    }): Promise<UniversalConcept[]>;
+    /**
+     * Link two concepts together
+     */
+    linkConcept(fromConceptId: string, toConceptId: string, relationship: ConceptLink['relationship'], options?: {
+        strength?: number;
+        context?: string;
+        bidirectional?: boolean;
+    }): Promise<string>;
+    /**
+     * Record a manifestation of a concept in a file
+     */
+    recordManifestation(conceptId: string, filePath: string, context: string, form: ConceptManifestation['form'], options?: {
+        position?: ConceptManifestation['position'];
+        confidence?: number;
+        extractedBy?: ConceptManifestation['extractedBy'];
+    }): Promise<string>;
+    /**
+     * Extract and link concepts from content
+     */
+    extractAndLinkConcepts(filePath: string, content: Buffer): Promise<string[]>;
+    /**
+     * Get concept graph for visualization
+     */
+    getConceptGraph(options?: {
+        domain?: string;
+        minStrength?: number;
+        maxConcepts?: number;
+    }): Promise<ConceptGraph>;
+    /**
+     * Find appearances of a concept
+     */
+    findAppearances(conceptId: string, options?: {
+        filePath?: string;
+        form?: ConceptManifestation['form'];
+        minConfidence?: number;
+        limit?: number;
+    }): Promise<ConceptManifestation[]>;
+    /**
+     * Auto-link concept to similar concepts
+     */
+    private autoLinkConcept;
+    /**
+     * Get concept by ID
+     */
+    private getConcept;
+    /**
+     * Update stored concept
+     */
+    private updateConcept;
+    /**
+     * Calculate similarity between two concepts
+     */
+    private calculateConceptSimilarity;
+    /**
+     * Generate embedding for concept
+     */
+    private generateConceptEmbedding;
+    /**
+     * Generate embedding for text
+     */
+    private generateTextEmbedding;
+    /**
+     * Get reverse relationship type
+     */
+    private getReverseRelationship;
+    /**
+     * Map concept relationship to VerbType
+     */
+    private getVerbType;
+    /**
+     * Detect concept domain from context
+     */
+    private detectDomain;
+    /**
+     * Detect concept category
+     */
+    private detectCategory;
+    /**
+     * Detect manifestation form from context
+     */
+    private detectManifestationForm;
+    /**
+     * Extract context around a position
+     */
+    private extractContext;
+    /**
+     * Clear concept cache
+     */
+    clearCache(conceptId?: string): void;
+}