@soulcraft/cortex 1.3.1 → 1.4.1

package/dist/hnsw/NativeHNSWWrapper.js

@@ -0,0 +1,515 @@
+/**
+ * NativeHNSWWrapper — Thin TS wrapper around the Rust HNSW graph engine
+ *
+ * Bridges brainy's HNSWIndex API to the native Rust NativeHNSWIndex.
+ * The Rust engine handles all graph operations (insert, search, remove) in-memory
+ * while this wrapper handles async storage I/O for persistence.
+ *
+ * Key design:
+ * - Uses addItemFull() for single-FFI-call inserts (returns node + neighbors + system)
+ * - Persistence modes: 'immediate' (safe) or 'deferred' (30-50x faster for cloud)
+ * - COW via native fork() (Arc-based copy-on-write in Rust)
+ * - rebuild() restores pre-computed graph from storage (O(N) vs O(N log N) rebuild)
+ */
+import { loadNativeModule } from '../native/index.js';
+import { getGlobalCache, prodLog } from '@soulcraft/brainy/internals';
+const DEFAULT_CONFIG = {
+    M: 16,
+    efConstruction: 200,
+    efSearch: 50,
+    ml: 16,
+};
+export class NativeHNSWWrapper {
+    native;
+    config;
+    distanceFunction;
+    storage;
+    persistMode;
+    dirtyNodes = new Set();
+    dirtySystem = false;
+    useParallelization = true;
+    unifiedCache;
+    // COW support
+    cowEnabled = false;
+    constructor(config = {}, distanceFunction, options = {}) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.distanceFunction = distanceFunction;
+        this.storage = options.storage || null;
+        this.persistMode = options.persistMode || 'immediate';
+        this.unifiedCache = getGlobalCache();
+        const bindings = loadNativeModule();
+        const nativeConfig = {
+            m: this.config.M,
+            efConstruction: this.config.efConstruction,
+            efSearch: this.config.efSearch,
+            ml: this.config.ml,
+        };
+        this.native = new bindings.NativeHNSWIndex(nativeConfig);
+        prodLog.info(`NativeHNSWWrapper initialized (M=${this.config.M}, ef=${this.config.efSearch}, ` +
+            `persist=${this.persistMode})`);
+    }
+    // ---------------------------------------------------------------------------
+    // Core CRUD
+    // ---------------------------------------------------------------------------
+    async addItem(item) {
+        if (!item)
+            throw new Error('Item is undefined or null');
+        if (!item.vector)
+            throw new Error('Vector is undefined or null');
+        const { id, vector } = item;
+        // Use addItemFull — single FFI call returns all persistence data
+        const result = this.native.addItemFull(id, vector);
+        if (this.persistMode === 'immediate' && this.storage) {
+            // Persist new node
+            await this.storage.saveHNSWData(result.id, result.nodeData).catch((e) => {
+                console.error(`[HNSW native] Failed to persist node ${result.id}:`, e);
+            });
+            // Persist modified neighbors
+            const neighborPromises = result.modifiedNeighbors.map(neighbor => this.storage.saveHNSWData(neighbor.id, {
+                level: neighbor.level,
+                connections: neighbor.connections,
+            }).catch((e) => {
+                console.error(`[HNSW native] Failed to persist neighbor ${neighbor.id}:`, e);
+            }));
+            await Promise.allSettled(neighborPromises);
+            // Persist system data
+            await this.storage.saveHNSWSystem(result.systemData).catch((e) => {
+                console.error('[HNSW native] Failed to persist system data:', e);
+            });
+        }
+        else if (this.persistMode === 'deferred') {
+            this.dirtyNodes.add(result.id);
+            for (const neighbor of result.modifiedNeighbors) {
+                this.dirtyNodes.add(neighbor.id);
+            }
+            this.dirtySystem = true;
+        }
+        return result.id;
+    }
+    async removeItem(id) {
+        if (!this.native.hasNode(id))
+            return false;
+        // Get neighbors before removal (they'll be modified)
+        const nodeData = this.native.getNodeData(id);
+        const neighborIds = new Set();
+        if (nodeData) {
+            for (const ids of Object.values(nodeData.connections)) {
+                for (const nid of ids) {
+                    neighborIds.add(nid);
+                }
+            }
+        }
+        const removed = this.native.removeItem(id);
+        if (!removed)
+            return false;
+        if (this.persistMode === 'immediate' && this.storage) {
+            // Persist updated neighbors (their connections changed)
+            const promises = Array.from(neighborIds).map(async (nid) => {
+                const nData = this.native.getNodeData(nid);
+                if (nData) {
+                    await this.storage.saveHNSWData(nid, nData).catch((e) => {
+                        console.error(`[HNSW native] Failed to persist neighbor ${nid} after removal:`, e);
+                    });
+                }
+            });
+            await Promise.allSettled(promises);
+            // Persist system data (entry point may have changed)
+            const sysData = this.native.getSystemData();
+            await this.storage.saveHNSWSystem(sysData).catch((e) => {
+                console.error('[HNSW native] Failed to persist system data after removal:', e);
+            });
+        }
+        else if (this.persistMode === 'deferred') {
+            for (const nid of neighborIds) {
+                this.dirtyNodes.add(nid);
+            }
+            // Remove the deleted node from dirty set (no need to persist it)
+            this.dirtyNodes.delete(id);
+            this.dirtySystem = true;
+        }
+        return true;
+    }
+    async search(queryVector, k = 10, filter, options) {
+        if (this.native.size() === 0)
+            return [];
+        if (!queryVector)
+            throw new Error('Query vector is undefined or null');
+        let results;
+        if (options?.candidateIds && options.candidateIds.length > 0) {
+            // Pre-filtered search: use native searchWithCandidates for optimal performance
+            // The Rust engine only traverses candidate nodes, avoiding unnecessary distance calculations
+            results = this.native.searchWithCandidates(queryVector, k, this.config.efSearch, options.candidateIds);
+        }
+        else if (filter) {
+            // Fallback: search with larger k and post-filter asynchronously
+            const overRetrieve = k * 5;
+            const rawResults = this.native.search(queryVector, overRetrieve, this.config.efSearch);
+            const filtered = [];
+            for (const r of rawResults) {
+                if (await filter(r.id)) {
+                    filtered.push(r);
+                    if (filtered.length >= k)
+                        break;
+                }
+            }
+            results = filtered;
+        }
+        else {
+            results = this.native.search(queryVector, k, this.config.efSearch);
+        }
+        // Convert NativeSearchResult[] to [string, number][] tuples
+        return results.map(r => [r.id, r.distance]);
+    }
+    // ---------------------------------------------------------------------------
+    // Persistence
+    // ---------------------------------------------------------------------------
+    async flush() {
+        if (!this.storage)
+            return 0;
+        if (this.dirtyNodes.size === 0 && !this.dirtySystem)
+            return 0;
+        const startTime = Date.now();
+        const nodeCount = this.dirtyNodes.size;
+        // Batch persist dirty nodes
+        if (this.dirtyNodes.size > 0) {
+            const batchSize = 50;
+            const nodeIds = Array.from(this.dirtyNodes);
+            for (let i = 0; i < nodeIds.length; i += batchSize) {
+                const batch = nodeIds.slice(i, i + batchSize);
+                const promises = batch.map(nodeId => {
+                    const nodeData = this.native.getNodeData(nodeId);
+                    if (!nodeData)
+                        return Promise.resolve(); // Node was deleted
+                    return this.storage.saveHNSWData(nodeId, nodeData).catch((e) => {
+                        console.error(`[HNSW native flush] Failed to persist node ${nodeId}:`, e);
+                    });
+                });
+                await Promise.allSettled(promises);
+            }
+            this.dirtyNodes.clear();
+        }
+        // Persist system data
+        if (this.dirtySystem) {
+            const sysData = this.native.getSystemData();
+            await this.storage.saveHNSWSystem(sysData).catch((e) => {
+                console.error('[HNSW native flush] Failed to persist system data:', e);
+            });
+            this.dirtySystem = false;
+        }
+        const duration = Date.now() - startTime;
+        if (nodeCount > 0) {
+            prodLog.info(`[HNSW native] Flushed ${nodeCount} dirty nodes in ${duration}ms`);
+        }
+        return nodeCount;
+    }
+    getDirtyNodeCount() {
+        return this.dirtyNodes.size;
+    }
+    getPersistMode() {
+        return this.persistMode;
+    }
+    // ---------------------------------------------------------------------------
+    // Rebuild from storage
+    // ---------------------------------------------------------------------------
+    async rebuild(options = {}) {
+        if (!this.storage) {
+            prodLog.warn('NativeHNSWWrapper rebuild skipped: no storage adapter configured');
+            return;
+        }
+        const batchSize = options.batchSize || 1000;
+        try {
+            // Step 1: Clear native index
+            this.native.clear();
+            // Step 2: Load system data
+            const systemData = await this.storage.getHNSWSystem();
+            if (systemData && systemData.entryPointId) {
+                this.native.setSystemData(systemData.entryPointId, systemData.maxLevel);
+            }
+            // Step 3: Load nouns from storage and restore into native engine
+            const storageType = this.storage?.constructor.name || '';
+            const isLocalStorage = storageType === 'FileSystemStorage' ||
+                storageType === 'MemoryStorage' ||
+                storageType === 'OPFSStorage' ||
+                storageType === 'MmapFileSystemStorage';
+            let loadedCount = 0;
+            let totalCount;
+            if (isLocalStorage) {
+                // Local: load all at once
+                const result = await this.storage.getNounsWithPagination({
+                    limit: 10000000
+                });
+                totalCount = result.totalCount || result.items.length;
+                for (const nounData of result.items) {
+                    try {
+                        const hnswData = await this.storage.getHNSWData(nounData.id);
+                        if (!hnswData)
+                            continue;
+                        this.native.restoreNode(nounData.id, nounData.vector, hnswData.level, hnswData.connections);
+                        loadedCount++;
+                    }
+                    catch (error) {
+                        console.error(`Failed to rebuild HNSW data for ${nounData.id}:`, error);
+                    }
+                }
+                if (options.onProgress && totalCount !== undefined) {
+                    options.onProgress(loadedCount, totalCount);
+                }
+                prodLog.info(`NativeHNSW: Loaded ${loadedCount.toLocaleString()} nodes at once (local storage)`);
+            }
+            else {
+                // Cloud: paginated loading
+                let hasMore = true;
+                let offset = 0;
+                while (hasMore) {
+                    const result = await this.storage.getNounsWithPagination({
+                        limit: batchSize,
+                        offset,
+                    });
+                    if (totalCount === undefined && result.totalCount !== undefined) {
+                        totalCount = result.totalCount;
+                    }
+                    for (const nounData of result.items) {
+                        try {
+                            const hnswData = await this.storage.getHNSWData(nounData.id);
+                            if (!hnswData)
+                                continue;
+                            this.native.restoreNode(nounData.id, nounData.vector, hnswData.level, hnswData.connections);
+                            loadedCount++;
+                        }
+                        catch (error) {
+                            console.error(`Failed to rebuild HNSW data for ${nounData.id}:`, error);
+                        }
+                    }
+                    if (options.onProgress && totalCount !== undefined) {
+                        options.onProgress(loadedCount, totalCount);
+                    }
+                    hasMore = result.hasMore;
+                    offset += batchSize;
+                }
+            }
+            // Step 4: Recover entry point if needed
+            if (this.native.size() > 0 && !this.native.getEntryPointId()) {
+                // Find highest level node from native
+                const allIds = this.native.getAllIds();
+                let bestId = null;
+                let bestLevel = -1;
+                for (const nid of allIds) {
+                    const nd = this.native.getNodeData(nid);
+                    if (nd && nd.level > bestLevel) {
+                        bestLevel = nd.level;
+                        bestId = nid;
+                    }
+                }
+                if (bestId) {
+                    this.native.setSystemData(bestId, bestLevel);
+                    prodLog.info(`NativeHNSW entry point recovered: ${bestId} at level ${bestLevel}`);
+                }
+            }
+            prodLog.info(`NativeHNSW index rebuilt: ${loadedCount.toLocaleString()} entities, ` +
+                `${this.native.getMaxLevel() + 1} levels, ` +
+                `entry point: ${this.native.getEntryPointId() || 'none'}`);
+        }
+        catch (error) {
+            prodLog.error('NativeHNSW rebuild failed:', error);
+            throw new Error(`Failed to rebuild NativeHNSW index: ${error}`);
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // COW
+    // ---------------------------------------------------------------------------
+    enableCOW(parent) {
+        this.cowEnabled = true;
+        this.native = parent.native.fork();
+        this.config = parent.config;
+        this.distanceFunction = parent.distanceFunction;
+        this.useParallelization = parent.useParallelization;
+        this.unifiedCache = parent.unifiedCache;
+        prodLog.info(`NativeHNSW COW enabled: ${this.native.size()} nodes (Arc-based fork)`);
+    }
+    setUseParallelization(useParallelization) {
+        this.useParallelization = useParallelization;
+    }
+    getUseParallelization() {
+        return this.useParallelization;
+    }
+    // ---------------------------------------------------------------------------
+    // Info / Introspection
+    // ---------------------------------------------------------------------------
+    size() {
+        return this.native.size();
+    }
+    clear() {
+        this.native.clear();
+        this.dirtyNodes.clear();
+        this.dirtySystem = false;
+    }
+    getEntryPointId() {
+        return this.native.getEntryPointId() || null;
+    }
+    getMaxLevel() {
+        return this.native.getMaxLevel();
+    }
+    getDimension() {
+        return this.native.getDimension() ?? null;
+    }
+    getConfig() {
+        return { ...this.config };
+    }
+    getDistanceFunction() {
+        return this.distanceFunction;
+    }
+    /**
+     * Get all nouns — builds HNSWNoun objects from native data.
+     * @deprecated Use getNounsPaginated() instead
+     */
+    getNouns() {
+        const result = new Map();
+        const allIds = this.native.getAllIds();
+        for (const id of allIds) {
+            const nodeData = this.native.getNodeData(id);
+            const vector = this.native.getVector(id);
+            if (!nodeData || !vector)
+                continue;
+            const connections = new Map();
+            for (const [levelStr, ids] of Object.entries(nodeData.connections)) {
+                connections.set(parseInt(levelStr, 10), new Set(ids));
+            }
+            result.set(id, {
+                id,
+                vector,
+                connections,
+                level: nodeData.level,
+            });
+        }
+        return result;
+    }
+    getNounsPaginated(options = {}) {
+        const offset = options.offset || 0;
+        const limit = options.limit || 100;
+        const allIds = this.native.getAllIds();
+        const totalCount = allIds.length;
+        // Build noun objects for the requested page
+        const items = new Map();
+        let added = 0;
+        let skipped = 0;
+        for (const id of allIds) {
+            const nodeData = this.native.getNodeData(id);
+            const vector = this.native.getVector(id);
+            if (!nodeData || !vector)
+                continue;
+            const connections = new Map();
+            for (const [levelStr, ids] of Object.entries(nodeData.connections)) {
+                connections.set(parseInt(levelStr, 10), new Set(ids));
+            }
+            const noun = { id, vector, connections, level: nodeData.level };
+            if (options.filter && !options.filter(noun))
+                continue;
+            if (skipped < offset) {
+                skipped++;
+                continue;
+            }
+            items.set(id, noun);
+            added++;
+            if (added >= limit)
+                break;
+        }
+        return {
+            items,
+            totalCount,
+            hasMore: offset + limit < totalCount,
+        };
+    }
+    getNodesAtLevel(level) {
+        const ids = this.native.getNodesAtLevel(level);
+        const nodes = [];
+        for (const id of ids) {
+            const nodeData = this.native.getNodeData(id);
+            const vector = this.native.getVector(id);
+            if (!nodeData || !vector)
+                continue;
+            const connections = new Map();
+            for (const [levelStr, nids] of Object.entries(nodeData.connections)) {
+                connections.set(parseInt(levelStr, 10), new Set(nids));
+            }
+            nodes.push({ id, vector, connections, level: nodeData.level });
+        }
+        return nodes;
+    }
+    getLevelStats() {
+        const nativeStats = this.native.getLevelStats();
+        return nativeStats.map(s => ({
+            level: s.level,
+            nodeCount: s.nodeCount,
+            avgConnections: s.avgConnections,
+        }));
+    }
+    getIndexHealth() {
+        const health = this.native.getIndexHealth();
+        return {
+            averageConnections: health.averageConnections,
+            layerDistribution: health.layerDistribution,
+            maxLayer: health.maxLayer,
+            totalNodes: health.totalNodes,
+        };
+    }
+    getCacheStats() {
+        const cacheStats = this.unifiedCache.getStats();
+        const entityCount = this.native.size();
+        const vectorDimension = this.native.getDimension() || 384;
+        const bytesPerVector = vectorDimension * 4;
+        const estimatedVectorMemoryMB = (entityCount * bytesPerVector) / (1024 * 1024);
+        const availableCacheMB = (cacheStats.maxSize * 0.8) / (1024 * 1024);
+        const vectorsInCache = cacheStats.typeCounts.hnsw || 0;
+        const hnswMemoryBytes = cacheStats.typeSizes.hnsw || 0;
+        const hnswAccessCount = cacheStats.typeAccessCounts.hnsw || 0;
+        const totalAccessCount = cacheStats.totalAccessCount;
+        const hnswAccessPercent = totalAccessCount > 0 ? (hnswAccessCount / totalAccessCount) * 100 : 0;
+        const hnswCachePercent = cacheStats.maxSize > 0 ? (hnswMemoryBytes / cacheStats.maxSize) * 100 : 0;
+        const fairnessViolation = hnswCachePercent > 90 && hnswAccessPercent < 10;
+        const hitRatePercent = (cacheStats.hitRate * 100) || 0;
+        const cachingStrategy = estimatedVectorMemoryMB < availableCacheMB ? 'preloaded' : 'on-demand';
+        const recommendations = [];
+        if (cachingStrategy === 'on-demand' && hitRatePercent < 50) {
+            recommendations.push(`Low cache hit rate (${hitRatePercent.toFixed(1)}%). Consider increasing UnifiedCache size`);
+        }
+        if (fairnessViolation) {
+            recommendations.push(`Fairness violation: HNSW using ${hnswCachePercent.toFixed(1)}% cache with only ${hnswAccessPercent.toFixed(1)}% access`);
+        }
+        if (recommendations.length === 0) {
+            recommendations.push('All metrics healthy - no action needed');
+        }
+        return {
+            cachingStrategy,
+            autoDetection: {
+                entityCount,
+                estimatedVectorMemoryMB: parseFloat(estimatedVectorMemoryMB.toFixed(2)),
+                availableCacheMB: parseFloat(availableCacheMB.toFixed(2)),
+                threshold: 0.8,
+                rationale: cachingStrategy === 'preloaded'
+                    ? `Vectors in native Rust memory (${estimatedVectorMemoryMB.toFixed(1)}MB)`
+                    : `Adaptive on-demand loading (${estimatedVectorMemoryMB.toFixed(1)}MB > ${availableCacheMB.toFixed(1)}MB threshold)`
+            },
+            unifiedCache: {
+                totalSize: cacheStats.totalSize,
+                maxSize: cacheStats.maxSize,
+                utilizationPercent: parseFloat((cacheStats.utilization * 100).toFixed(2)),
+                itemCount: cacheStats.itemCount,
+                hitRatePercent: parseFloat(hitRatePercent.toFixed(2)),
+                totalAccessCount: cacheStats.totalAccessCount,
+            },
+            hnswCache: {
+                vectorsInCache,
+                cacheKeyPrefix: 'hnsw:vector:',
+                estimatedMemoryMB: parseFloat((hnswMemoryBytes / (1024 * 1024)).toFixed(2)),
+            },
+            fairness: {
+                hnswAccessCount,
+                hnswAccessPercent: parseFloat(hnswAccessPercent.toFixed(2)),
+                totalAccessCount,
+                fairnessViolation,
+            },
+            recommendations,
+        };
+    }
+}
+//# sourceMappingURL=NativeHNSWWrapper.js.map

package/dist/plugin.d.ts

@@ -8,13 +8,17 @@
  * can always access their data. Compute acceleration requires a valid license.
  *
  * Provider registration order:
- * 1. storage:mmap-filesystem — ALWAYS registered (data access)
+ * 1. storage:filesystem — mmap-enhanced FileSystemStorage (ALWAYS registered)
  * 2. distance    — SIMD-accelerated cosine (licensed)
- * 3. metadataIndex — Native Rust query/mutation engine (licensed)
- * 4. graphIndex    — Native 4 LSM-trees with verb tracking (licensed)
- * 5. embeddings    — Candle ML native engine (CPU/CUDA/Metal) (licensed)
- * 6. roaring       — CRoaring bitmap bindings (licensed)
- * 7. msgpack       — Native encode/decode (licensed)
+ * 3. hnsw        — Native Rust HNSW graph engine (licensed)
+ * 4. cache       — Native Rust eviction engine with JS data storage (licensed)
+ * 5. metadataIndex — Native Rust query/mutation engine (licensed)
+ * 6. graphIndex    — Native 4 LSM-trees with verb tracking (licensed)
+ * 7. embeddings    — Candle ML native engine (CPU/CUDA/Metal) (licensed)
+ * 8. embedBatch    — Native batch embedding (single forward pass) (licensed)
+ * 9. entityIdMapper — Native UUID ↔ integer mapping (licensed)
+ * 10. roaring       — CRoaring bitmap bindings (licensed)
+ * 11. msgpack       — Native encode/decode (licensed)
  */
 import type { BrainyPlugin } from '@soulcraft/brainy/plugin';
 declare const cortexPlugin: BrainyPlugin;

package/dist/plugin.js

@@ -8,25 +8,33 @@
  * can always access their data. Compute acceleration requires a valid license.
  *
  * Provider registration order:
- * 1. storage:mmap-filesystem — ALWAYS registered (data access)
+ * 1. storage:filesystem — mmap-enhanced FileSystemStorage (ALWAYS registered)
  * 2. distance    — SIMD-accelerated cosine (licensed)
- * 3. metadataIndex — Native Rust query/mutation engine (licensed)
- * 4. graphIndex    — Native 4 LSM-trees with verb tracking (licensed)
- * 5. embeddings    — Candle ML native engine (CPU/CUDA/Metal) (licensed)
- * 6. roaring       — CRoaring bitmap bindings (licensed)
- * 7. msgpack       — Native encode/decode (licensed)
+ * 3. hnsw        — Native Rust HNSW graph engine (licensed)
+ * 4. cache       — Native Rust eviction engine with JS data storage (licensed)
+ * 5. metadataIndex — Native Rust query/mutation engine (licensed)
+ * 6. graphIndex    — Native 4 LSM-trees with verb tracking (licensed)
+ * 7. embeddings    — Candle ML native engine (CPU/CUDA/Metal) (licensed)
+ * 8. embedBatch    — Native batch embedding (single forward pass) (licensed)
+ * 9. entityIdMapper — Native UUID ↔ integer mapping (licensed)
+ * 10. roaring       — CRoaring bitmap bindings (licensed)
+ * 11. msgpack       — Native encode/decode (licensed)
  */
 import { loadNativeModule, isNativeAvailable } from './native/index.js';
 import { validateLicense } from './license.js';
 const cortexPlugin = {
     name: '@soulcraft/cortex',
     async activate(context) {
-        // Storage adapters are ALWAYS registered — users must always be able
-        // to access their data, even if the license expires. Brainy falls back
-        // to JS compute but needs the storage adapter to read the files.
+        // Storage: Override built-in FileSystemStorage with mmap-enhanced version.
+        // MmapFileSystemStorage extends FileSystemStorage, adding binary blob methods
+        // (saveBinaryBlob/loadBinaryBlob/getBinaryBlobPath) that enable Rust native
+        // indexes to mmap SSTable files directly. Zero-config — users with
+        // type: 'filesystem' automatically get the enhanced version.
+        //
+        // ALWAYS registered (even without a license) so users can always access data.
         const { MmapFileSystemStorage } = await import('./storage/mmapFileSystemStorage.js');
-        context.registerProvider('storage:mmap-filesystem', {
-            name: 'mmap-filesystem',
+        context.registerProvider('storage:filesystem', {
+            name: 'filesystem',
             create: (config) => new MmapFileSystemStorage(config.rootDirectory, {
                 compression: config.compression,
                 compressionLevel: config.compressionLevel
@@ -70,6 +78,15 @@ const cortexPlugin = {
         });
         // Product Quantization: 16-32x compression for large datasets
         context.registerProvider('quantization:pq', native.NativePqCodebook);
+        // HNSW: Native Rust graph engine with SIMD distance and Arc-based COW
+        const { NativeHNSWWrapper } = await import('./hnsw/NativeHNSWWrapper.js');
+        context.registerProvider('hnsw', (config, distanceFn, options) => {
+            return new NativeHNSWWrapper(config, distanceFn, options);
+        });
+        // Cache: Native Rust eviction engine with JS data storage
+        // Registered before indexes so they use native cache for memory management
+        const { NativeUnifiedCacheWrapper } = await import('./utils/NativeUnifiedCache.js');
+        context.registerProvider('cache', new NativeUnifiedCacheWrapper());
         // Metadata index: Native Rust query/mutation engine
         const { MetadataIndexManager: NativeMetadataIndex } = await import('./utils/NativeMetadataIndex.js');
         context.registerProvider('metadataIndex', (storage) => new NativeMetadataIndex(storage));
@@ -88,6 +105,19 @@ const cortexPlugin = {
             }
             return engine.embed(text);
         });
+        // Batch embeddings: single Rust forward pass for multiple texts
+        // Brainy uses this for bulk operations (import, reindex, batch add)
+        // when available, bypassing the N-individual-calls path
+        context.registerProvider('embedBatch', async (texts) => {
+            if (!engine.isInitialized()) {
+                await engine.initialize();
+            }
+            return engine.embedBatch(texts);
+        });
+        // Entity ID mapper: Native UUID ↔ integer mapping in Rust (O(1) HashMap)
+        // Used by MetadataIndexManager for roaring bitmap integration
+        const { NativeEntityIdMapperWrapper } = await import('./utils/nativeEntityIdMapper.js');
+        context.registerProvider('entityIdMapper', (storage) => new NativeEntityIdMapperWrapper({ storage }));
         // Roaring bitmaps: CRoaring bindings (binary-compatible with roaring-wasm)
         const { RoaringBitmap32 } = await import('./native/NativeRoaringBitmap32.js');
         context.registerProvider('roaring', RoaringBitmap32);

package/dist/utils/NativeMetadataIndex.js

@@ -879,7 +879,7 @@ export class MetadataIndexManager {
             catch { }
             // Adaptive rebuild strategy
             const storageType = this.storage.constructor.name;
-            const isLocalStorage = storageType === 'FileSystemStorage' || storageType === 'MemoryStorage';
+            const isLocalStorage = storageType === 'FileSystemStorage' || storageType === 'MmapFileSystemStorage' || storageType === 'MemoryStorage';
             let totalNounsProcessed = 0;
             if (isLocalStorage) {
                 const result = await this.storage.getNouns({