@soulcraft/cortex 1.3.1

package/dist/native/types.js

@@ -0,0 +1,16 @@
+/**
+ * Shared types for the native module integration.
+ *
+ * These types are used by both the napi-rs and bun:ffi loaders.
+ * Ported from src/embeddings/wasm/types.ts — keeps the same API surface.
+ */
+/**
+ * Model constants for all-MiniLM-L6-v2
+ */
+export const MODEL_CONSTANTS = {
+    HIDDEN_SIZE: 384,
+    MAX_SEQUENCE_LENGTH: 256,
+    VOCAB_SIZE: 30522,
+    MODEL_NAME: 'all-MiniLM-L6-v2',
+};
+//# sourceMappingURL=types.js.map

package/dist/plugin.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @soulcraft/cortex Plugin Entry Point
+ *
+ * Implements the BrainyPlugin interface to register all native providers.
+ * Auto-detected by brainy during init() via dynamic import.
+ *
+ * Storage adapters are ALWAYS registered (even without a license) so users
+ * can always access their data. Compute acceleration requires a valid license.
+ *
+ * Provider registration order:
+ * 1. storage:mmap-filesystem — ALWAYS registered (data access)
+ * 2. distance    — SIMD-accelerated cosine (licensed)
+ * 3. metadataIndex — Native Rust query/mutation engine (licensed)
+ * 4. graphIndex    — Native 4 LSM-trees with verb tracking (licensed)
+ * 5. embeddings    — Candle ML native engine (CPU/CUDA/Metal) (licensed)
+ * 6. roaring       — CRoaring bitmap bindings (licensed)
+ * 7. msgpack       — Native encode/decode (licensed)
+ */
+import type { BrainyPlugin } from '@soulcraft/brainy/plugin';
+declare const cortexPlugin: BrainyPlugin;
+export default cortexPlugin;
+//# sourceMappingURL=plugin.d.ts.map

package/dist/plugin.js

@@ -0,0 +1,115 @@
+/**
+ * @soulcraft/cortex Plugin Entry Point
+ *
+ * Implements the BrainyPlugin interface to register all native providers.
+ * Auto-detected by brainy during init() via dynamic import.
+ *
+ * Storage adapters are ALWAYS registered (even without a license) so users
+ * can always access their data. Compute acceleration requires a valid license.
+ *
+ * Provider registration order:
+ * 1. storage:mmap-filesystem — ALWAYS registered (data access)
+ * 2. distance    — SIMD-accelerated cosine (licensed)
+ * 3. metadataIndex — Native Rust query/mutation engine (licensed)
+ * 4. graphIndex    — Native 4 LSM-trees with verb tracking (licensed)
+ * 5. embeddings    — Candle ML native engine (CPU/CUDA/Metal) (licensed)
+ * 6. roaring       — CRoaring bitmap bindings (licensed)
+ * 7. msgpack       — Native encode/decode (licensed)
+ */
+import { loadNativeModule, isNativeAvailable } from './native/index.js';
+import { validateLicense } from './license.js';
+const cortexPlugin = {
+    name: '@soulcraft/cortex',
+    async activate(context) {
+        // Storage adapters are ALWAYS registered — users must always be able
+        // to access their data, even if the license expires. Brainy falls back
+        // to JS compute but needs the storage adapter to read the files.
+        const { MmapFileSystemStorage } = await import('./storage/mmapFileSystemStorage.js');
+        context.registerProvider('storage:mmap-filesystem', {
+            name: 'mmap-filesystem',
+            create: (config) => new MmapFileSystemStorage(config.rootDirectory, {
+                compression: config.compression,
+                compressionLevel: config.compressionLevel
+            })
+        });
+        // Gate: native module must be available for compute acceleration
+        if (!isNativeAvailable()) {
+            console.log('[cortex] Native module not available — using JavaScript fallback for compute');
+            return true; // storage registered, no native acceleration
+        }
+        // Gate: license must be valid for compute acceleration
+        if (!await validateLicense()) {
+            console.log('[cortex] Native acceleration disabled — using JavaScript fallback for compute');
+            return true; // storage registered, no native acceleration
+        }
+        const native = loadNativeModule();
+        // Distance: SIMD-accelerated cosine
+        // This is the highest-impact provider — every HNSW search uses it
+        context.registerProvider('distance', native.cosineDistance);
+        // Quantized distance: SQ8 cosine distance on uint8 arrays (no dequantization)
+        context.registerProvider('distance:sq8', native.cosineDistanceSq8);
+        context.registerProvider('distance:sq8:batch', native.cosineDistanceSq8Batch);
+        // Quantized distance: SQ4 cosine distance on nibble-packed arrays
+        context.registerProvider('distance:sq4', native.cosineDistanceSq4);
+        context.registerProvider('distance:sq4:batch', native.cosineDistanceSq4Batch);
+        // Quantization: native SQ8/SQ4 quantize/dequantize
+        context.registerProvider('quantization:sq8', {
+            quantize: native.quantizeSq8,
+            dequantize: native.dequantizeSq8,
+            serialize: native.serializeSq8,
+            deserialize: native.deserializeSq8,
+        });
+        context.registerProvider('quantization:sq4', {
+            quantize: native.quantizeSq4,
+            dequantize: native.dequantizeSq4,
+        });
+        // Graph compression: delta-varint encoded connection lists
+        context.registerProvider('graph:compression', {
+            encode: native.encodeConnections,
+            decode: native.decodeConnections,
+        });
+        // Product Quantization: 16-32x compression for large datasets
+        context.registerProvider('quantization:pq', native.NativePqCodebook);
+        // Metadata index: Native Rust query/mutation engine
+        const { MetadataIndexManager: NativeMetadataIndex } = await import('./utils/NativeMetadataIndex.js');
+        context.registerProvider('metadataIndex', (storage) => new NativeMetadataIndex(storage));
+        // Graph adjacency: Native 4 LSM-trees with verb tracking
+        const { GraphAdjacencyIndex: NativeGraphAdjacencyIndex } = await import('./graph/NativeGraphAdjacencyIndex.js');
+        context.registerProvider('graphIndex', (storage) => new NativeGraphAdjacencyIndex(storage));
+        // Embeddings: Candle ML native engine
+        const { NativeEmbeddingEngine } = await import('./native/NativeEmbeddingEngine.js');
+        const engine = NativeEmbeddingEngine.getInstance();
+        context.registerProvider('embeddings', async (text) => {
+            if (!engine.isInitialized()) {
+                await engine.initialize();
+            }
+            if (Array.isArray(text)) {
+                return engine.embedBatch(text);
+            }
+            return engine.embed(text);
+        });
+        // Roaring bitmaps: CRoaring bindings (binary-compatible with roaring-wasm)
+        const { RoaringBitmap32 } = await import('./native/NativeRoaringBitmap32.js');
+        context.registerProvider('roaring', RoaringBitmap32);
+        // Msgpack: Native encode/decode
+        context.registerProvider('msgpack', {
+            encode: native.msgpackEncode,
+            decode: native.msgpackDecode
+        });
+        // Mmap Vector Store: zero-copy vector file (mmap-filesystem only)
+        context.registerProvider('vectorStore:mmap', native.NativeMmapVectorStore);
+        // Batch I/O: direct file reads without mmap (reranking, fallback)
+        context.registerProvider('io:batchReadVectors', native.batchReadVectorsDirect);
+        context.registerProvider('io:vectorFileInfo', native.readVectorFileInfo);
+        // Graph-aware compaction: BFS traversal order for disk locality
+        context.registerProvider('compaction:bfsOrder', native.computeBfsOrder);
+        context.registerProvider('compaction:hnswOrder', native.computeHnswTraversalOrder);
+        return true;
+    },
+    async deactivate() {
+        // Native resources are cleaned up by the Rust drop() mechanism
+        // when the process exits. No explicit cleanup needed.
+    }
+};
+export default cortexPlugin;
+//# sourceMappingURL=plugin.js.map

package/dist/storage/mmapFileSystemStorage.d.ts

@@ -0,0 +1,24 @@
+/**
+ * MmapFileSystemStorage — FileSystemStorage with binary blob support for mmap SSTables
+ *
+ * Extends FileSystemStorage with saveBinaryBlob/loadBinaryBlob/deleteBinaryBlob/getBinaryBlobPath
+ * methods that enable the NativeGraphAdjacencyIndex to use adapter-controlled mmap I/O.
+ *
+ * Binary blobs are stored in a _blobs/ subdirectory with atomic writes (tmp + rename).
+ * getBinaryBlobPath returns the filesystem path so Rust can mmap the file directly.
+ */
+import { FileSystemStorage } from '@soulcraft/brainy';
+export declare class MmapFileSystemStorage extends FileSystemStorage {
+    private blobsDir;
+    constructor(rootDirectory: string, options?: {
+        compression?: boolean;
+        compressionLevel?: number;
+    });
+    init(): Promise<void>;
+    private blobPath;
+    saveBinaryBlob(key: string, data: Buffer): Promise<void>;
+    loadBinaryBlob(key: string): Promise<Buffer | null>;
+    deleteBinaryBlob(key: string): Promise<void>;
+    getBinaryBlobPath(key: string): string | null;
+}
+//# sourceMappingURL=mmapFileSystemStorage.d.ts.map

package/dist/storage/mmapFileSystemStorage.js

@@ -0,0 +1,73 @@
+/**
+ * MmapFileSystemStorage — FileSystemStorage with binary blob support for mmap SSTables
+ *
+ * Extends FileSystemStorage with saveBinaryBlob/loadBinaryBlob/deleteBinaryBlob/getBinaryBlobPath
+ * methods that enable the NativeGraphAdjacencyIndex to use adapter-controlled mmap I/O.
+ *
+ * Binary blobs are stored in a _blobs/ subdirectory with atomic writes (tmp + rename).
+ * getBinaryBlobPath returns the filesystem path so Rust can mmap the file directly.
+ */
+import { FileSystemStorage } from '@soulcraft/brainy';
+// Node.js modules - loaded lazily
+let fs;
+let fsPromises;
+let pathMod;
+let modulesLoaded = false;
+async function ensureModules() {
+    if (modulesLoaded)
+        return;
+    const [fsModule, pathModule] = await Promise.all([
+        import('node:fs'),
+        import('node:path'),
+    ]);
+    fs = fsModule;
+    fsPromises = fsModule.promises;
+    pathMod = pathModule;
+    modulesLoaded = true;
+}
+export class MmapFileSystemStorage extends FileSystemStorage {
+    blobsDir;
+    constructor(rootDirectory, options) {
+        super(rootDirectory, options);
+        this.blobsDir = rootDirectory + '/_blobs';
+    }
+    async init() {
+        await super.init();
+        await ensureModules();
+        // Re-resolve with proper path.join after modules are loaded
+        this.blobsDir = pathMod.join(this.rootDir, '_blobs');
+        await fsPromises.mkdir(this.blobsDir, { recursive: true });
+    }
+    blobPath(key) {
+        // key: "graph-lsm/source/sstable-123"
+        // path: {rootDir}/_blobs/graph-lsm/source/sstable-123.bin
+        return pathMod.join(this.blobsDir, ...key.split('/')) + '.bin';
+    }
+    async saveBinaryBlob(key, data) {
+        const filePath = this.blobPath(key);
+        await fsPromises.mkdir(pathMod.dirname(filePath), { recursive: true });
+        const tmpPath = filePath + '.tmp';
+        await fsPromises.writeFile(tmpPath, data);
+        await fsPromises.rename(tmpPath, filePath);
+    }
+    async loadBinaryBlob(key) {
+        try {
+            return await fsPromises.readFile(this.blobPath(key));
+        }
+        catch {
+            return null;
+        }
+    }
+    async deleteBinaryBlob(key) {
+        try {
+            await fsPromises.unlink(this.blobPath(key));
+        }
+        catch {
+            /* ignore missing files */
+        }
+    }
+    getBinaryBlobPath(key) {
+        return this.blobPath(key);
+    }
+}
+//# sourceMappingURL=mmapFileSystemStorage.js.map

package/dist/utils/NativeMetadataIndex.d.ts

@@ -0,0 +1,185 @@
+/**
+ * NativeMetadataIndex — TypeScript wrapper around the Rust NativeMetadataIndex.
+ *
+ * Implements the same public API as the old MetadataIndexManager (3,721 lines),
+ * delegating core operations (query, mutation, normalization) to Rust.
+ *
+ * Architecture:
+ * - Rust owns: bitmap operations, filter evaluation, value normalization,
+ *   field extraction, chunk management, text search, entity ID mapping
+ * - TS owns: async storage I/O, lazy field loading, rebuild orchestration,
+ *   flush, getSortedIdsForFilter (loads sort values from storage)
+ *
+ * Buffer exchange pattern: TS loads data from storage, passes JSON to Rust.
+ * Rust operates in-memory, returns serialized state for TS to persist.
+ */
+import type { StorageAdapter } from '@soulcraft/brainy';
+import { NounType, VerbType } from '@soulcraft/brainy/types/graphTypes';
+export interface MetadataIndexEntry {
+    field: string;
+    value: string | number | boolean;
+    ids: Set<string>;
+    lastUpdated: number;
+}
+export interface FieldIndexData {
+    values: Record<string, number>;
+    lastUpdated: number;
+}
+export interface MetadataIndexStats {
+    totalEntries: number;
+    totalIds: number;
+    fieldsIndexed: string[];
+    lastRebuild: number;
+    indexSize: number;
+}
+export interface MetadataIndexConfig {
+    maxIndexSize?: number;
+    rebuildThreshold?: number;
+    autoOptimize?: boolean;
+    indexedFields?: string[];
+    excludeFields?: string[];
+}
+interface CardinalityInfo {
+    uniqueValues: number;
+    totalValues: number;
+    distribution: 'uniform' | 'skewed' | 'sparse';
+    updateFrequency: number;
+    lastAnalyzed: number;
+}
+interface FieldStats {
+    cardinality: CardinalityInfo;
+    queryCount: number;
+    rangeQueryCount: number;
+    exactQueryCount: number;
+    avgQueryTime: number;
+    indexType: 'hash';
+    normalizationStrategy?: 'none' | 'precision' | 'bucket';
+}
+/**
+ * MetadataIndexManager — native Rust implementation with TS storage bridge.
+ *
+ * Drop-in replacement for the old pure-TS MetadataIndexManager.
+ * All bitmap AND/OR/NOT operations execute in Rust without crossing FFI.
+ */
+export declare class MetadataIndexManager {
+    private storage;
+    private config;
+    private native;
+    private isRebuilding;
+    private lastFlushTime;
+    private autoFlushThreshold;
+    private dirtyFields;
+    private loadedFields;
+    private knownFields;
+    private fieldStats;
+    private typeFieldAffinity;
+    private totalEntitiesByType;
+    private entityCountsByTypeFixed;
+    private verbCountsByTypeFixed;
+    private unifiedCache;
+    private fieldTypeInference;
+    constructor(storage: StorageAdapter, config?: MetadataIndexConfig);
+    init(): Promise<void>;
+    private loadEntityIdMapper;
+    private saveEntityIdMapper;
+    private loadFieldRegistry;
+    private saveFieldRegistry;
+    /**
+     * Ensure a field's sparse index + all chunks are loaded into Rust.
+     * Matches the lazy-loading pattern of the old UnifiedCache approach.
+     */
+    private ensureFieldLoaded;
+    /**
+     * Ensure multiple fields are loaded (parallel).
+     */
+    private ensureFieldsLoaded;
+    /**
+     * Extract filter field names for lazy loading.
+     */
+    private extractFilterFields;
+    /**
+     * Persist dirty chunks, sparse indices, field indexes from a MutationResult.
+     */
+    private persistMutationResult;
+    private warmCache;
+    private warmCacheForTopTypes;
+    private lazyLoadCounts;
+    private syncTypeCountsToFixed;
+    private syncTypeCountsFromFixed;
+    getIds(field: string, value: any): Promise<string[]>;
+    getIdsForFilter(filter: any): Promise<string[]>;
+    private filterNeedsAllIds;
+    getIdsForMultipleFields(fieldValuePairs: Array<{
+        field: string;
+        value: any;
+    }>): Promise<string[]>;
+    getIdsForTextQuery(query: string): Promise<Array<{
+        id: string;
+        matchCount: number;
+    }>>;
+    getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise<string[]>;
+    getFieldValueForEntity(entityId: string, field: string): Promise<any>;
+    private denormalizeValue;
+    getFilterValues(field: string): Promise<string[]>;
+    getFilterFields(): Promise<string[]>;
+    addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean, deferWrites?: boolean): Promise<void>;
+    removeFromIndex(id: string, metadata?: any): Promise<void>;
+    private updateTypeTracking;
+    tokenize(text: string): string[];
+    hashWord(word: string): number;
+    extractTextContent(data: any): string;
+    getEntityCountByType(type: string): number;
+    getTotalEntityCount(): number;
+    getAllEntityCounts(): Map<string, number>;
+    getEntityCountByTypeEnum(type: NounType): number;
+    getVerbCountByTypeEnum(type: VerbType): number;
+    getTopNounTypes(n: number): NounType[];
+    getTopVerbTypes(n: number): VerbType[];
+    getAllNounTypeCounts(): Map<NounType, number>;
+    getAllVerbTypeCounts(): Map<VerbType, number>;
+    getVFSEntityCountByType(type: string): Promise<number>;
+    getAllVFSEntityCounts(): Promise<Map<string, number>>;
+    getTotalVFSEntityCount(): Promise<number>;
+    getCountForCriteria(field: string, value: any): Promise<number>;
+    getAllIds(): Promise<string[]>;
+    flush(): Promise<void>;
+    getStats(): Promise<MetadataIndexStats>;
+    validateConsistency(): Promise<{
+        healthy: boolean;
+        avgEntriesPerEntity: number;
+        entityCount: number;
+        indexEntryCount: number;
+        recommendation: string | null;
+    }>;
+    detectAndRepairCorruption(): Promise<void>;
+    rebuild(): Promise<void>;
+    /**
+     * During rebuild, persist all dirty data accumulated so far.
+     */
+    private flushRebuildDirty;
+    clearAllIndexData(): Promise<void>;
+    private getPersistedFieldList;
+    private deleteFieldChunks;
+    getFieldStatistics(): Promise<Map<string, FieldStats>>;
+    getFieldCardinality(field: string): Promise<CardinalityInfo | null>;
+    getFieldsWithCardinality(): Promise<Array<{
+        field: string;
+        cardinality: number;
+        distribution: string;
+    }>>;
+    getOptimalQueryPlan(filters: Record<string, any>): Promise<{
+        strategy: 'exact' | 'range' | 'hybrid';
+        fieldOrder: string[];
+        estimatedCost: number;
+    }>;
+    getFieldsForType(nounType: NounType): Promise<Array<{
+        field: string;
+        affinity: number;
+        occurrences: number;
+        totalEntities: number;
+    }>>;
+    getIdsForCriteria(criteria: Record<string, any>): Promise<string[]>;
+    private yieldToEventLoop;
+}
+export {};
+//# sourceMappingURL=NativeMetadataIndex.d.ts.map