@soulcraft/cor 3.0.0-rc.1

package/dist/native/NativeEmbeddingEngine.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Native Embedding Engine
+ *
+ * TypeScript wrapper matching the CandleEmbeddingEngine / WASMEmbeddingEngine API.
+ * Uses the native Rust module (napi-rs or bun:ffi) instead of WASM.
+ *
+ * Drop-in replacement: same singleton pattern, same method signatures,
+ * same model loading flow. Consumers don't need to change.
+ */
+import type { EmbeddingResult, EngineStats } from './types.js';
+/**
+ * Native embedding engine using Rust (Candle) via napi-rs.
+ *
+ * Replaces WASMEmbeddingEngine and CandleEmbeddingEngine.
+ * Same API surface for backward compatibility.
+ */
+export declare class NativeEmbeddingEngine {
+    private engine;
+    private initialized;
+    private embedCount;
+    private totalProcessingTimeMs;
+    private constructor();
+    /**
+     * Get the singleton instance
+     */
+    static getInstance(): NativeEmbeddingEngine;
+    /**
+     * Initialize the embedding engine
+     */
+    initialize(): Promise<void>;
+    /**
+     * Perform actual initialization.
+     *
+     * Loads the native module and model assets.
+     * Native module loads in <100ms (vs 3-5s for WASM compilation).
+     */
+    private performInit;
+    /**
+     * Load model assets from filesystem.
+     * Same resolution logic as the WASM modelLoader.ts.
+     */
+    private loadModelAssets;
+    private loadBunAssets;
+    private loadNodeAssets;
+    /**
+     * Generate embedding for text
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Generate embedding with metadata
+     */
+    embedWithMetadata(text: string): Promise<EmbeddingResult>;
+    /**
+     * Batch embed multiple texts
+     */
+    embedBatch(texts: string[]): Promise<number[][]>;
+    /**
+     * Check if initialized
+     */
+    isInitialized(): boolean;
+    /**
+     * Get engine statistics
+     */
+    getStats(): EngineStats;
+    /**
+     * Dispose and free resources
+     */
+    dispose(): Promise<void>;
+    /**
+     * Reset singleton (for testing)
+     */
+    static resetInstance(): void;
+}
+/**
+ * Calculate cosine similarity between two embeddings (native)
+ */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+export declare const nativeEmbeddingEngine: NativeEmbeddingEngine;
+//# sourceMappingURL=NativeEmbeddingEngine.d.ts.map

package/dist/native/NativeEmbeddingEngine.js

@@ -0,0 +1,318 @@
+/**
+ * Native Embedding Engine
+ *
+ * TypeScript wrapper matching the CandleEmbeddingEngine / WASMEmbeddingEngine API.
+ * Uses the native Rust module (napi-rs or bun:ffi) instead of WASM.
+ *
+ * Drop-in replacement: same singleton pattern, same method signatures,
+ * same model loading flow. Consumers don't need to change.
+ */
+import { MODEL_CONSTANTS } from './types.js';
+import { loadNativeModule } from './index.js';
+// Global singleton
+let globalInstance = null;
+let globalInitPromise = null;
+/**
+ * Native embedding engine using Rust (Candle) via napi-rs.
+ *
+ * Replaces WASMEmbeddingEngine and CandleEmbeddingEngine.
+ * Same API surface for backward compatibility.
+ */
+export class NativeEmbeddingEngine {
+    engine = null;
+    initialized = false;
+    embedCount = 0;
+    totalProcessingTimeMs = 0;
+    constructor() {
+        // Private constructor for singleton
+    }
+    /**
+     * Get the singleton instance
+     */
+    static getInstance() {
+        if (!globalInstance) {
+            globalInstance = new NativeEmbeddingEngine();
+        }
+        return globalInstance;
+    }
+    /**
+     * Initialize the embedding engine
+     */
+    async initialize() {
+        if (this.initialized) {
+            return;
+        }
+        if (globalInitPromise) {
+            await globalInitPromise;
+            return;
+        }
+        globalInitPromise = this.performInit();
+        try {
+            await globalInitPromise;
+        }
+        finally {
+            globalInitPromise = null;
+        }
+    }
+    /**
+     * Perform actual initialization.
+     *
+     * Loads the native module and model assets.
+     * Native module loads in <100ms (vs 3-5s for WASM compilation).
+     */
+    async performInit() {
+        const startTime = Date.now();
+        console.log('Initializing Native Embedding Engine...');
+        try {
+            // 1. Load native module
+            console.log('Loading native module...');
+            const native = loadNativeModule();
+            this.engine = new native.NativeEmbeddingEngine();
+            const nativeTime = Date.now() - startTime;
+            console.log(`   Native module loaded in ${nativeTime}ms`);
+            // 2. Load model assets (same flow as WASM version)
+            console.log('Loading model assets (~88MB)...');
+            const modelStartTime = Date.now();
+            const assets = await this.loadModelAssets();
+            const modelTime = Date.now() - modelStartTime;
+            console.log(`   Model loaded in ${modelTime}ms`);
+            // 3. Initialize engine with model
+            console.log('Initializing embedding engine...');
+            this.engine.load(assets.model, assets.tokenizer, assets.config);
+            if (!this.engine.isReady()) {
+                throw new Error('Engine failed to initialize');
+            }
+            this.initialized = true;
+            const initTime = Date.now() - startTime;
+            console.log(`Native Embedding Engine ready in ${initTime}ms`);
+        }
+        catch (error) {
+            this.initialized = false;
+            this.engine = null;
+            throw new Error(`Failed to initialize Native Embedding Engine: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Load model assets from filesystem.
+     * Same resolution logic as the WASM modelLoader.ts.
+     */
+    async loadModelAssets() {
+        const isBun = typeof Bun !== 'undefined';
+        if (isBun) {
+            return this.loadBunAssets();
+        }
+        return this.loadNodeAssets();
+    }
+    async loadBunAssets() {
+        const pathsToTry = [];
+        // Strategy 1: URL-resolved paths (works in Bun runtime)
+        try {
+            const modelPath = new URL('../../assets/models/all-MiniLM-L6-v2/model.safetensors', import.meta.url).pathname;
+            const tokenizerPath = new URL('../../assets/models/all-MiniLM-L6-v2/tokenizer.json', import.meta.url).pathname;
+            const configPath = new URL('../../assets/models/all-MiniLM-L6-v2/config.json', import.meta.url).pathname;
+            pathsToTry.push([modelPath, tokenizerPath, configPath]);
+        }
+        catch {
+            // skip
+        }
+        // Strategy 2: node_modules path (installed packages)
+        const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2';
+        pathsToTry.push([
+            `${nmPath}/model.safetensors`,
+            `${nmPath}/tokenizer.json`,
+            `${nmPath}/config.json`,
+        ]);
+        // Strategy 3: assets folder relative to CWD (local development)
+        pathsToTry.push([
+            './assets/models/all-MiniLM-L6-v2/model.safetensors',
+            './assets/models/all-MiniLM-L6-v2/tokenizer.json',
+            './assets/models/all-MiniLM-L6-v2/config.json',
+        ]);
+        for (const [modelPath, tokenizerPath, configPath] of pathsToTry) {
+            try {
+                const [model, tokenizer, config] = await Promise.all([
+                    Bun.file(modelPath).arrayBuffer(),
+                    Bun.file(tokenizerPath).arrayBuffer(),
+                    Bun.file(configPath).arrayBuffer(),
+                ]);
+                if (model.byteLength > 0 && tokenizer.byteLength > 0 && config.byteLength > 0) {
+                    return {
+                        model: Buffer.from(model),
+                        tokenizer: Buffer.from(tokenizer),
+                        config: Buffer.from(config),
+                    };
+                }
+            }
+            catch {
+                continue;
+            }
+        }
+        throw new Error('Could not load model assets. Ensure model files are accessible:\n' +
+            '  Option 1: npm install @soulcraft/brainy (includes assets)\n' +
+            '  Option 2: Copy assets/ folder to your working directory\n' +
+            '  Option 3: For bun --compile, use --asset flag');
+    }
+    async loadNodeAssets() {
+        const fs = await import('node:fs');
+        const nodePath = await import('node:path');
+        const { fileURLToPath } = await import('node:url');
+        const thisDir = nodePath.dirname(fileURLToPath(import.meta.url));
+        // Try multiple paths (same strategy order as Bun):
+        // 1. Relative to source file (works when running from brainy's own repo)
+        // 2. node_modules/@soulcraft/brainy (works when cor is installed as dependency)
+        // 3. CWD-relative assets/ (local development / symlinked)
+        const pathsToTry = [
+            nodePath.resolve(thisDir, '../../assets/models/all-MiniLM-L6-v2'),
+            nodePath.resolve(process.cwd(), 'node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2'),
+            nodePath.resolve(process.cwd(), 'assets/models/all-MiniLM-L6-v2'),
+        ];
+        for (const assetsDir of pathsToTry) {
+            if (!fs.existsSync(assetsDir))
+                continue;
+            try {
+                const [model, tokenizer, config] = await Promise.all([
+                    fs.promises.readFile(nodePath.join(assetsDir, 'model.safetensors')),
+                    fs.promises.readFile(nodePath.join(assetsDir, 'tokenizer.json')),
+                    fs.promises.readFile(nodePath.join(assetsDir, 'config.json')),
+                ]);
+                return {
+                    model: Buffer.from(model),
+                    tokenizer: Buffer.from(tokenizer),
+                    config: Buffer.from(config),
+                };
+            }
+            catch {
+                continue;
+            }
+        }
+        throw new Error(`Model assets not found. Tried:\n` +
+            pathsToTry.map(p => `  - ${p}`).join('\n') + '\n' +
+            `Ensure @soulcraft/brainy is installed correctly.`);
+    }
+    /**
+     * Generate embedding for text
+     */
+    async embed(text) {
+        const result = await this.embedWithMetadata(text);
+        return result.embedding;
+    }
+    /**
+     * Generate embedding with metadata
+     */
+    async embedWithMetadata(text) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (!this.engine) {
+            throw new Error('Engine not properly initialized');
+        }
+        try {
+            const startTime = Date.now();
+            const embedding = this.engine.embed(text);
+            const processingTimeMs = Date.now() - startTime;
+            this.embedCount++;
+            this.totalProcessingTimeMs += processingTimeMs;
+            return {
+                embedding,
+                tokenCount: 0, // Candle handles tokenization internally
+                processingTimeMs,
+            };
+        }
+        catch (error) {
+            console.error('Native embed failed, marking engine for re-initialization:', error);
+            this.initialized = false;
+            this.engine = null;
+            throw error;
+        }
+    }
+    /**
+     * Batch embed multiple texts
+     */
+    async embedBatch(texts) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (!this.engine) {
+            throw new Error('Engine not properly initialized');
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        try {
+            const embeddings = this.engine.embedBatch(texts);
+            this.embedCount += texts.length;
+            return embeddings;
+        }
+        catch (error) {
+            console.error('Native embedBatch failed, marking engine for re-initialization:', error);
+            this.initialized = false;
+            this.engine = null;
+            throw error;
+        }
+    }
+    /**
+     * Check if initialized
+     */
+    isInitialized() {
+        return this.initialized;
+    }
+    /**
+     * Get engine statistics
+     */
+    getStats() {
+        if (this.engine) {
+            return this.engine.getStats();
+        }
+        return {
+            initialized: this.initialized,
+            embedCount: this.embedCount,
+            totalProcessingTimeMs: this.totalProcessingTimeMs,
+            avgProcessingTimeMs: this.embedCount > 0 ? this.totalProcessingTimeMs / this.embedCount : 0,
+            modelName: MODEL_CONSTANTS.MODEL_NAME,
+        };
+    }
+    /**
+     * Dispose and free resources
+     */
+    async dispose() {
+        // napi-rs handles cleanup via GC; just clear references
+        this.engine = null;
+        this.initialized = false;
+    }
+    /**
+     * Reset singleton (for testing)
+     */
+    static resetInstance() {
+        if (globalInstance) {
+            globalInstance.dispose();
+        }
+        globalInstance = null;
+        globalInitPromise = null;
+    }
+}
+/**
+ * Calculate cosine similarity between two embeddings (native)
+ */
+export function cosineSimilarity(a, b) {
+    try {
+        const native = loadNativeModule();
+        return native.cosineSimilarity(a, b);
+    }
+    catch {
+        // Fallback to JS implementation
+        if (a.length !== b.length || a.length === 0)
+            return 0;
+        let dot = 0, normA = 0, normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dot += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        if (normA === 0 || normB === 0)
+            return 0;
+        return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+}
+// Export singleton access
+export const nativeEmbeddingEngine = NativeEmbeddingEngine.getInstance();
+//# sourceMappingURL=NativeEmbeddingEngine.js.map

package/dist/native/NativeRoaringBitmap32.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Native Roaring Bitmap - Drop-in replacement for roaring-wasm
+ *
+ * Wraps the native Rust CRoaring module to provide the EXACT same API
+ * as roaring-wasm's RoaringBitmap32.
+ *
+ * CRoaring portable serialization format is binary-identical to
+ * roaring-wasm's portable format. Existing persisted indexes are
+ * readable without migration.
+ *
+ * All roaring bitmap consumers import from src/utils/roaring/index.ts —
+ * that module swaps the implementation to use this native wrapper.
+ */
+/**
+ * RoaringBitmap32 - native drop-in replacement for roaring-wasm.
+ *
+ * Same API surface:
+ *   Instance: .size, .isEmpty, .add(u32), .delete(u32), .tryAdd(u32), .has(u32),
+ *             .toArray(), .serialize('portable'), .deserialize(buf, 'portable'),
+ *             .getSerializationSizeInBytes('portable'), [Symbol.iterator]
+ *   Static:   RoaringBitmap32.and(a, b), .or(a, b), .orMany(arr)
+ *   Constructors: new RoaringBitmap32(), new RoaringBitmap32([1,2,3])
+ */
+export declare class RoaringBitmap32 {
+    private inner;
+    constructor(values?: Iterable<number>);
+    /** Create from an existing native instance (internal use) */
+    private static fromNative;
+    /** Number of values in the bitmap */
+    get size(): number;
+    /** Whether the bitmap is empty */
+    get isEmpty(): boolean;
+    /** Add a value to the bitmap */
+    add(value: number): void;
+    /** Remove a value from the bitmap */
+    delete(value: number): void;
+    /** Add a value, return true if newly added */
+    tryAdd(value: number): boolean;
+    /** Check if the bitmap contains a value */
+    has(value: number): boolean;
+    /** Convert to array of u32 values (sorted) */
+    toArray(): number[];
+    /**
+     * Serialize to portable format.
+     * The format parameter is accepted for roaring-wasm API compatibility
+     * but native always uses portable format (binary-identical).
+     */
+    serialize(_format?: 'portable' | boolean): Buffer;
+    /**
+     * Deserialize from portable format buffer (instance method).
+     * Replaces the contents of this bitmap with the deserialized data.
+     * Matches roaring-wasm instance method API.
+     */
+    deserialize(buffer: Buffer | Uint8Array, _format?: 'portable' | boolean): void;
+    /**
+     * Get serialization size in bytes.
+     * The format parameter is accepted for roaring-wasm API compatibility.
+     */
+    getSerializationSizeInBytes(_format?: 'portable' | boolean): number;
+    /** Get minimum value (null if empty — napi maps Rust None → null, not undefined). */
+    minimum(): number | null;
+    /** Get maximum value (null if empty — napi maps Rust None → null, not undefined). */
+    maximum(): number | null;
+    /** Iterator support for for...of loops */
+    [Symbol.iterator](): IterableIterator<number>;
+    /** Deserialize from portable format buffer */
+    static deserialize(buffer: Buffer | Uint8Array, _format?: 'portable' | boolean): RoaringBitmap32;
+    /** Compute intersection (AND) of two bitmaps */
+    static and(a: RoaringBitmap32, b: RoaringBitmap32): RoaringBitmap32;
+    /** Compute union (OR) of two bitmaps */
+    static or(a: RoaringBitmap32, b: RoaringBitmap32): RoaringBitmap32;
+    /** Compute union (OR) of multiple bitmaps */
+    static orMany(bitmaps: RoaringBitmap32[]): RoaringBitmap32;
+    /** Compute difference (AND NOT) */
+    static andNot(a: RoaringBitmap32, b: RoaringBitmap32): RoaringBitmap32;
+    /** Compute symmetric difference (XOR) */
+    static xor(a: RoaringBitmap32, b: RoaringBitmap32): RoaringBitmap32;
+}
+/**
+ * RoaringBitmap32Iterator - for roaring-wasm API compatibility.
+ * Uses the native toArray() under the hood.
+ */
+export declare class RoaringBitmap32Iterator implements IterableIterator<number> {
+    private values;
+    private index;
+    constructor(bitmap: RoaringBitmap32);
+    next(): IteratorResult<number>;
+    [Symbol.iterator](): IterableIterator<number>;
+}
+/**
+ * roaringLibraryInitialize - no-op for native (ready at load time).
+ * Kept for API compatibility with roaring-wasm.
+ */
+export declare function roaringLibraryInitialize(): Promise<void>;
+/**
+ * roaringLibraryIsReady - always true for native.
+ * Kept for API compatibility with roaring-wasm.
+ */
+export declare function roaringLibraryIsReady(): boolean;
+/**
+ * Serialization format enum - for API compatibility with roaring-wasm.
+ */
+export declare const SerializationFormat: {
+    readonly portable: "portable";
+    readonly croaring: "croaring";
+};
+/**
+ * Deserialization format enum - for API compatibility with roaring-wasm.
+ */
+export declare const DeserializationFormat: {
+    readonly portable: "portable";
+    readonly croaring: "croaring";
+};
+//# sourceMappingURL=NativeRoaringBitmap32.d.ts.map