agentdb 3.0.0-alpha.8 → 3.0.0

package/dist/src/backends/ruvector/RuVectorBackend.js

@@ -1,432 +1,869 @@
 /**
- * RuVectorBackend - High-Performance Vector Storage (v0.1.99+)
+ * RuVectorBackend - High-Performance Vector Storage
  *
- * Implements VectorBackend using ruvector with native SIMD and multi-threading.
- * Updated for ruvector 0.1.99+ async API with object-style insert/search.
+ * Implements VectorBackend using @ruvector/core with optional GNN support.
+ * Provides <100µs search latency with native SIMD optimizations.
  *
  * Features:
- * - Native SIMD acceleration (2-4x faster vector ops)
- * - Automatic fallback when ruvector packages not installed
+ * - Automatic fallback when @ruvector packages not installed
  * - Separate metadata storage for rich queries
- * - Score-to-similarity conversion for all metrics
+ * - Distance-to-similarity conversion for all metrics
  * - Batch operations for optimal throughput
- * - Parallel batch search via searchBatch()
  * - Persistent storage with separate metadata files
+ * - Parallel batch insert with configurable concurrency
+ * - Buffer pooling to reduce memory allocations
+ * - Adaptive index parameters based on dataset size
+ * - Memory-mapped support for large indices
+ * - Statistics tracking for performance monitoring
  */
+// ============================================================================
+// Performance & Security Constants
+// ============================================================================
+/** @inline Maximum supported vector dimension for bounds checking */
+export const MAX_VECTOR_DIMENSION = 4096;
+/** @inline Maximum batch size for optimal cache utilization */
+export const MAX_BATCH_SIZE = 10000;
+/** @inline Default cache size for embedding storage */
+export const DEFAULT_CACHE_SIZE = 10000;
+/** @inline Small epsilon for numerical stability */
+const EPSILON = 1e-10;
+/** @inline Maximum metadata entries to prevent memory exhaustion */
+const MAX_METADATA_ENTRIES = 10_000_000;
+/** @inline Maximum path length for file operations */
+const MAX_PATH_LENGTH = 4096;
+/** Forbidden path patterns for security */
+const FORBIDDEN_PATH_PATTERNS = [
+    /\.\./, // Path traversal
+    /^\/etc\//i, // System config
+    /^\/proc\//i, // Process info
+    /^\/sys\//i, // System info
+    /^\/dev\//i, // Devices
+];
+// ============================================================================
+// Security Validation Helpers
+// ============================================================================
+/**
+ * Validate a file path is safe
+ */
+function validatePath(inputPath) {
+    if (!inputPath || typeof inputPath !== 'string') {
+        throw new Error('Path must be a non-empty string');
+    }
+    if (inputPath.length > MAX_PATH_LENGTH) {
+        throw new Error(`Path exceeds maximum length of ${MAX_PATH_LENGTH}`);
+    }
+    for (const pattern of FORBIDDEN_PATH_PATTERNS) {
+        if (pattern.test(inputPath)) {
+            throw new Error(`Path contains forbidden pattern`);
+        }
+    }
+}
+/**
+ * Validate vector dimension is within safe limits
+ */
+function validateDimension(dimension) {
+    if (!Number.isFinite(dimension) || dimension < 1 || dimension > MAX_VECTOR_DIMENSION) {
+        throw new Error(`Dimension must be between 1 and ${MAX_VECTOR_DIMENSION}`);
+    }
+}
+/**
+ * Semaphore for controlling concurrent operations
+ */
+class Semaphore {
+    permits;
+    waiting = [];
+    constructor(permits) {
+        this.permits = permits;
+    }
+    async acquire() {
+        if (this.permits > 0) {
+            this.permits--;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.waiting.push(resolve);
+        });
+    }
+    release() {
+        if (this.waiting.length > 0) {
+            const resolve = this.waiting.shift();
+            resolve();
+        }
+        else {
+            this.permits++;
+        }
+    }
+    get available() {
+        return this.permits;
+    }
+}
+/**
+ * Buffer pool for reusing Float32Array buffers
+ */
+class BufferPool {
+    pools = new Map();
+    maxPoolSize;
+    constructor(maxPoolSize = 100) {
+        this.maxPoolSize = maxPoolSize;
+    }
+    /**
+     * Acquire a buffer of the specified size
+     */
+    acquire(size) {
+        const pool = this.pools.get(size);
+        if (pool && pool.length > 0) {
+            return pool.pop();
+        }
+        return new Float32Array(size);
+    }
+    /**
+     * Release a buffer back to the pool
+     */
+    release(buffer) {
+        const size = buffer.length;
+        if (!this.pools.has(size)) {
+            this.pools.set(size, []);
+        }
+        const pool = this.pools.get(size);
+        if (pool.length < this.maxPoolSize) {
+            // Clear buffer before returning to pool
+            buffer.fill(0);
+            pool.push(buffer);
+        }
+    }
+    /**
+     * Clear all pooled buffers
+     */
+    clear() {
+        this.pools.clear();
+    }
+    /**
+     * Get pool statistics
+     */
+    getStats() {
+        let totalBuffers = 0;
+        let totalMemory = 0;
+        for (const [size, pool] of this.pools) {
+            totalBuffers += pool.length;
+            totalMemory += pool.length * size * 4; // Float32 = 4 bytes
+        }
+        return { totalBuffers, totalMemory };
+    }
+}
+/** ruvector-core distance metric names */
+const NATIVE_METRIC = {
+    cosine: 'Cosine',
+    l2: 'Euclidean',
+    ip: 'DotProduct',
+};
 export class RuVectorBackend {
     name = 'ruvector';
-    db; // VectorDB from ruvector 0.1.99+
+    db; // VectorDb from ruvector / @ruvector/core (every method is async)
     config;
     metadata = new Map();
     initialized = false;
-    learning = null;
-    nativeVersion = 'unknown';
-    isNativeImpl = false;
+    // Native calls are async; they run one at a time through this queue so a
+    // sync insert followed by a remove (or a search) observes them in order.
+    queue = Promise.resolve();
+    lastWriteError = null;
+    cachedCount = 0;
+    knownIds = new Set();
+    storeWasEmpty = true;
+    storagePath = '';
+    ownedTempDir = null;
+    defaultEfSearch;
+    VectorDbCtor;
+    nativeOptions = {};
+    // Concurrency control
+    semaphore;
+    bufferPool;
+    // Statistics tracking
+    stats = {
+        insertCount: 0,
+        insertTotalLatencyMs: 0,
+        insertMinLatencyMs: Infinity,
+        insertMaxLatencyMs: 0,
+        searchCount: 0,
+        searchTotalLatencyMs: 0,
+        searchMinLatencyMs: Infinity,
+        searchMaxLatencyMs: 0,
+        cacheHits: 0,
+        cacheMisses: 0,
+        lastMemoryUsage: 0,
+        indexRebuildCount: 0,
+    };
+    // Mmap support
+    mmapEnabled = false;
+    mmapBuffer = null;
     constructor(config) {
         // Handle both dimension and dimensions for backward compatibility
         const dimension = config.dimension ?? config.dimensions;
         if (!dimension) {
             throw new Error('Vector dimension is required (use dimension or dimensions)');
         }
+        // Validate dimension for security
+        validateDimension(dimension);
         // Store both forms for compatibility with different backends
         this.config = { ...config, dimension, dimensions: dimension };
+        // Initialize concurrency control with configurable limit (bounded for security)
+        const concurrency = Math.min(Math.max(1, config.parallelConcurrency ?? 4), 32);
+        this.semaphore = new Semaphore(concurrency);
+        // Initialize buffer pool with bounded size
+        const bufferPoolSize = Math.min(Math.max(1, config.bufferPoolSize ?? 100), 1000);
+        this.bufferPool = new BufferPool(bufferPoolSize);
+        // Mmap configuration - validate path if provided
+        this.mmapEnabled = config.enableMmap ?? false;
+        if (this.mmapEnabled && config.mmapPath) {
+            validatePath(config.mmapPath);
+        }
+    }
+    /**
+     * Get adaptive HNSW parameters based on expected dataset size
+     */
+    getAdaptiveParams(datasetSize) {
+        if (datasetSize < 1000) {
+            // Small dataset: lower M and efConstruction for faster builds
+            return { M: 8, efConstruction: 100, efSearch: 50 };
+        }
+        else if (datasetSize <= 100000) {
+            // Medium dataset: balanced parameters
+            return { M: 16, efConstruction: 200, efSearch: 100 };
+        }
+        else {
+            // Large dataset: higher M for better recall
+            return { M: 32, efConstruction: 400, efSearch: 200 };
+        }
+    }
+    /**
+     * Pick the native storage file: the configured path, or a fresh private
+     * temp directory that close() removes.
+     */
+    async resolveStoragePath() {
+        const pathModule = await import('path');
+        // ':memory:' would become a literal file of that name in the cwd
+        if (this.config.storagePath && this.config.storagePath !== ':memory:') {
+            validatePath(this.config.storagePath);
+            return pathModule.resolve(this.config.storagePath);
+        }
+        const fs = await import('fs');
+        const os = await import('os');
+        this.ownedTempDir = fs.mkdtempSync(pathModule.join(os.tmpdir(), 'agentdb-ruvector-'));
+        return pathModule.join(this.ownedTempDir, 'index.db');
+    }
+    removeOwnedTempDir() {
+        if (!this.ownedTempDir)
+            return;
+        const dir = this.ownedTempDir;
+        this.ownedTempDir = null;
+        import('fs')
+            .then((fs) => fs.rmSync(dir, { recursive: true, force: true }))
+            .catch(() => { });
+    }
+    /**
+     * Run a native operation after every earlier one has settled.
+     * A failure is recorded (and surfaced by flush()) without breaking the queue.
+     */
+    enqueue(op) {
+        const result = this.queue.then(op);
+        this.queue = result.then(() => undefined, (err) => {
+            this.lastWriteError = err;
+            console.error(`[RuVectorBackend] native operation failed: ${err?.message ?? err}`);
+        });
+        return result;
     }
     /**
-     * Initialize RuVector database with SIMD and multi-threading (0.1.99+)
+     * Write entries to the native index with upsert semantics. ruvector-core
+     * keeps a stale HNSW node when an existing id is re-inserted, so an id that
+     * may already be stored is deleted first.
+     */
+    async writeNative(entries) {
+        const latest = new Map();
+        for (const e of entries)
+            latest.set(e.id, e.vector);
+        let replaced = 0;
+        for (const id of latest.keys()) {
+            if (this.knownIds.has(id) || !this.storeWasEmpty) {
+                if (await this.db.delete(id))
+                    replaced++;
+            }
+        }
+        const batch = Array.from(latest, ([id, vector]) => ({ id, vector }));
+        if (batch.length === 1) {
+            await this.db.insert(batch[0]);
+        }
+        else {
+            await this.db.insertBatch(batch);
+        }
+        for (const { id } of batch)
+            this.knownIds.add(id);
+        this.cachedCount += batch.length - replaced;
+    }
+    async deleteNative(id) {
+        const removed = await this.db.delete(id);
+        this.knownIds.delete(id);
+        if (removed)
+            this.cachedCount--;
+        return removed;
+    }
+    validateEntry(id, embedding, metadata) {
+        if (!id || typeof id !== 'string') {
+            throw new Error('Vector ID must be a non-empty string');
+        }
+        if (!(embedding instanceof Float32Array) && !Array.isArray(embedding)) {
+            throw new Error('Embedding must be a Float32Array or array');
+        }
+        if (embedding.length !== this.config.dimension) {
+            throw new Error(`Vector dimension ${embedding.length} does not match expected ${this.config.dimension}`);
+        }
+        if (metadata && !this.metadata.has(id) && this.metadata.size >= MAX_METADATA_ENTRIES) {
+            throw new Error(`Metadata store has reached maximum capacity of ${MAX_METADATA_ENTRIES}`);
+        }
+        // Copy: callers (and insertWithPooledBuffer) may reuse the buffer before
+        // the queued native write runs.
+        return new Float32Array(embedding);
+    }
+    recordInsertLatency(startTime) {
+        if (this.config.enableStats === false)
+            return;
+        const latency = performance.now() - startTime;
+        this.stats.insertCount++;
+        this.stats.insertTotalLatencyMs += latency;
+        this.stats.insertMinLatencyMs = Math.min(this.stats.insertMinLatencyMs, latency);
+        this.stats.insertMaxLatencyMs = Math.max(this.stats.insertMaxLatencyMs, latency);
+    }
+    /**
+     * Initialize RuVector database with optional dependency handling
      */
     async initialize() {
         if (this.initialized)
             return;
         try {
-            // Try main ruvector package first (0.1.99+ with native SIMD)
+            // Try main ruvector package first (includes core, gnn, graph)
             let VectorDB;
-            let ruvectorModule;
             try {
-                ruvectorModule = await import('ruvector');
-                VectorDB = ruvectorModule.VectorDB || ruvectorModule.default?.VectorDB;
+                const ruvector = await import('ruvector');
+                VectorDB = ruvector.VectorDb || ruvector.VectorDB
+                    || ruvector.default?.VectorDb || ruvector.default?.VectorDB;
             }
             catch {
-                // Fallback to @ruvector/core for backward compatibility
-                ruvectorModule = await import('@ruvector/core');
-                VectorDB = ruvectorModule.VectorDB || ruvectorModule.default?.VectorDB;
+                // Fallback to @ruvector/core. Its canonical export is VectorDb;
+                // VectorDB is only an alias in some versions.
+                const core = await import('@ruvector/core');
+                VectorDB = core.VectorDb || core.VectorDB
+                    || core.default?.VectorDb || core.default?.VectorDB;
             }
             if (!VectorDB) {
-                throw new Error('Could not find VectorDB export in ruvector');
+                throw new Error('Could not find VectorDb export in ruvector or @ruvector/core');
             }
             // Handle both 'dimension' and 'dimensions' for backward compatibility
             const dimensions = this.config.dimension ?? this.config.dimensions;
             if (!dimensions) {
                 throw new Error('Vector dimension is required (use dimension or dimensions)');
             }
-            // RuVector 0.1.99+ constructor - uses 'dimensions' (plural)
-            this.db = new VectorDB({
-                dimensions: dimensions,
-                metric: this.config.metric,
-                maxElements: this.config.maxElements || 100000,
-                efConstruction: this.config.efConstruction || 200,
-                m: this.config.M || 16
-            });
-            // Detect native SIMD availability
-            if (ruvectorModule.isNative) {
-                this.isNativeImpl = ruvectorModule.isNative();
-            }
-            if (ruvectorModule.getVersion) {
-                const vInfo = ruvectorModule.getVersion();
-                this.nativeVersion = typeof vInfo === 'string' ? vInfo : vInfo?.version || 'unknown';
+            // Determine HNSW parameters (adaptive or explicit)
+            const maxElements = this.config.maxElements || 100000;
+            const useAdaptive = this.config.adaptiveParams !== false;
+            const adaptiveParams = useAdaptive ? this.getAdaptiveParams(maxElements) : null;
+            const m = this.config.M ?? adaptiveParams?.M ?? 16;
+            const efConstruction = this.config.efConstruction ?? adaptiveParams?.efConstruction ?? 200;
+            const efSearch = this.config.efSearch ?? adaptiveParams?.efSearch ?? 100;
+            this.storagePath = await this.resolveStoragePath();
+            // HNSW parameters must go under `hnswConfig`: the binding ignores
+            // top-level keys, and an omitted hnswConfig selects a flat index.
+            this.VectorDbCtor = VectorDB;
+            this.nativeOptions = {
+                dimensions,
+                distanceMetric: NATIVE_METRIC[this.config.metric] ?? NATIVE_METRIC.cosine,
+                hnswConfig: { m, efConstruction, efSearch, maxElements },
+            };
+            this.db = new VectorDB({ ...this.nativeOptions, storagePath: this.storagePath });
+            this.defaultEfSearch = efSearch;
+            this.cachedCount = await this.db.len();
+            this.storeWasEmpty = this.cachedCount === 0;
+            // Initialize memory-mapped storage if enabled and available
+            if (this.mmapEnabled && this.config.mmapPath) {
+                await this.initializeMmap();
             }
             this.initialized = true;
         }
         catch (error) {
+            this.removeOwnedTempDir();
             const errorMessage = error.message;
+            // Special handling for path validation errors (from ruvector package)
+            // When using :memory:, ruvector may reject it as a path traversal attempt
+            // This is expected and not critical - users should use file-based paths for ruvector persistence
             if (errorMessage.includes('Path traversal') || errorMessage.includes('Invalid path')) {
                 throw new Error(`RuVector does not support :memory: database paths.\n` +
                     `Use a file path instead, or RuVector will be skipped and fallback backend will be used.\n` +
                     `Original error: ${errorMessage}`);
             }
-            throw new Error(`RuVector initialization failed. Please install: npm install ruvector@latest\n` +
+            throw new Error(`RuVector initialization failed. Please install: npm install ruvector\n` +
                 `Or legacy packages: npm install @ruvector/core\n` +
                 `Error: ${errorMessage}`);
         }
     }
     /**
-     * Insert single vector with optional metadata.
-     * Uses ruvector 0.1.99+ object-style API: insert({ id, vector, metadata? })
+     * Initialize memory-mapped storage for large indices
      */
-    insert(id, embedding, metadata) {
-        this.ensureInitialized();
-        // ruvector 0.1.99+ uses object-style insert with async support
-        // The VectorBackend interface is sync, so we call without awaiting.
-        // The underlying native impl handles this via napi-rs sync path.
-        const entry = {
-            id,
-            vector: Array.from(embedding)
-        };
-        if (metadata) {
-            entry.metadata = metadata;
-            this.metadata.set(id, metadata);
+    async initializeMmap() {
+        if (!this.config.mmapPath)
+            return;
+        // Validate path for security
+        validatePath(this.config.mmapPath);
+        let fd = null;
+        try {
+            const fs = await import('fs');
+            const pathModule = await import('path');
+            // Check if mmap module is available (optional native dependency)
+            let mmapModule = null;
+            try {
+                // Dynamic import of optional native module - known safe module
+                const moduleName = 'mmap-io';
+                mmapModule = await import(moduleName);
+            }
+            catch {
+                // mmap-io not available, fall back to regular I/O
+                console.error('[RuVectorBackend] mmap-io not available, using standard I/O');
+                return;
+            }
+            if (!mmapModule)
+                return;
+            const mmapPath = pathModule.resolve(this.config.mmapPath);
+            // Re-validate resolved path
+            validatePath(mmapPath);
+            const mmap = mmapModule;
+            // Create or open the mmap file
+            if (fs.existsSync(mmapPath)) {
+                fd = fs.openSync(mmapPath, 'r+');
+                const stats = fs.fstatSync(fd);
+                // Validate file size to prevent excessive memory mapping
+                const maxMmapSize = 4 * 1024 * 1024 * 1024; // 4GB max
+                if (stats.size > maxMmapSize) {
+                    throw new Error(`Mmap file size ${stats.size} exceeds maximum of ${maxMmapSize}`);
+                }
+                this.mmapBuffer = mmap.map(stats.size, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED, fd, 0);
+                fs.closeSync(fd);
+                fd = null;
+            }
+            this.mmapEnabled = true;
         }
-        // ruvector 0.1.99+ insert returns a promise, but the native layer
-        // executes synchronously. We fire-and-forget for interface compat.
-        const result = this.db.insert(entry);
-        if (result && typeof result.catch === 'function') {
-            result.catch((err) => {
-                console.error(`[RuVectorBackend] Insert failed for ${id}: ${err.message}`);
-            });
+        catch (error) {
+            console.error(`[RuVectorBackend] Failed to initialize mmap: ${error.message}`);
+            this.mmapEnabled = false;
+        }
+        finally {
+            // Ensure file descriptor is closed even on error
+            if (fd !== null) {
+                try {
+                    const fs = await import('fs');
+                    fs.closeSync(fd);
+                }
+                catch {
+                    // Ignore close errors in cleanup
+                }
+            }
         }
     }
     /**
-     * Batch insert for optimal performance.
-     * Uses ruvector 0.1.99+ insertBatch([{ id, vector, metadata? }])
+     * Insert single vector with optional metadata.
+     *
+     * The native write is queued; searchAsync()/flush() wait for it. Use
+     * insertAsync() to await the write and observe its errors directly.
      */
-    insertBatch(items) {
+    insert(id, embedding, metadata) {
+        // Validation errors throw here; native write errors are recorded by
+        // enqueue() and surfaced by flush()
+        this.queueInsert(id, embedding, metadata).catch(() => { });
+    }
+    async insertAsync(id, embedding, metadata) {
+        await this.queueInsert(id, embedding, metadata);
+    }
+    queueInsert(id, embedding, metadata) {
         this.ensureInitialized();
-        const entries = items.map(item => {
-            const entry = {
-                id: item.id,
-                vector: Array.from(item.embedding)
-            };
-            if (item.metadata) {
-                entry.metadata = item.metadata;
-                this.metadata.set(item.id, item.metadata);
-            }
-            return entry;
-        });
-        const result = this.db.insertBatch(entries);
-        if (result && typeof result.catch === 'function') {
-            result.catch((err) => {
-                console.error(`[RuVectorBackend] Batch insert failed: ${err.message}`);
-            });
-        }
+        const startTime = this.config.enableStats !== false ? performance.now() : 0;
+        const vector = this.validateEntry(id, embedding, metadata);
+        this.setMetadata(id, metadata);
+        const write = this.enqueue(() => this.writeNative([{ id, vector }]));
+        this.recordInsertLatency(startTime);
+        return write;
     }
     /**
-     * Set a RuVectorLearning instance for GNN-enhanced search
+     * Batch insert (queued; see insert())
      */
-    setLearning(learning) {
-        this.learning = learning;
+    insertBatch(items) {
+        // Validation errors throw here; see insert()
+        this.queueBatch(items).catch(() => { });
+    }
+    async insertBatchAsync(items) {
+        await this.queueBatch(items);
+    }
+    queueBatch(items) {
+        this.ensureInitialized();
+        const entries = [];
+        for (const item of items) {
+            entries.push({ id: item.id, vector: this.validateEntry(item.id, item.embedding, item.metadata) });
+        }
+        for (const item of items)
+            this.setMetadata(item.id, item.metadata);
+        const writes = [];
+        for (let i = 0; i < entries.length; i += MAX_BATCH_SIZE) {
+            const chunk = entries.slice(i, i + MAX_BATCH_SIZE);
+            writes.push(this.enqueue(() => this.writeNative(chunk)));
+        }
+        return Promise.all(writes).then(() => undefined);
     }
     /**
-     * Get the current RuVectorLearning instance, if any
+     * Batch insert in chunks of `batchSize`.
+     *
+     * Native writes are serialized through the backend queue, so `concurrency`
+     * no longer changes throughput; it is kept for API compatibility.
      */
-    getLearning() {
-        return this.learning;
+    async insertBatchParallel(items, options) {
+        const batchSize = Math.max(1, options?.batchSize ?? 100);
+        for (let i = 0; i < items.length; i += batchSize) {
+            await this.insertBatchAsync(items.slice(i, i + batchSize));
+        }
     }
     /**
-     * Search for k-nearest neighbors with optional filtering and GNN enhancement.
-     * Uses ruvector 0.1.99+ search({ vector, k, efSearch?, filter? })
-     * Results contain { id, score, vector?, metadata? }
+     * Insert using buffer pooling to reduce allocations
+     *
+     * Acquires a buffer from the pool, copies the embedding data,
+     * performs the insert, and returns the buffer to the pool.
+     *
+     * @param id - Vector ID
+     * @param embedding - Vector data (can be regular array or Float32Array)
+     * @param metadata - Optional metadata
      */
-    search(query, k, options) {
+    insertWithPooledBuffer(id, embedding, metadata) {
         this.ensureInitialized();
-        // Build search query for ruvector 0.1.99+ API
-        const searchQuery = {
-            vector: Array.from(query),
-            k
-        };
-        if (options?.efSearch) {
-            searchQuery.efSearch = options.efSearch;
-        }
-        if (options?.filter) {
-            searchQuery.filter = options.filter;
-        }
-        // ruvector 0.1.99+ search is async but native layer is sync
-        let rawResults = [];
-        const searchPromise = this.db.search(searchQuery);
-        if (searchPromise && typeof searchPromise.then === 'function') {
-            // Synchronous extraction for interface compat - the native binding
-            // resolves immediately when using N-API sync path
-            // For truly async scenarios, use searchBatch() instead
-            let resolved = false;
-            searchPromise.then((r) => {
-                rawResults = r || [];
-                resolved = true;
-            });
-            // If promise resolved synchronously (common with native bindings)
-            if (!resolved) {
-                // Fallback: return empty and log warning
-                console.warn('[RuVectorBackend] Search returned async promise; use searchBatch() for async searches');
-                return [];
-            }
-        }
-        else {
-            rawResults = searchPromise || [];
-        }
-        // Convert ruvector 0.1.99+ results (score-based) to SearchResult format
-        let results = rawResults
-            .map((r) => ({
-            id: r.id,
-            distance: this.scoreToDistance(r.score),
-            similarity: r.score,
-            metadata: r.metadata || this.metadata.get(r.id)
-        }))
-            .filter((r) => {
-            if (options?.threshold && r.similarity < options.threshold) {
-                return false;
-            }
-            if (options?.filter && r.metadata) {
-                return Object.entries(options.filter).every(([key, value]) => r.metadata[key] === value);
+        const dimension = this.config.dimension;
+        // Acquire buffer from pool
+        const buffer = this.bufferPool.acquire(dimension);
+        try {
+            // Copy data to pooled buffer
+            if (embedding instanceof Float32Array) {
+                buffer.set(embedding);
             }
-            return true;
-        });
-        // Enhance with GNN if available
-        if (this.learning && this.learning.getState().initialized && results.length > 0) {
-            try {
-                const neighbors = [];
-                const weights = [];
-                for (const r of results) {
-                    weights.push(r.similarity);
-                    neighbors.push(query);
-                }
-                for (let i = 0; i < rawResults.length && i < results.length; i++) {
-                    const raw = rawResults[i];
-                    if (raw.vector) {
-                        const vec = raw.vector instanceof Float32Array
-                            ? raw.vector
-                            : new Float32Array(Object.values(raw.vector));
-                        neighbors[i] = vec;
-                    }
+            else {
+                for (let i = 0; i < dimension; i++) {
+                    buffer[i] = embedding[i] ?? 0;
                 }
-                const enhanced = this.learning.enhance(query, neighbors, weights);
-                if (enhanced && enhanced.length > 0) {
-                    const enhancedQuery = {
-                        vector: Array.from(enhanced),
-                        k
-                    };
-                    if (options?.efSearch)
-                        enhancedQuery.efSearch = options.efSearch;
-                    const enhancedPromise = this.db.search(enhancedQuery);
-                    let enhancedRaw = [];
-                    if (enhancedPromise && typeof enhancedPromise.then === 'function') {
-                        let resolved = false;
-                        enhancedPromise.then((r) => { enhancedRaw = r || []; resolved = true; });
-                        if (!resolved)
-                            return results;
-                    }
-                    else {
-                        enhancedRaw = enhancedPromise || [];
-                    }
-                    const enhancedResults = enhancedRaw
-                        .map((r) => ({
-                        id: r.id,
-                        distance: this.scoreToDistance(r.score),
-                        similarity: r.score,
-                        metadata: r.metadata || this.metadata.get(r.id)
-                    }))
-                        .filter((r) => {
-                        if (options?.threshold && r.similarity < options.threshold)
-                            return false;
-                        if (options?.filter && r.metadata) {
-                            return Object.entries(options.filter).every(([key, value]) => r.metadata[key] === value);
-                        }
-                        return true;
-                    });
-                    if (enhancedResults.length > 0) {
-                        results = enhancedResults;
-                    }
-                }
-            }
-            catch {
-                // Fall through to raw results if GNN enhancement fails
             }
+            // insert() copies the vector before queueing, so the buffer can be reused
+            this.insert(id, buffer, metadata);
         }
-        return results;
+        finally {
+            // Return buffer to pool
+            this.bufferPool.release(buffer);
+        }
+    }
+    /**
+     * Synchronous search is not possible: every ruvector-core call is async.
+     * Use searchAsync() (or searchVectors() from VectorBackend.ts).
+     */
+    search(_query, _k, _options) {
+        throw new Error('RuVectorBackend search is async-only. Use searchAsync() or the VectorBackendAsync interface.');
     }
     /**
-     * Batch search for parallel query processing (0.1.99+).
-     * Executes multiple searches concurrently for throughput.
+     * Search for k-nearest neighbors with optional filtering.
+     * Waits for queued writes so a preceding insert is visible.
      */
-    async searchBatch(queries, k, options) {
+    async searchAsync(query, k, options) {
         this.ensureInitialized();
-        const results = await Promise.all(queries.map(async (query) => {
-            const searchQuery = {
-                vector: Array.from(query),
-                k
-            };
-            if (options?.efSearch)
-                searchQuery.efSearch = options.efSearch;
-            if (options?.filter)
-                searchQuery.filter = options.filter;
-            const rawResults = await this.db.search(searchQuery);
-            return (rawResults || []).map((r) => ({
-                id: r.id,
-                distance: this.scoreToDistance(r.score),
-                similarity: r.score,
-                metadata: r.metadata || this.metadata.get(r.id)
-            })).filter((r) => {
-                if (options?.threshold && r.similarity < options.threshold)
-                    return false;
-                if (options?.filter && r.metadata) {
-                    return Object.entries(options.filter).every(([key, value]) => r.metadata[key] === value);
-                }
-                return true;
-            });
-        }));
+        await this.queue;
+        const startTime = this.config.enableStats !== false ? performance.now() : 0;
+        const hasFilter = !!options?.filter && Object.keys(options.filter).length > 0;
+        // Metadata filters are applied after the ANN search, so over-fetch.
+        const fetchK = hasFilter ? Math.min(k * 4, Math.max(k, this.cachedCount)) : k;
+        const raw = fetchK > 0
+            ? await this.db.search({
+                vector: query instanceof Float32Array ? query : new Float32Array(query),
+                k: fetchK,
+                efSearch: options?.efSearch ?? this.defaultEfSearch,
+            })
+            : [];
+        const results = [];
+        for (const r of raw) {
+            // ruvector-core reports a distance in `score` (0 = identical for cosine)
+            const distance = r.score;
+            const similarity = this.distanceToSimilarity(distance);
+            if (options?.threshold !== undefined && similarity < options.threshold)
+                continue;
+            const metadata = this.metadata.get(r.id);
+            if (hasFilter && !this.matchesFilter(metadata, options.filter))
+                continue;
+            results.push({ id: r.id, distance, similarity, metadata });
+            if (results.length >= k)
+                break;
+        }
+        if (this.config.enableStats !== false) {
+            const latency = performance.now() - startTime;
+            this.stats.searchCount++;
+            this.stats.searchTotalLatencyMs += latency;
+            this.stats.searchMinLatencyMs = Math.min(this.stats.searchMinLatencyMs, latency);
+            this.stats.searchMaxLatencyMs = Math.max(this.stats.searchMaxLatencyMs, latency);
+        }
         return results;
     }
+    matchesFilter(metadata, filter) {
+        if (!metadata)
+            return false;
+        for (const [key, value] of Object.entries(filter)) {
+            if (metadata[key] !== value)
+                return false;
+        }
+        return true;
+    }
+    setMetadata(id, metadata) {
+        if (metadata) {
+            this.metadata.set(id, metadata);
+        }
+        else {
+            this.metadata.delete(id);
+        }
+    }
     /**
-     * Remove vector by ID.
-     * Uses ruvector 0.1.99+ delete(id) instead of remove(id)
+     * Remove vector by ID. The native delete is queued; the return value only
+     * reports whether this instance has seen the id. Use removeAsync() for the
+     * authoritative result.
      */
     remove(id) {
+        this.ensureInitialized();
+        const known = this.knownIds.has(id) || this.metadata.has(id);
+        this.removeAsync(id).catch(() => { });
+        return known;
+    }
+    async removeAsync(id) {
         this.ensureInitialized();
         this.metadata.delete(id);
-        try {
-            const result = this.db.delete(id);
-            // Handle async result
-            if (result && typeof result.then === 'function') {
-                let resolved = false;
-                let success = false;
-                result.then((r) => { success = r; resolved = true; });
-                if (resolved)
-                    return success;
-                // Fire-and-forget for async case
-                result.catch(() => { });
-                return true;
-            }
-            return result !== false;
-        }
-        catch {
-            return false;
-        }
+        return this.enqueue(() => this.deleteNative(id));
     }
     /**
-     * Get database statistics including native SIMD status
+     * Get database statistics. `count` reflects settled writes; await flush()
+     * or use getStatsAsync() to include queued ones.
      */
     getStats() {
         this.ensureInitialized();
-        let count = 0;
-        const lenResult = this.db.len();
-        if (lenResult && typeof lenResult.then === 'function') {
-            let resolved = false;
-            lenResult.then((c) => { count = c; resolved = true; });
-            if (!resolved)
-                count = this.metadata.size; // fallback
-        }
-        else {
-            count = lenResult || 0;
-        }
         return {
-            count,
+            count: this.cachedCount,
             dimension: this.config.dimension || 384,
             metric: this.config.metric,
             backend: 'ruvector',
             memoryUsage: 0,
-            // Extended stats for 0.1.99+
-            ...(this.isNativeImpl && {
-                nativeVersion: this.nativeVersion,
-                simdEnabled: true,
-                implementation: 'native'
-            })
         };
     }
+    async getStatsAsync() {
+        this.ensureInitialized();
+        await this.queue;
+        this.cachedCount = await this.db.len();
+        return this.getStats();
+    }
+    /**
+     * Wait for every queued native write. Throws the most recent write failure
+     * since the last flush, if any.
+     */
+    async flush() {
+        if (!this.initialized)
+            return;
+        await this.queue;
+        if (this.lastWriteError) {
+            const err = this.lastWriteError;
+            this.lastWriteError = null;
+            throw err;
+        }
+    }
     /**
-     * Get extended stats including SIMD and version info
+     * Get extended performance statistics
+     *
+     * Returns detailed metrics including latencies, cache stats, and buffer pool info.
      */
     getExtendedStats() {
-        const base = this.getStats();
         return {
-            count: base.count,
-            dimension: base.dimension,
-            metric: base.metric,
-            backend: base.backend,
-            nativeVersion: this.nativeVersion,
-            isNative: this.isNativeImpl,
-            simdEnabled: this.isNativeImpl
+            basic: this.getStats(),
+            performance: {
+                ...this.stats,
+                insertAvgLatencyMs: this.stats.insertCount > 0
+                    ? this.stats.insertTotalLatencyMs / this.stats.insertCount
+                    : 0,
+                searchAvgLatencyMs: this.stats.searchCount > 0
+                    ? this.stats.searchTotalLatencyMs / this.stats.searchCount
+                    : 0,
+            },
+            bufferPool: this.bufferPool.getStats(),
+            config: {
+                parallelConcurrency: this.config.parallelConcurrency ?? 4,
+                adaptiveParams: this.config.adaptiveParams !== false,
+                mmapEnabled: this.mmapEnabled,
+            },
         };
     }
+    /**
+     * Reset performance statistics
+     */
+    resetStats() {
+        this.stats = {
+            insertCount: 0,
+            insertTotalLatencyMs: 0,
+            insertMinLatencyMs: Infinity,
+            insertMaxLatencyMs: 0,
+            searchCount: 0,
+            searchTotalLatencyMs: 0,
+            searchMinLatencyMs: Infinity,
+            searchMaxLatencyMs: 0,
+            cacheHits: 0,
+            cacheMisses: 0,
+            lastMemoryUsage: 0,
+            indexRebuildCount: 0,
+        };
+    }
+    /**
+     * Update index parameters adaptively based on current dataset size
+     *
+     * This triggers an index rebuild with optimal parameters for the current size.
+     * Should be called after significant data changes.
+     */
+    async updateAdaptiveParams() {
+        this.ensureInitialized();
+        if (this.config.adaptiveParams === false) {
+            return;
+        }
+        const params = this.getAdaptiveParams(this.cachedCount);
+        // M and efConstruction are fixed when the native index is built; efSearch
+        // is a per-query parameter, so adopt the recommended value as the default.
+        this.defaultEfSearch = params.efSearch;
+        this.stats.indexRebuildCount++;
+    }
     /**
      * Save index and metadata to disk
      */
-    async save(path) {
+    async save(savePath) {
         this.ensureInitialized();
-        // Save vector index (if ruvector supports it)
-        if (typeof this.db.save === 'function') {
-            await this.db.save(path);
+        // Validate path for security
+        validatePath(savePath);
+        // The native index persists itself at storagePath; once queued writes have
+        // settled, saving elsewhere is a file copy.
+        await this.flush();
+        const pathModule = await import('path');
+        const target = pathModule.resolve(savePath);
+        if (target !== this.storagePath) {
+            const fsp = await import('fs/promises');
+            await fsp.copyFile(this.storagePath, target);
         }
         // Save metadata separately as JSON
-        const metadataPath = path + '.meta.json';
+        const metadataPath = savePath + '.meta.json';
+        validatePath(metadataPath);
         const fs = await import('fs/promises');
         await fs.writeFile(metadataPath, JSON.stringify(Object.fromEntries(this.metadata), null, 2));
     }
     /**
      * Load index and metadata from disk
      */
-    async load(path) {
+    async load(loadPath) {
         this.ensureInitialized();
-        // Load vector index (if ruvector supports it)
-        if (typeof this.db.load === 'function') {
-            await this.db.load(path);
+        // Validate path for security
+        validatePath(loadPath);
+        // Reopen the native index at loadPath
+        await this.flush();
+        const pathModule = await import('path');
+        const source = pathModule.resolve(loadPath);
+        if (source !== this.storagePath) {
+            const fsSync = await import('fs');
+            if (!fsSync.existsSync(source)) {
+                throw new Error(`RuVector index not found: ${loadPath}`);
+            }
+            this.db = new this.VectorDbCtor({ ...this.nativeOptions, storagePath: source });
+            this.removeOwnedTempDir();
+            this.storagePath = source;
         }
+        this.cachedCount = await this.db.len();
+        this.storeWasEmpty = this.cachedCount === 0;
+        this.knownIds.clear();
+        this.metadata.clear();
         // Load metadata
-        const metadataPath = path + '.meta.json';
+        const metadataPath = loadPath + '.meta.json';
+        validatePath(metadataPath);
         try {
             const fs = await import('fs/promises');
             const data = await fs.readFile(metadataPath, 'utf-8');
-            this.metadata = new Map(Object.entries(JSON.parse(data)));
+            // Safely parse metadata with validation
+            let parsed;
+            try {
+                parsed = JSON.parse(data);
+            }
+            catch {
+                throw new Error('Invalid JSON in metadata file');
+            }
+            // Validate parsed data is a plain object
+            if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+                throw new Error('Metadata must be a plain object');
+            }
+            // Check for prototype pollution attempts
+            const unsafeKeys = ['__proto__', 'constructor', 'prototype'];
+            for (const key of Object.keys(parsed)) {
+                if (unsafeKeys.includes(key)) {
+                    throw new Error(`Forbidden key in metadata: ${key}`);
+                }
+            }
+            this.metadata = new Map(Object.entries(parsed));
+            // Enforce size limit
+            if (this.metadata.size > MAX_METADATA_ENTRIES) {
+                this.metadata.clear();
+                throw new Error(`Loaded metadata exceeds maximum of ${MAX_METADATA_ENTRIES} entries`);
+            }
         }
-        catch {
-            console.debug(`[RuVectorBackend] No metadata file found at ${metadataPath}`);
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                // No metadata file - this is okay for backward compatibility
+                console.error(`[RuVectorBackend] No metadata file found at ${metadataPath}`);
+            }
+            else {
+                throw error;
+            }
         }
     }
     /**
      * Close and cleanup resources
      */
     close() {
+        // The native binding has no close(); drop our reference once queued
+        // writes settle, then remove a temp index this instance created.
+        const ownedTempDir = this.ownedTempDir;
+        this.ownedTempDir = null;
+        if (ownedTempDir) {
+            this.queue.finally(() => {
+                this.ownedTempDir = ownedTempDir;
+                this.removeOwnedTempDir();
+            });
+        }
+        this.initialized = false;
+        this.knownIds.clear();
         this.metadata.clear();
+        // Clear buffer pool
+        this.bufferPool.clear();
+        // Clean up mmap buffer
+        if (this.mmapBuffer) {
+            this.mmapBuffer = null;
+        }
+        // Reset stats
+        this.resetStats();
     }
     /**
-     * Convert score to distance for backward compatibility.
-     * ruvector 0.1.99+ returns scores (higher = more similar).
+     * Convert distance to similarity score based on metric
+     *
+     * Cosine: distance is already in [0, 2], where 0 = identical
+     * L2: exponential decay for unbounded distances
+     * IP: negative inner product, so negate for similarity
      */
-    scoreToDistance(score) {
+    distanceToSimilarity(distance) {
         switch (this.config.metric) {
             case 'cosine':
-                return 1 - score;
+                return 1 - distance; // cosine distance is 1 - similarity
             case 'l2':
-                return score === 0 ? Infinity : -Math.log(score);
+                return Math.exp(-distance); // exponential decay
             case 'ip':
-                return -score;
+                return -distance; // inner product: higher is better
             default:
-                return 1 - score;
+                return 1 - distance;
         }
     }
     /**
@@ -437,5 +874,42 @@ export class RuVectorBackend {
             throw new Error('RuVectorBackend not initialized. Call initialize() first.');
         }
     }
+    /**
+     * Get the buffer pool instance for advanced use cases
+     */
+    getBufferPool() {
+        return this.bufferPool;
+    }
+    /**
+     * Get the current concurrency semaphore status
+     */
+    getConcurrencyStatus() {
+        return {
+            available: this.semaphore.available,
+            configured: this.config.parallelConcurrency ?? 4,
+        };
+    }
+    /**
+     * Check if memory-mapped storage is active
+     */
+    isMmapEnabled() {
+        return this.mmapEnabled && this.mmapBuffer !== null;
+    }
+    /**
+     * Get recommended adaptive parameters for a given dataset size
+     */
+    static getRecommendedParams(datasetSize) {
+        if (datasetSize < 1000) {
+            return { M: 8, efConstruction: 100, efSearch: 50 };
+        }
+        else if (datasetSize <= 100000) {
+            return { M: 16, efConstruction: 200, efSearch: 100 };
+        }
+        else {
+            return { M: 32, efConstruction: 400, efSearch: 200 };
+        }
+    }
 }
+// Export helper classes for advanced use cases
+export { Semaphore, BufferPool };
 //# sourceMappingURL=RuVectorBackend.js.map