agentdb 1.0.0

package/dist/cli/core/native-backend.js

@@ -0,0 +1,432 @@
+"use strict";
+/**
+ * Native backend implementation using better-sqlite3
+ * High-performance backend for Node.js environments
+ */
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NativeBackend = void 0;
+var better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+var hnsw_1 = require("../index/hnsw");
+/**
+ * Native backend using better-sqlite3
+ * Optimized for Node.js with native SQLite bindings
+ * Features HNSW index for high-performance approximate nearest neighbor search
+ */
+var NativeBackend = /** @class */ (function () {
+    function NativeBackend() {
+        this.db = null;
+        this.insertStmt = null;
+        this.initialized = false;
+        this.hnswIndex = null;
+        this.hnswConfig = hnsw_1.DEFAULT_HNSW_CONFIG;
+    }
+    /**
+     * Initialize the native backend
+     */
+    NativeBackend.prototype.initialize = function (config) {
+        if (config === void 0) { config = {}; }
+        if (this.initialized) {
+            return;
+        }
+        var _a = config.path, path = _a === void 0 ? ':memory:' : _a, _b = config.memoryMode, memoryMode = _b === void 0 ? true : _b, _c = config.cacheSize, cacheSize = _c === void 0 ? 100 * 1024 : _c, // 100MB in KB
+        _d = config.walMode, // 100MB in KB
+        walMode = _d === void 0 ? true : _d, _e = config.mmapSize, mmapSize = _e === void 0 ? 256 * 1024 * 1024 : _e, // 256MB
+        _f = config.hnsw, // 256MB
+        hnsw = _f === void 0 ? {} : _f;
+        // Create database instance
+        this.db = new better_sqlite3_1.default(memoryMode ? ':memory:' : path);
+        // Initialize schema and prepare statements
+        this.initializeSchema(cacheSize, walMode, mmapSize);
+        this.registerCustomFunctions();
+        this.prepareStatements();
+        // Initialize HNSW index
+        this.hnswConfig = __assign(__assign({}, hnsw_1.DEFAULT_HNSW_CONFIG), hnsw);
+        if (this.hnswConfig.enabled) {
+            this.hnswIndex = new hnsw_1.HNSWIndex(this.db, this.hnswConfig);
+        }
+        this.initialized = true;
+    };
+    /**
+     * Initialize database schema
+     */
+    NativeBackend.prototype.initializeSchema = function (cacheSize, walMode, mmapSize) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Optimize SQLite for vector operations
+        this.db.pragma('journal_mode = ' + (walMode ? 'WAL' : 'MEMORY'));
+        this.db.pragma('synchronous = NORMAL');
+        this.db.pragma("cache_size = -".concat(cacheSize));
+        this.db.pragma('temp_store = MEMORY');
+        this.db.pragma("mmap_size = ".concat(mmapSize));
+        this.db.pragma('page_size = 4096');
+        // Create vectors table with optimized schema
+        this.db.exec("\n      CREATE TABLE IF NOT EXISTS vectors (\n        id TEXT PRIMARY KEY,\n        embedding BLOB NOT NULL,\n        norm REAL NOT NULL,\n        metadata TEXT,\n        timestamp INTEGER NOT NULL\n      );\n\n      CREATE INDEX IF NOT EXISTS idx_vectors_norm ON vectors(norm);\n      CREATE INDEX IF NOT EXISTS idx_vectors_timestamp ON vectors(timestamp);\n    ");
+    };
+    /**
+     * Register custom SQL functions for similarity calculations
+     */
+    NativeBackend.prototype.registerCustomFunctions = function () {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Cosine similarity
+        this.db.function('cosine_similarity', { deterministic: true }, function (a, b, normA, normB) {
+            var arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
+            var arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
+            var dotProduct = 0;
+            for (var i = 0; i < arrA.length; i++) {
+                dotProduct += arrA[i] * arrB[i];
+            }
+            return dotProduct / (normA * normB);
+        });
+        // Euclidean distance
+        this.db.function('euclidean_distance', { deterministic: true }, function (a, b) {
+            var arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
+            var arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
+            var sum = 0;
+            for (var i = 0; i < arrA.length; i++) {
+                var diff = arrA[i] - arrB[i];
+                sum += diff * diff;
+            }
+            return Math.sqrt(sum);
+        });
+        // Dot product
+        this.db.function('dot_product', { deterministic: true }, function (a, b) {
+            var arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
+            var arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
+            var dotProduct = 0;
+            for (var i = 0; i < arrA.length; i++) {
+                dotProduct += arrA[i] * arrB[i];
+            }
+            return dotProduct;
+        });
+    };
+    /**
+     * Prepare frequently used statements
+     */
+    NativeBackend.prototype.prepareStatements = function () {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        this.insertStmt = this.db.prepare("\n      INSERT OR REPLACE INTO vectors (id, embedding, norm, metadata, timestamp)\n      VALUES (?, ?, ?, ?, ?)\n    ");
+    };
+    /**
+     * Calculate L2 norm of a vector
+     */
+    NativeBackend.prototype.calculateNorm = function (embedding) {
+        var sum = 0;
+        for (var _i = 0, embedding_1 = embedding; _i < embedding_1.length; _i++) {
+            var val = embedding_1[_i];
+            sum += val * val;
+        }
+        return Math.sqrt(sum);
+    };
+    /**
+     * Serialize embedding to binary format
+     */
+    NativeBackend.prototype.serializeEmbedding = function (embedding) {
+        var buffer = Buffer.allocUnsafe(embedding.length * 4);
+        var view = new Float32Array(buffer.buffer, buffer.byteOffset, embedding.length);
+        view.set(embedding);
+        return buffer;
+    };
+    /**
+     * Deserialize embedding from binary format
+     */
+    NativeBackend.prototype.deserializeEmbedding = function (buffer) {
+        var view = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
+        return Array.from(view);
+    };
+    /**
+     * Generate unique ID
+     */
+    NativeBackend.prototype.generateId = function () {
+        return "vec_".concat(Date.now(), "_").concat(Math.random().toString(36).substr(2, 9));
+    };
+    /**
+     * Insert a single vector
+     */
+    NativeBackend.prototype.insert = function (vector) {
+        if (!this.db || !this.insertStmt)
+            throw new Error('Database not initialized');
+        var id = vector.id || this.generateId();
+        var norm = this.calculateNorm(vector.embedding);
+        var embedding = this.serializeEmbedding(vector.embedding);
+        var metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
+        var timestamp = vector.timestamp || Date.now();
+        this.insertStmt.run(id, embedding, norm, metadata, timestamp);
+        // Update HNSW index if enabled
+        if (this.hnswIndex && this.shouldUseHNSW()) {
+            this.hnswIndex.insert(id, vector.embedding);
+        }
+        return id;
+    };
+    /**
+     * Insert multiple vectors in a transaction with optimized batch processing
+     * Performance improvements:
+     * - Single transaction for all inserts
+     * - Reused prepared statement
+     * - No wrapper function overhead
+     * - Chunked processing for large batches
+     * - Automatic HNSW index building after batch insert
+     */
+    NativeBackend.prototype.insertBatch = function (vectors) {
+        var _this = this;
+        if (!this.db || !this.insertStmt)
+            throw new Error('Database not initialized');
+        var ids = [];
+        var CHUNK_SIZE = 5000; // Process in chunks to avoid memory spikes
+        // Helper function to process a chunk of vectors
+        var processChunk = function (chunk) {
+            var chunkIds = [];
+            var transaction = _this.db.transaction(function (vecs) {
+                // Reuse the prepared statement for all inserts
+                for (var _i = 0, vecs_1 = vecs; _i < vecs_1.length; _i++) {
+                    var vector = vecs_1[_i];
+                    var id = vector.id || _this.generateId();
+                    var norm = _this.calculateNorm(vector.embedding);
+                    var embedding = _this.serializeEmbedding(vector.embedding);
+                    var metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
+                    var timestamp = vector.timestamp || Date.now();
+                    _this.insertStmt.run(id, embedding, norm, metadata, timestamp);
+                    chunkIds.push(id);
+                }
+            });
+            transaction(chunk);
+            return chunkIds;
+        };
+        // Process vectors in chunks
+        if (vectors.length <= CHUNK_SIZE) {
+            // Small batch - process directly
+            ids.push.apply(ids, processChunk(vectors));
+        }
+        else {
+            // Large batch - process in chunks to manage memory
+            for (var i = 0; i < vectors.length; i += CHUNK_SIZE) {
+                var chunk = vectors.slice(i, i + CHUNK_SIZE);
+                ids.push.apply(ids, processChunk(chunk));
+            }
+        }
+        // Build HNSW index if threshold reached and not already built
+        if (this.hnswIndex && this.shouldUseHNSW() && !this.hnswIndex.isReady()) {
+            console.log('Building HNSW index after batch insert...');
+            this.hnswIndex.build();
+        }
+        return ids;
+    };
+    /**
+     * Check if HNSW index should be used
+     */
+    NativeBackend.prototype.shouldUseHNSW = function () {
+        if (!this.hnswIndex || !this.hnswConfig.enabled) {
+            return false;
+        }
+        var stats = this.stats();
+        return stats.count >= this.hnswConfig.minVectorsForIndex;
+    };
+    /**
+     * Search for k-nearest neighbors
+     * Automatically uses HNSW index when available and beneficial
+     */
+    NativeBackend.prototype.search = function (queryEmbedding, k, metric, threshold) {
+        if (k === void 0) { k = 5; }
+        if (metric === void 0) { metric = 'cosine'; }
+        if (threshold === void 0) { threshold = 0.0; }
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Use HNSW index for euclidean distance if available
+        // Note: HNSW currently optimized for euclidean, can be extended for other metrics
+        if (this.hnswIndex && this.hnswIndex.isReady() && this.shouldUseHNSW() && metric === 'euclidean') {
+            return this.searchWithHNSW(queryEmbedding, k, threshold);
+        }
+        // Fall back to brute-force search
+        return this.bruteForceSearch(queryEmbedding, k, metric, threshold);
+    };
+    /**
+     * Search using HNSW index (fast approximate search)
+     */
+    NativeBackend.prototype.searchWithHNSW = function (queryEmbedding, k, threshold) {
+        var _this = this;
+        if (!this.hnswIndex || !this.db) {
+            throw new Error('HNSW index not initialized');
+        }
+        var results = this.hnswIndex.search(queryEmbedding, k);
+        // Filter by threshold and get metadata from main table
+        return results
+            .filter(function (r) { return threshold === 0 || r.distance <= threshold; })
+            .map(function (result) {
+            var stmt = _this.db.prepare('SELECT metadata FROM vectors WHERE id = ?');
+            var row = stmt.get(result.vectorId);
+            return {
+                id: result.vectorId,
+                score: result.distance,
+                embedding: result.embedding,
+                metadata: (row === null || row === void 0 ? void 0 : row.metadata) ? JSON.parse(row.metadata) : undefined
+            };
+        });
+    };
+    /**
+     * Brute-force search (accurate but slower)
+     */
+    NativeBackend.prototype.bruteForceSearch = function (queryEmbedding, k, metric, threshold) {
+        var _this = this;
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var queryBuffer = this.serializeEmbedding(queryEmbedding);
+        var queryNorm = this.calculateNorm(queryEmbedding);
+        var sql;
+        var params;
+        switch (metric) {
+            case 'cosine':
+                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            cosine_similarity(embedding, ?, norm, ?) as score\n          FROM vectors\n          WHERE cosine_similarity(embedding, ?, norm, ?) >= ?\n          ORDER BY score DESC\n          LIMIT ?\n        ";
+                params = [queryBuffer, queryNorm, queryBuffer, queryNorm, threshold, k];
+                break;
+            case 'euclidean':
+                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            euclidean_distance(embedding, ?) as score\n          FROM vectors\n          WHERE euclidean_distance(embedding, ?) <= ? OR ? = 0\n          ORDER BY score ASC\n          LIMIT ?\n        ";
+                params = [queryBuffer, queryBuffer, threshold, threshold, k];
+                break;
+            case 'dot':
+                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            dot_product(embedding, ?) as score\n          FROM vectors\n          WHERE dot_product(embedding, ?) >= ?\n          ORDER BY score DESC\n          LIMIT ?\n        ";
+                params = [queryBuffer, queryBuffer, threshold, k];
+                break;
+            default:
+                throw new Error("Unsupported similarity metric: ".concat(metric));
+        }
+        var stmt = this.db.prepare(sql);
+        var rows = stmt.all.apply(stmt, params);
+        return rows.map(function (row) { return ({
+            id: row.id,
+            score: row.score,
+            embedding: _this.deserializeEmbedding(row.embedding),
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+        }); });
+    };
+    /**
+     * Get vector by ID
+     */
+    NativeBackend.prototype.get = function (id) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var stmt = this.db.prepare('SELECT * FROM vectors WHERE id = ?');
+        var row = stmt.get(id);
+        if (!row)
+            return null;
+        return {
+            id: row.id,
+            embedding: this.deserializeEmbedding(row.embedding),
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
+            norm: row.norm,
+            timestamp: row.timestamp
+        };
+    };
+    /**
+     * Delete vector by ID
+     */
+    NativeBackend.prototype.delete = function (id) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Delete from HNSW index if enabled
+        if (this.hnswIndex && this.shouldUseHNSW()) {
+            this.hnswIndex.delete(id);
+        }
+        var stmt = this.db.prepare('DELETE FROM vectors WHERE id = ?');
+        var result = stmt.run(id);
+        return result.changes > 0;
+    };
+    /**
+     * Get database statistics
+     */
+    NativeBackend.prototype.stats = function () {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var countStmt = this.db.prepare('SELECT COUNT(*) as count FROM vectors');
+        var sizeStmt = this.db.prepare('SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()');
+        var count = countStmt.get().count;
+        var size = sizeStmt.get().size;
+        return { count: count, size: size };
+    };
+    /**
+     * Close database connection
+     */
+    NativeBackend.prototype.close = function () {
+        if (this.db) {
+            this.db.close();
+            this.db = null;
+        }
+        this.insertStmt = null;
+        this.initialized = false;
+    };
+    /**
+     * Get raw database instance (for advanced usage)
+     */
+    NativeBackend.prototype.getDatabase = function () {
+        return this.db;
+    };
+    /**
+     * Check if backend is initialized
+     */
+    NativeBackend.prototype.isInitialized = function () {
+        return this.initialized;
+    };
+    /**
+     * Build HNSW index manually
+     * Useful for pre-building index on existing data
+     */
+    NativeBackend.prototype.buildHNSWIndex = function () {
+        if (!this.hnswIndex) {
+            throw new Error('HNSW index not enabled');
+        }
+        if (!this.shouldUseHNSW()) {
+            var stats = this.stats();
+            console.warn("Vector count (".concat(stats.count, ") below threshold (").concat(this.hnswConfig.minVectorsForIndex, "). Index build skipped."));
+            return;
+        }
+        this.hnswIndex.build();
+    };
+    /**
+     * Get HNSW index statistics
+     */
+    NativeBackend.prototype.getHNSWStats = function () {
+        if (!this.hnswIndex) {
+            return null;
+        }
+        var stats = this.hnswIndex.getStats();
+        return __assign({ enabled: this.hnswConfig.enabled, ready: this.hnswIndex.isReady() }, stats);
+    };
+    /**
+     * Clear HNSW index
+     */
+    NativeBackend.prototype.clearHNSWIndex = function () {
+        if (this.hnswIndex) {
+            this.hnswIndex.clear();
+        }
+    };
+    /**
+     * Update HNSW configuration and rebuild if needed
+     */
+    NativeBackend.prototype.updateHNSWConfig = function (config, rebuild) {
+        if (rebuild === void 0) { rebuild = false; }
+        if (!this.hnswIndex) {
+            throw new Error('HNSW index not enabled');
+        }
+        this.hnswIndex.updateConfig(config);
+        this.hnswConfig = __assign(__assign({}, this.hnswConfig), config);
+        if (rebuild) {
+            this.hnswIndex.build();
+        }
+    };
+    return NativeBackend;
+}());
+exports.NativeBackend = NativeBackend;

package/dist/cli/core/vector-db.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Core SQLiteVector database implementation with backend abstraction
+ * Supports both native (better-sqlite3) and WASM (sql.js) backends
+ */
+import { Vector, SearchResult, SimilarityMetric, DatabaseConfig } from '../types';
+import { VectorBackend, BackendType, ExtendedDatabaseConfig } from './backend-interface';
+import { QueryCache } from '../cache/query-cache';
+import { ProductQuantizer } from '../quantization/product-quantization';
+import { VectorQueryBuilder } from '../query/query-builder';
+export declare class SQLiteVectorDB {
+    private backend;
+    private backendType;
+    private queryCache?;
+    private quantizer?;
+    constructor(config?: ExtendedDatabaseConfig);
+    /**
+     * Detect appropriate backend based on environment and configuration
+     */
+    private detectBackend;
+    /**
+     * Create backend instance
+     */
+    private createBackend;
+    /**
+     * Async initialization for WASM backend
+     * Must be called after constructor if using WASM backend
+     */
+    initializeAsync(config?: DatabaseConfig): Promise<void>;
+    /**
+     * Get backend type
+     */
+    getBackendType(): BackendType;
+    /**
+     * Check if backend is initialized
+     */
+    isInitialized(): boolean;
+    /**
+     * Insert a single vector
+     */
+    insert(vector: Vector): string;
+    /**
+     * Insert multiple vectors in a transaction
+     */
+    insertBatch(vectors: Vector[]): string[];
+    /**
+     * Search for k-nearest neighbors with optional caching
+     */
+    search(queryEmbedding: number[], k?: number, metric?: SimilarityMetric, threshold?: number): SearchResult[];
+    /**
+     * Get vector by ID
+     */
+    get(id: string): Vector | null;
+    /**
+     * Delete vector by ID
+     */
+    delete(id: string): boolean;
+    /**
+     * Get database statistics
+     */
+    stats(): {
+        count: number;
+        size: number;
+    };
+    /**
+     * Close database connection
+     */
+    close(): void;
+    /**
+     * Export database to binary format (WASM only)
+     */
+    export(): Uint8Array | undefined;
+    /**
+     * Import database from binary format (WASM only)
+     */
+    importAsync(data: Uint8Array): Promise<void>;
+    /**
+     * Get raw backend instance (for advanced usage)
+     */
+    getBackend(): VectorBackend;
+    /**
+     * Get raw database instance (for advanced usage with native backend)
+     * @deprecated Use getBackend() instead for cross-backend compatibility
+     */
+    getDatabase(): any;
+    /**
+     * Get query cache instance
+     */
+    getQueryCache(): QueryCache<SearchResult[]> | undefined;
+    /**
+     * Get query cache statistics
+     */
+    getCacheStats(): import("../cache/query-cache").CacheStats;
+    /**
+     * Clear query cache
+     */
+    clearCache(): void;
+    /**
+     * Get quantizer instance
+     */
+    getQuantizer(): ProductQuantizer | undefined;
+    /**
+     * Train quantizer with existing vectors
+     */
+    trainQuantizer(): Promise<void>;
+    /**
+     * Get compression statistics (if quantization enabled)
+     */
+    getCompressionStats(): import("../quantization/product-quantization").CompressionStats;
+    /**
+     * Create a fluent query builder for complex queries
+     *
+     * @returns VectorQueryBuilder instance
+     *
+     * @example
+     * ```typescript
+     * const results = await db.query()
+     *   .similarTo(queryVector)
+     *   .where('metadata.category', '=', 'tech')
+     *   .whereBetween('metadata.year', 2020, 2024)
+     *   .orderBySimilarity('desc')
+     *   .limit(10)
+     *   .execute();
+     * ```
+     */
+    query<T = any>(): VectorQueryBuilder<T>;
+}