agentdb 1.0.11 → 1.1.0

package/dist/cli/core/native-backend.js

@@ -1,432 +0,0 @@
-"use strict";
-/**
- * Native backend implementation using better-sqlite3
- * High-performance backend for Node.js environments
- */
-var __assign = (this && this.__assign) || function () {
-    __assign = Object.assign || function(t) {
-        for (var s, i = 1, n = arguments.length; i < n; i++) {
-            s = arguments[i];
-            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
-                t[p] = s[p];
-        }
-        return t;
-    };
-    return __assign.apply(this, arguments);
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NativeBackend = void 0;
-var better_sqlite3_1 = __importDefault(require("better-sqlite3"));
-var hnsw_1 = require("../index/hnsw");
-/**
- * Native backend using better-sqlite3
- * Optimized for Node.js with native SQLite bindings
- * Features HNSW index for high-performance approximate nearest neighbor search
- */
-var NativeBackend = /** @class */ (function () {
-    function NativeBackend() {
-        this.db = null;
-        this.insertStmt = null;
-        this.initialized = false;
-        this.hnswIndex = null;
-        this.hnswConfig = hnsw_1.DEFAULT_HNSW_CONFIG;
-    }
-    /**
-     * Initialize the native backend
-     */
-    NativeBackend.prototype.initialize = function (config) {
-        if (config === void 0) { config = {}; }
-        if (this.initialized) {
-            return;
-        }
-        var _a = config.path, path = _a === void 0 ? ':memory:' : _a, _b = config.memoryMode, memoryMode = _b === void 0 ? true : _b, _c = config.cacheSize, cacheSize = _c === void 0 ? 100 * 1024 : _c, // 100MB in KB
-        _d = config.walMode, // 100MB in KB
-        walMode = _d === void 0 ? true : _d, _e = config.mmapSize, mmapSize = _e === void 0 ? 256 * 1024 * 1024 : _e, // 256MB
-        _f = config.hnsw, // 256MB
-        hnsw = _f === void 0 ? {} : _f;
-        // Create database instance
-        this.db = new better_sqlite3_1.default(memoryMode ? ':memory:' : path);
-        // Initialize schema and prepare statements
-        this.initializeSchema(cacheSize, walMode, mmapSize);
-        this.registerCustomFunctions();
-        this.prepareStatements();
-        // Initialize HNSW index
-        this.hnswConfig = __assign(__assign({}, hnsw_1.DEFAULT_HNSW_CONFIG), hnsw);
-        if (this.hnswConfig.enabled) {
-            this.hnswIndex = new hnsw_1.HNSWIndex(this.db, this.hnswConfig);
-        }
-        this.initialized = true;
-    };
-    /**
-     * Initialize database schema
-     */
-    NativeBackend.prototype.initializeSchema = function (cacheSize, walMode, mmapSize) {
-        if (!this.db)
-            throw new Error('Database not initialized');
-        // Optimize SQLite for vector operations
-        this.db.pragma('journal_mode = ' + (walMode ? 'WAL' : 'MEMORY'));
-        this.db.pragma('synchronous = NORMAL');
-        this.db.pragma("cache_size = -".concat(cacheSize));
-        this.db.pragma('temp_store = MEMORY');
-        this.db.pragma("mmap_size = ".concat(mmapSize));
-        this.db.pragma('page_size = 4096');
-        // Create vectors table with optimized schema
-        this.db.exec("\n      CREATE TABLE IF NOT EXISTS vectors (\n        id TEXT PRIMARY KEY,\n        embedding BLOB NOT NULL,\n        norm REAL NOT NULL,\n        metadata TEXT,\n        timestamp INTEGER NOT NULL\n      );\n\n      CREATE INDEX IF NOT EXISTS idx_vectors_norm ON vectors(norm);\n      CREATE INDEX IF NOT EXISTS idx_vectors_timestamp ON vectors(timestamp);\n    ");
-    };
-    /**
-     * Register custom SQL functions for similarity calculations
-     */
-    NativeBackend.prototype.registerCustomFunctions = function () {
-        if (!this.db)
-            throw new Error('Database not initialized');
-        // Cosine similarity
-        this.db.function('cosine_similarity', { deterministic: true }, function (a, b, normA, normB) {
-            var arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
-            var arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
-            var dotProduct = 0;
-            for (var i = 0; i < arrA.length; i++) {
-                dotProduct += arrA[i] * arrB[i];
-            }
-            return dotProduct / (normA * normB);
-        });
-        // Euclidean distance
-        this.db.function('euclidean_distance', { deterministic: true }, function (a, b) {
-            var arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
-            var arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
-            var sum = 0;
-            for (var i = 0; i < arrA.length; i++) {
-                var diff = arrA[i] - arrB[i];
-                sum += diff * diff;
-            }
-            return Math.sqrt(sum);
-        });
-        // Dot product
-        this.db.function('dot_product', { deterministic: true }, function (a, b) {
-            var arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
-            var arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
-            var dotProduct = 0;
-            for (var i = 0; i < arrA.length; i++) {
-                dotProduct += arrA[i] * arrB[i];
-            }
-            return dotProduct;
-        });
-    };
-    /**
-     * Prepare frequently used statements
-     */
-    NativeBackend.prototype.prepareStatements = function () {
-        if (!this.db)
-            throw new Error('Database not initialized');
-        this.insertStmt = this.db.prepare("\n      INSERT OR REPLACE INTO vectors (id, embedding, norm, metadata, timestamp)\n      VALUES (?, ?, ?, ?, ?)\n    ");
-    };
-    /**
-     * Calculate L2 norm of a vector
-     */
-    NativeBackend.prototype.calculateNorm = function (embedding) {
-        var sum = 0;
-        for (var _i = 0, embedding_1 = embedding; _i < embedding_1.length; _i++) {
-            var val = embedding_1[_i];
-            sum += val * val;
-        }
-        return Math.sqrt(sum);
-    };
-    /**
-     * Serialize embedding to binary format
-     */
-    NativeBackend.prototype.serializeEmbedding = function (embedding) {
-        var buffer = Buffer.allocUnsafe(embedding.length * 4);
-        var view = new Float32Array(buffer.buffer, buffer.byteOffset, embedding.length);
-        view.set(embedding);
-        return buffer;
-    };
-    /**
-     * Deserialize embedding from binary format
-     */
-    NativeBackend.prototype.deserializeEmbedding = function (buffer) {
-        var view = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
-        return Array.from(view);
-    };
-    /**
-     * Generate unique ID
-     */
-    NativeBackend.prototype.generateId = function () {
-        return "vec_".concat(Date.now(), "_").concat(Math.random().toString(36).substr(2, 9));
-    };
-    /**
-     * Insert a single vector
-     */
-    NativeBackend.prototype.insert = function (vector) {
-        if (!this.db || !this.insertStmt)
-            throw new Error('Database not initialized');
-        var id = vector.id || this.generateId();
-        var norm = this.calculateNorm(vector.embedding);
-        var embedding = this.serializeEmbedding(vector.embedding);
-        var metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
-        var timestamp = vector.timestamp || Date.now();
-        this.insertStmt.run(id, embedding, norm, metadata, timestamp);
-        // Update HNSW index if enabled
-        if (this.hnswIndex && this.shouldUseHNSW()) {
-            this.hnswIndex.insert(id, vector.embedding);
-        }
-        return id;
-    };
-    /**
-     * Insert multiple vectors in a transaction with optimized batch processing
-     * Performance improvements:
-     * - Single transaction for all inserts
-     * - Reused prepared statement
-     * - No wrapper function overhead
-     * - Chunked processing for large batches
-     * - Automatic HNSW index building after batch insert
-     */
-    NativeBackend.prototype.insertBatch = function (vectors) {
-        var _this = this;
-        if (!this.db || !this.insertStmt)
-            throw new Error('Database not initialized');
-        var ids = [];
-        var CHUNK_SIZE = 5000; // Process in chunks to avoid memory spikes
-        // Helper function to process a chunk of vectors
-        var processChunk = function (chunk) {
-            var chunkIds = [];
-            var transaction = _this.db.transaction(function (vecs) {
-                // Reuse the prepared statement for all inserts
-                for (var _i = 0, vecs_1 = vecs; _i < vecs_1.length; _i++) {
-                    var vector = vecs_1[_i];
-                    var id = vector.id || _this.generateId();
-                    var norm = _this.calculateNorm(vector.embedding);
-                    var embedding = _this.serializeEmbedding(vector.embedding);
-                    var metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
-                    var timestamp = vector.timestamp || Date.now();
-                    _this.insertStmt.run(id, embedding, norm, metadata, timestamp);
-                    chunkIds.push(id);
-                }
-            });
-            transaction(chunk);
-            return chunkIds;
-        };
-        // Process vectors in chunks
-        if (vectors.length <= CHUNK_SIZE) {
-            // Small batch - process directly
-            ids.push.apply(ids, processChunk(vectors));
-        }
-        else {
-            // Large batch - process in chunks to manage memory
-            for (var i = 0; i < vectors.length; i += CHUNK_SIZE) {
-                var chunk = vectors.slice(i, i + CHUNK_SIZE);
-                ids.push.apply(ids, processChunk(chunk));
-            }
-        }
-        // Build HNSW index if threshold reached and not already built
-        if (this.hnswIndex && this.shouldUseHNSW() && !this.hnswIndex.isReady()) {
-            console.log('Building HNSW index after batch insert...');
-            this.hnswIndex.build();
-        }
-        return ids;
-    };
-    /**
-     * Check if HNSW index should be used
-     */
-    NativeBackend.prototype.shouldUseHNSW = function () {
-        if (!this.hnswIndex || !this.hnswConfig.enabled) {
-            return false;
-        }
-        var stats = this.stats();
-        return stats.count >= this.hnswConfig.minVectorsForIndex;
-    };
-    /**
-     * Search for k-nearest neighbors
-     * Automatically uses HNSW index when available and beneficial
-     */
-    NativeBackend.prototype.search = function (queryEmbedding, k, metric, threshold) {
-        if (k === void 0) { k = 5; }
-        if (metric === void 0) { metric = 'cosine'; }
-        if (threshold === void 0) { threshold = 0.0; }
-        if (!this.db)
-            throw new Error('Database not initialized');
-        // Use HNSW index for euclidean distance if available
-        // Note: HNSW currently optimized for euclidean, can be extended for other metrics
-        if (this.hnswIndex && this.hnswIndex.isReady() && this.shouldUseHNSW() && metric === 'euclidean') {
-            return this.searchWithHNSW(queryEmbedding, k, threshold);
-        }
-        // Fall back to brute-force search
-        return this.bruteForceSearch(queryEmbedding, k, metric, threshold);
-    };
-    /**
-     * Search using HNSW index (fast approximate search)
-     */
-    NativeBackend.prototype.searchWithHNSW = function (queryEmbedding, k, threshold) {
-        var _this = this;
-        if (!this.hnswIndex || !this.db) {
-            throw new Error('HNSW index not initialized');
-        }
-        var results = this.hnswIndex.search(queryEmbedding, k);
-        // Filter by threshold and get metadata from main table
-        return results
-            .filter(function (r) { return threshold === 0 || r.distance <= threshold; })
-            .map(function (result) {
-            var stmt = _this.db.prepare('SELECT metadata FROM vectors WHERE id = ?');
-            var row = stmt.get(result.vectorId);
-            return {
-                id: result.vectorId,
-                score: result.distance,
-                embedding: result.embedding,
-                metadata: (row === null || row === void 0 ? void 0 : row.metadata) ? JSON.parse(row.metadata) : undefined
-            };
-        });
-    };
-    /**
-     * Brute-force search (accurate but slower)
-     */
-    NativeBackend.prototype.bruteForceSearch = function (queryEmbedding, k, metric, threshold) {
-        var _this = this;
-        if (!this.db)
-            throw new Error('Database not initialized');
-        var queryBuffer = this.serializeEmbedding(queryEmbedding);
-        var queryNorm = this.calculateNorm(queryEmbedding);
-        var sql;
-        var params;
-        switch (metric) {
-            case 'cosine':
-                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            cosine_similarity(embedding, ?, norm, ?) as score\n          FROM vectors\n          WHERE cosine_similarity(embedding, ?, norm, ?) >= ?\n          ORDER BY score DESC\n          LIMIT ?\n        ";
-                params = [queryBuffer, queryNorm, queryBuffer, queryNorm, threshold, k];
-                break;
-            case 'euclidean':
-                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            euclidean_distance(embedding, ?) as score\n          FROM vectors\n          WHERE euclidean_distance(embedding, ?) <= ? OR ? = 0\n          ORDER BY score ASC\n          LIMIT ?\n        ";
-                params = [queryBuffer, queryBuffer, threshold, threshold, k];
-                break;
-            case 'dot':
-                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            dot_product(embedding, ?) as score\n          FROM vectors\n          WHERE dot_product(embedding, ?) >= ?\n          ORDER BY score DESC\n          LIMIT ?\n        ";
-                params = [queryBuffer, queryBuffer, threshold, k];
-                break;
-            default:
-                throw new Error("Unsupported similarity metric: ".concat(metric));
-        }
-        var stmt = this.db.prepare(sql);
-        var rows = stmt.all.apply(stmt, params);
-        return rows.map(function (row) { return ({
-            id: row.id,
-            score: row.score,
-            embedding: _this.deserializeEmbedding(row.embedding),
-            metadata: row.metadata ? JSON.parse(row.metadata) : undefined
-        }); });
-    };
-    /**
-     * Get vector by ID
-     */
-    NativeBackend.prototype.get = function (id) {
-        if (!this.db)
-            throw new Error('Database not initialized');
-        var stmt = this.db.prepare('SELECT * FROM vectors WHERE id = ?');
-        var row = stmt.get(id);
-        if (!row)
-            return null;
-        return {
-            id: row.id,
-            embedding: this.deserializeEmbedding(row.embedding),
-            metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
-            norm: row.norm,
-            timestamp: row.timestamp
-        };
-    };
-    /**
-     * Delete vector by ID
-     */
-    NativeBackend.prototype.delete = function (id) {
-        if (!this.db)
-            throw new Error('Database not initialized');
-        // Delete from HNSW index if enabled
-        if (this.hnswIndex && this.shouldUseHNSW()) {
-            this.hnswIndex.delete(id);
-        }
-        var stmt = this.db.prepare('DELETE FROM vectors WHERE id = ?');
-        var result = stmt.run(id);
-        return result.changes > 0;
-    };
-    /**
-     * Get database statistics
-     */
-    NativeBackend.prototype.stats = function () {
-        if (!this.db)
-            throw new Error('Database not initialized');
-        var countStmt = this.db.prepare('SELECT COUNT(*) as count FROM vectors');
-        var sizeStmt = this.db.prepare('SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()');
-        var count = countStmt.get().count;
-        var size = sizeStmt.get().size;
-        return { count: count, size: size };
-    };
-    /**
-     * Close database connection
-     */
-    NativeBackend.prototype.close = function () {
-        if (this.db) {
-            this.db.close();
-            this.db = null;
-        }
-        this.insertStmt = null;
-        this.initialized = false;
-    };
-    /**
-     * Get raw database instance (for advanced usage)
-     */
-    NativeBackend.prototype.getDatabase = function () {
-        return this.db;
-    };
-    /**
-     * Check if backend is initialized
-     */
-    NativeBackend.prototype.isInitialized = function () {
-        return this.initialized;
-    };
-    /**
-     * Build HNSW index manually
-     * Useful for pre-building index on existing data
-     */
-    NativeBackend.prototype.buildHNSWIndex = function () {
-        if (!this.hnswIndex) {
-            throw new Error('HNSW index not enabled');
-        }
-        if (!this.shouldUseHNSW()) {
-            var stats = this.stats();
-            console.warn("Vector count (".concat(stats.count, ") below threshold (").concat(this.hnswConfig.minVectorsForIndex, "). Index build skipped."));
-            return;
-        }
-        this.hnswIndex.build();
-    };
-    /**
-     * Get HNSW index statistics
-     */
-    NativeBackend.prototype.getHNSWStats = function () {
-        if (!this.hnswIndex) {
-            return null;
-        }
-        var stats = this.hnswIndex.getStats();
-        return __assign({ enabled: this.hnswConfig.enabled, ready: this.hnswIndex.isReady() }, stats);
-    };
-    /**
-     * Clear HNSW index
-     */
-    NativeBackend.prototype.clearHNSWIndex = function () {
-        if (this.hnswIndex) {
-            this.hnswIndex.clear();
-        }
-    };
-    /**
-     * Update HNSW configuration and rebuild if needed
-     */
-    NativeBackend.prototype.updateHNSWConfig = function (config, rebuild) {
-        if (rebuild === void 0) { rebuild = false; }
-        if (!this.hnswIndex) {
-            throw new Error('HNSW index not enabled');
-        }
-        this.hnswIndex.updateConfig(config);
-        this.hnswConfig = __assign(__assign({}, this.hnswConfig), config);
-        if (rebuild) {
-            this.hnswIndex.build();
-        }
-    };
-    return NativeBackend;
-}());
-exports.NativeBackend = NativeBackend;

package/dist/cli/core/vector-db.d.ts

@@ -1,126 +0,0 @@
-/**
- * Core SQLiteVector database implementation with backend abstraction
- * Supports both native (better-sqlite3) and WASM (sql.js) backends
- */
-import { Vector, SearchResult, SimilarityMetric, DatabaseConfig } from '../types';
-import { VectorBackend, BackendType, ExtendedDatabaseConfig } from './backend-interface';
-import { QueryCache } from '../cache/query-cache';
-import { ProductQuantizer } from '../quantization/product-quantization';
-import { VectorQueryBuilder } from '../query/query-builder';
-export declare class SQLiteVectorDB {
-    private backend;
-    private backendType;
-    private queryCache?;
-    private quantizer?;
-    constructor(config?: ExtendedDatabaseConfig);
-    /**
-     * Detect appropriate backend based on environment and configuration
-     */
-    private detectBackend;
-    /**
-     * Create backend instance
-     */
-    private createBackend;
-    /**
-     * Async initialization for WASM backend
-     * Must be called after constructor if using WASM backend
-     */
-    initializeAsync(config?: DatabaseConfig): Promise<void>;
-    /**
-     * Get backend type
-     */
-    getBackendType(): BackendType;
-    /**
-     * Check if backend is initialized
-     */
-    isInitialized(): boolean;
-    /**
-     * Insert a single vector
-     */
-    insert(vector: Vector): string;
-    /**
-     * Insert multiple vectors in a transaction
-     */
-    insertBatch(vectors: Vector[]): string[];
-    /**
-     * Search for k-nearest neighbors with optional caching
-     */
-    search(queryEmbedding: number[], k?: number, metric?: SimilarityMetric, threshold?: number): SearchResult[];
-    /**
-     * Get vector by ID
-     */
-    get(id: string): Vector | null;
-    /**
-     * Delete vector by ID
-     */
-    delete(id: string): boolean;
-    /**
-     * Get database statistics
-     */
-    stats(): {
-        count: number;
-        size: number;
-    };
-    /**
-     * Close database connection
-     */
-    close(): void;
-    /**
-     * Export database to binary format (WASM only)
-     */
-    export(): Uint8Array | undefined;
-    /**
-     * Import database from binary format (WASM only)
-     */
-    importAsync(data: Uint8Array): Promise<void>;
-    /**
-     * Get raw backend instance (for advanced usage)
-     */
-    getBackend(): VectorBackend;
-    /**
-     * Get raw database instance (for advanced usage with native backend)
-     * @deprecated Use getBackend() instead for cross-backend compatibility
-     */
-    getDatabase(): any;
-    /**
-     * Get query cache instance
-     */
-    getQueryCache(): QueryCache<SearchResult[]> | undefined;
-    /**
-     * Get query cache statistics
-     */
-    getCacheStats(): import("../cache/query-cache").CacheStats;
-    /**
-     * Clear query cache
-     */
-    clearCache(): void;
-    /**
-     * Get quantizer instance
-     */
-    getQuantizer(): ProductQuantizer | undefined;
-    /**
-     * Train quantizer with existing vectors
-     */
-    trainQuantizer(): Promise<void>;
-    /**
-     * Get compression statistics (if quantization enabled)
-     */
-    getCompressionStats(): import("../quantization/product-quantization").CompressionStats;
-    /**
-     * Create a fluent query builder for complex queries
-     *
-     * @returns VectorQueryBuilder instance
-     *
-     * @example
-     * ```typescript
-     * const results = await db.query()
-     *   .similarTo(queryVector)
-     *   .where('metadata.category', '=', 'tech')
-     *   .whereBetween('metadata.year', 2020, 2024)
-     *   .orderBySimilarity('desc')
-     *   .limit(10)
-     *   .execute();
-     * ```
-     */
-    query<T = any>(): VectorQueryBuilder<T>;
-}