agentdb 1.0.0

package/dist/cli/core/vector-db.js

@@ -0,0 +1,338 @@
+"use strict";
+/**
+ * Core SQLiteVector database implementation with backend abstraction
+ * Supports both native (better-sqlite3) and WASM (sql.js) backends
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SQLiteVectorDB = void 0;
+var backend_interface_1 = require("./backend-interface");
+var native_backend_1 = require("./native-backend");
+var wasm_backend_1 = require("./wasm-backend");
+var query_cache_1 = require("../cache/query-cache");
+var product_quantization_1 = require("../quantization/product-quantization");
+var query_builder_1 = require("../query/query-builder");
+var SQLiteVectorDB = /** @class */ (function () {
+    function SQLiteVectorDB(config) {
+        if (config === void 0) { config = {}; }
+        var _a, _b;
+        // If path is provided but memoryMode not specified, default to file mode
+        if (config.path && config.memoryMode === undefined) {
+            config.memoryMode = false;
+        }
+        // Determine backend type
+        this.backendType = this.detectBackend(config);
+        // Create appropriate backend
+        this.backend = this.createBackend(this.backendType);
+        // Initialize query cache if enabled
+        if (((_a = config.queryCache) === null || _a === void 0 ? void 0 : _a.enabled) !== false) {
+            this.queryCache = new query_cache_1.QueryCache(config.queryCache);
+        }
+        // Initialize quantizer if enabled
+        if ((_b = config.quantization) === null || _b === void 0 ? void 0 : _b.enabled) {
+            this.quantizer = new product_quantization_1.ProductQuantizer(config.quantization);
+        }
+        // Initialize backend
+        if (this.backendType === backend_interface_1.BackendType.WASM) {
+            // WASM backend requires async initialization
+            // Store config for later async init
+            this.backend._pendingConfig = config;
+        }
+        else {
+            // Native backend can initialize synchronously
+            this.backend.initialize(config);
+        }
+    }
+    /**
+     * Detect appropriate backend based on environment and configuration
+     */
+    SQLiteVectorDB.prototype.detectBackend = function (config) {
+        // Explicit backend selection
+        if (config.backend) {
+            return config.backend;
+        }
+        // Auto-detect based on environment
+        if (typeof window !== 'undefined' || typeof process === 'undefined') {
+            // Browser environment
+            return backend_interface_1.BackendType.WASM;
+        }
+        // Node.js environment - check if better-sqlite3 is available
+        try {
+            require.resolve('better-sqlite3');
+            return backend_interface_1.BackendType.NATIVE;
+        }
+        catch (_a) {
+            return backend_interface_1.BackendType.WASM;
+        }
+    };
+    /**
+     * Create backend instance
+     */
+    SQLiteVectorDB.prototype.createBackend = function (type) {
+        switch (type) {
+            case backend_interface_1.BackendType.NATIVE:
+                return new native_backend_1.NativeBackend();
+            case backend_interface_1.BackendType.WASM:
+                return new wasm_backend_1.WasmBackend();
+            default:
+                throw new Error("Unsupported backend type: ".concat(type));
+        }
+    };
+    /**
+     * Async initialization for WASM backend
+     * Must be called after constructor if using WASM backend
+     */
+    SQLiteVectorDB.prototype.initializeAsync = function (config) {
+        return __awaiter(this, void 0, void 0, function () {
+            var wasmBackend, finalConfig;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!(this.backendType === backend_interface_1.BackendType.WASM)) return [3 /*break*/, 2];
+                        wasmBackend = this.backend;
+                        finalConfig = config || wasmBackend._pendingConfig || {};
+                        return [4 /*yield*/, wasmBackend.initializeAsync(finalConfig)];
+                    case 1:
+                        _a.sent();
+                        delete wasmBackend._pendingConfig;
+                        _a.label = 2;
+                    case 2: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get backend type
+     */
+    SQLiteVectorDB.prototype.getBackendType = function () {
+        return this.backendType;
+    };
+    /**
+     * Check if backend is initialized
+     */
+    SQLiteVectorDB.prototype.isInitialized = function () {
+        if (this.backendType === backend_interface_1.BackendType.WASM) {
+            return this.backend.isInitialized();
+        }
+        return this.backend.isInitialized();
+    };
+    /**
+     * Insert a single vector
+     */
+    SQLiteVectorDB.prototype.insert = function (vector) {
+        return this.backend.insert(vector);
+    };
+    /**
+     * Insert multiple vectors in a transaction
+     */
+    SQLiteVectorDB.prototype.insertBatch = function (vectors) {
+        return this.backend.insertBatch(vectors);
+    };
+    /**
+     * Search for k-nearest neighbors with optional caching
+     */
+    SQLiteVectorDB.prototype.search = function (queryEmbedding, k, metric, threshold) {
+        if (k === void 0) { k = 5; }
+        if (metric === void 0) { metric = 'cosine'; }
+        if (threshold === void 0) { threshold = 0.0; }
+        // Check cache if enabled
+        if (this.queryCache) {
+            var cacheKey = query_cache_1.QueryCache.generateKey(queryEmbedding, k, metric, threshold);
+            var cached = this.queryCache.get(cacheKey);
+            if (cached) {
+                return cached;
+            }
+            // Cache miss - perform search and cache result
+            var results = this.backend.search(queryEmbedding, k, metric, threshold);
+            this.queryCache.set(cacheKey, results);
+            return results;
+        }
+        // No cache - perform search directly
+        return this.backend.search(queryEmbedding, k, metric, threshold);
+    };
+    /**
+     * Get vector by ID
+     */
+    SQLiteVectorDB.prototype.get = function (id) {
+        return this.backend.get(id);
+    };
+    /**
+     * Delete vector by ID
+     */
+    SQLiteVectorDB.prototype.delete = function (id) {
+        return this.backend.delete(id);
+    };
+    /**
+     * Get database statistics
+     */
+    SQLiteVectorDB.prototype.stats = function () {
+        return this.backend.stats();
+    };
+    /**
+     * Close database connection
+     */
+    SQLiteVectorDB.prototype.close = function () {
+        this.backend.close();
+    };
+    /**
+     * Export database to binary format (WASM only)
+     */
+    SQLiteVectorDB.prototype.export = function () {
+        if (this.backend.export) {
+            return this.backend.export();
+        }
+        throw new Error('Export not supported by current backend');
+    };
+    /**
+     * Import database from binary format (WASM only)
+     */
+    SQLiteVectorDB.prototype.importAsync = function (data) {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!(this.backendType === backend_interface_1.BackendType.WASM)) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.backend.importAsync(data)];
+                    case 1:
+                        _a.sent();
+                        return [3 /*break*/, 3];
+                    case 2: throw new Error('Import not supported by native backend');
+                    case 3: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get raw backend instance (for advanced usage)
+     */
+    SQLiteVectorDB.prototype.getBackend = function () {
+        return this.backend;
+    };
+    /**
+     * Get raw database instance (for advanced usage with native backend)
+     * @deprecated Use getBackend() instead for cross-backend compatibility
+     */
+    SQLiteVectorDB.prototype.getDatabase = function () {
+        if (this.backendType === backend_interface_1.BackendType.NATIVE) {
+            return this.backend.getDatabase();
+        }
+        throw new Error('getDatabase() only supported on native backend. Use getBackend() instead.');
+    };
+    /**
+     * Get query cache instance
+     */
+    SQLiteVectorDB.prototype.getQueryCache = function () {
+        return this.queryCache;
+    };
+    /**
+     * Get query cache statistics
+     */
+    SQLiteVectorDB.prototype.getCacheStats = function () {
+        var _a;
+        return (_a = this.queryCache) === null || _a === void 0 ? void 0 : _a.getStats();
+    };
+    /**
+     * Clear query cache
+     */
+    SQLiteVectorDB.prototype.clearCache = function () {
+        var _a;
+        (_a = this.queryCache) === null || _a === void 0 ? void 0 : _a.clear();
+    };
+    /**
+     * Get quantizer instance
+     */
+    SQLiteVectorDB.prototype.getQuantizer = function () {
+        return this.quantizer;
+    };
+    /**
+     * Train quantizer with existing vectors
+     */
+    SQLiteVectorDB.prototype.trainQuantizer = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var allStats, vectors;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.quantizer) {
+                            throw new Error('Quantization not enabled');
+                        }
+                        allStats = this.stats();
+                        if (allStats.count === 0) {
+                            throw new Error('No vectors to train quantizer');
+                        }
+                        vectors = [];
+                        // Note: This would need backend support to efficiently fetch all embeddings
+                        console.log('Training quantizer with existing vectors...');
+                        return [4 /*yield*/, this.quantizer.train(vectors)];
+                    case 1:
+                        _a.sent();
+                        console.log('Quantizer training complete');
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get compression statistics (if quantization enabled)
+     */
+    SQLiteVectorDB.prototype.getCompressionStats = function () {
+        var _a;
+        return (_a = this.quantizer) === null || _a === void 0 ? void 0 : _a.getStats();
+    };
+    /**
+     * Create a fluent query builder for complex queries
+     *
+     * @returns VectorQueryBuilder instance
+     *
+     * @example
+     * ```typescript
+     * const results = await db.query()
+     *   .similarTo(queryVector)
+     *   .where('metadata.category', '=', 'tech')
+     *   .whereBetween('metadata.year', 2020, 2024)
+     *   .orderBySimilarity('desc')
+     *   .limit(10)
+     *   .execute();
+     * ```
+     */
+    SQLiteVectorDB.prototype.query = function () {
+        return new query_builder_1.VectorQueryBuilder(this);
+    };
+    return SQLiteVectorDB;
+}());
+exports.SQLiteVectorDB = SQLiteVectorDB;

package/dist/cli/core/wasm-backend.d.ts

@@ -0,0 +1,95 @@
+/**
+ * WASM backend implementation using sql.js
+ * Provides browser-compatible vector database operations
+ */
+import { Vector, SearchResult, SimilarityMetric, DatabaseConfig } from '../types';
+import { VectorBackend } from './backend-interface';
+/**
+ * WASM backend using sql.js
+ * Runs in both Node.js and browser environments
+ */
+export declare class WasmBackend implements VectorBackend {
+    private SQL;
+    private db;
+    private initialized;
+    /**
+     * Initialize the WASM backend
+     */
+    initializeAsync(config?: DatabaseConfig): Promise<void>;
+    /**
+     * Synchronous initialization (throws if not already initialized)
+     */
+    initialize(config?: DatabaseConfig): void;
+    /**
+     * Initialize database schema
+     */
+    private initializeSchema;
+    /**
+     * Register custom SQL functions for similarity calculations
+     */
+    private registerCustomFunctions;
+    /**
+     * Calculate L2 norm of a vector
+     */
+    private calculateNorm;
+    /**
+     * Serialize embedding to Uint8Array (binary format)
+     */
+    private serializeEmbedding;
+    /**
+     * Deserialize embedding from Uint8Array
+     */
+    private deserializeEmbedding;
+    /**
+     * Generate unique ID
+     */
+    private generateId;
+    /**
+     * Insert a single vector
+     */
+    insert(vector: Vector): string;
+    /**
+     * Insert multiple vectors in a transaction with optimized batch processing
+     * Performance improvements:
+     * - Single transaction for all inserts
+     * - Prepared statement created once and reused
+     * - No wrapper function overhead
+     * - Chunked processing for large batches
+     */
+    insertBatch(vectors: Vector[]): string[];
+    /**
+     * Search for k-nearest neighbors
+     */
+    search(queryEmbedding: number[], k?: number, metric?: SimilarityMetric, threshold?: number): SearchResult[];
+    /**
+     * Get vector by ID
+     */
+    get(id: string): Vector | null;
+    /**
+     * Delete vector by ID
+     */
+    delete(id: string): boolean;
+    /**
+     * Get database statistics
+     */
+    stats(): {
+        count: number;
+        size: number;
+    };
+    /**
+     * Export database to binary format (for persistence)
+     */
+    export(): Uint8Array;
+    /**
+     * Import database from binary format
+     */
+    importAsync(data: Uint8Array): Promise<void>;
+    /**
+     * Close database and cleanup resources
+     */
+    close(): void;
+    /**
+     * Check if backend is initialized
+     */
+    isInitialized(): boolean;
+}