agentdb 1.0.0

package/dist/cli/core/wasm-backend.js

@@ -0,0 +1,418 @@
+"use strict";
+/**
+ * WASM backend implementation using sql.js
+ * Provides browser-compatible vector database operations
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WasmBackend = void 0;
+var sql_js_1 = __importDefault(require("sql.js"));
+/**
+ * WASM backend using sql.js
+ * Runs in both Node.js and browser environments
+ */
+var WasmBackend = /** @class */ (function () {
+    function WasmBackend() {
+        this.SQL = null;
+        this.db = null;
+        this.initialized = false;
+    }
+    /**
+     * Initialize the WASM backend
+     */
+    WasmBackend.prototype.initializeAsync = function () {
+        return __awaiter(this, arguments, void 0, function (config) {
+            var _a;
+            if (config === void 0) { config = {}; }
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        if (this.initialized) {
+                            return [2 /*return*/];
+                        }
+                        // Load sql.js WASM module
+                        _a = this;
+                        return [4 /*yield*/, (0, sql_js_1.default)({
+                                // Use CDN for WASM binary in browser, local file in Node.js
+                                locateFile: function (file) {
+                                    if (typeof window !== 'undefined') {
+                                        return "https://sql.js.org/dist/".concat(file);
+                                    }
+                                    return "node_modules/sql.js/dist/".concat(file);
+                                }
+                            })];
+                    case 1:
+                        // Load sql.js WASM module
+                        _a.SQL = _b.sent();
+                        // Create database
+                        this.db = new this.SQL.Database();
+                        // Initialize schema and functions
+                        this.initializeSchema(config);
+                        this.registerCustomFunctions();
+                        this.initialized = true;
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Synchronous initialization (throws if not already initialized)
+     */
+    WasmBackend.prototype.initialize = function (config) {
+        if (config === void 0) { config = {}; }
+        if (!this.initialized) {
+            throw new Error('WASM backend must be initialized asynchronously. Call initializeAsync() first.');
+        }
+    };
+    /**
+     * Initialize database schema
+     */
+    WasmBackend.prototype.initializeSchema = function (config) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var _a = config.cacheSize, cacheSize = _a === void 0 ? 100 * 1024 : _a, _b = config.walMode, walMode = _b === void 0 ? false : _b, // WAL not supported in sql.js
+        _c = config.mmapSize, // WAL not supported in sql.js
+        mmapSize = _c === void 0 ? 256 * 1024 * 1024 : _c;
+        // Configure SQLite (limited options in sql.js)
+        this.db.run('PRAGMA journal_mode = MEMORY');
+        this.db.run('PRAGMA synchronous = OFF'); // Safe for in-memory
+        this.db.run("PRAGMA cache_size = -".concat(cacheSize));
+        this.db.run('PRAGMA temp_store = MEMORY');
+        this.db.run('PRAGMA page_size = 4096');
+        // Create vectors table
+        this.db.run("\n      CREATE TABLE IF NOT EXISTS vectors (\n        id TEXT PRIMARY KEY,\n        embedding BLOB NOT NULL,\n        norm REAL NOT NULL,\n        metadata TEXT,\n        timestamp INTEGER NOT NULL\n      )\n    ");
+        // Create indexes
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_vectors_norm ON vectors(norm)');
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_vectors_timestamp ON vectors(timestamp)');
+    };
+    /**
+     * Register custom SQL functions for similarity calculations
+     */
+    WasmBackend.prototype.registerCustomFunctions = function () {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Cosine similarity function
+        this.db.create_function('cosine_similarity', function (embeddingA, embeddingB, normA, normB) {
+            var arrA = new Float32Array(embeddingA.buffer, embeddingA.byteOffset, embeddingA.byteLength / 4);
+            var arrB = new Float32Array(embeddingB.buffer, embeddingB.byteOffset, embeddingB.byteLength / 4);
+            var dotProduct = 0;
+            var len = Math.min(arrA.length, arrB.length);
+            for (var i = 0; i < len; i++) {
+                dotProduct += arrA[i] * arrB[i];
+            }
+            return dotProduct / (normA * normB);
+        });
+        // Euclidean distance function
+        this.db.create_function('euclidean_distance', function (embeddingA, embeddingB) {
+            var arrA = new Float32Array(embeddingA.buffer, embeddingA.byteOffset, embeddingA.byteLength / 4);
+            var arrB = new Float32Array(embeddingB.buffer, embeddingB.byteOffset, embeddingB.byteLength / 4);
+            var sum = 0;
+            var len = Math.min(arrA.length, arrB.length);
+            for (var i = 0; i < len; i++) {
+                var diff = arrA[i] - arrB[i];
+                sum += diff * diff;
+            }
+            return Math.sqrt(sum);
+        });
+        // Dot product function
+        this.db.create_function('dot_product', function (embeddingA, embeddingB) {
+            var arrA = new Float32Array(embeddingA.buffer, embeddingA.byteOffset, embeddingA.byteLength / 4);
+            var arrB = new Float32Array(embeddingB.buffer, embeddingB.byteOffset, embeddingB.byteLength / 4);
+            var dotProduct = 0;
+            var len = Math.min(arrA.length, arrB.length);
+            for (var i = 0; i < len; i++) {
+                dotProduct += arrA[i] * arrB[i];
+            }
+            return dotProduct;
+        });
+    };
+    /**
+     * Calculate L2 norm of a vector
+     */
+    WasmBackend.prototype.calculateNorm = function (embedding) {
+        var sum = 0;
+        for (var _i = 0, embedding_1 = embedding; _i < embedding_1.length; _i++) {
+            var val = embedding_1[_i];
+            sum += val * val;
+        }
+        return Math.sqrt(sum);
+    };
+    /**
+     * Serialize embedding to Uint8Array (binary format)
+     */
+    WasmBackend.prototype.serializeEmbedding = function (embedding) {
+        var buffer = new ArrayBuffer(embedding.length * 4);
+        var view = new Float32Array(buffer);
+        view.set(embedding);
+        return new Uint8Array(buffer);
+    };
+    /**
+     * Deserialize embedding from Uint8Array
+     */
+    WasmBackend.prototype.deserializeEmbedding = function (buffer) {
+        var view = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
+        return Array.from(view);
+    };
+    /**
+     * Generate unique ID
+     */
+    WasmBackend.prototype.generateId = function () {
+        return "vec_".concat(Date.now(), "_").concat(Math.random().toString(36).substr(2, 9));
+    };
+    /**
+     * Insert a single vector
+     */
+    WasmBackend.prototype.insert = function (vector) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var id = vector.id || this.generateId();
+        var norm = this.calculateNorm(vector.embedding);
+        var embedding = this.serializeEmbedding(vector.embedding);
+        var metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
+        var timestamp = vector.timestamp || Date.now();
+        this.db.run('INSERT OR REPLACE INTO vectors (id, embedding, norm, metadata, timestamp) VALUES (?, ?, ?, ?, ?)', [id, embedding, norm, metadata, timestamp]);
+        return id;
+    };
+    /**
+     * Insert multiple vectors in a transaction with optimized batch processing
+     * Performance improvements:
+     * - Single transaction for all inserts
+     * - Prepared statement created once and reused
+     * - No wrapper function overhead
+     * - Chunked processing for large batches
+     */
+    WasmBackend.prototype.insertBatch = function (vectors) {
+        var _this = this;
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var ids = [];
+        var CHUNK_SIZE = 5000; // Process in chunks to avoid memory spikes
+        // Helper function to process a chunk of vectors
+        var processChunk = function (chunk) {
+            var chunkIds = [];
+            _this.db.run('BEGIN TRANSACTION');
+            try {
+                // Prepare statement once for all inserts in this chunk
+                var stmt = _this.db.prepare('INSERT OR REPLACE INTO vectors (id, embedding, norm, metadata, timestamp) VALUES (?, ?, ?, ?, ?)');
+                try {
+                    // Reuse the prepared statement for all inserts
+                    for (var _i = 0, chunk_1 = chunk; _i < chunk_1.length; _i++) {
+                        var vector = chunk_1[_i];
+                        var id = vector.id || _this.generateId();
+                        var norm = _this.calculateNorm(vector.embedding);
+                        var embedding = _this.serializeEmbedding(vector.embedding);
+                        var metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
+                        var timestamp = vector.timestamp || Date.now();
+                        stmt.bind([id, embedding, norm, metadata, timestamp]);
+                        stmt.step();
+                        stmt.reset();
+                        chunkIds.push(id);
+                    }
+                }
+                finally {
+                    stmt.free();
+                }
+                _this.db.run('COMMIT');
+            }
+            catch (error) {
+                _this.db.run('ROLLBACK');
+                throw error;
+            }
+            return chunkIds;
+        };
+        // Process vectors in chunks
+        if (vectors.length <= CHUNK_SIZE) {
+            // Small batch - process directly
+            ids.push.apply(ids, processChunk(vectors));
+        }
+        else {
+            // Large batch - process in chunks to manage memory
+            for (var i = 0; i < vectors.length; i += CHUNK_SIZE) {
+                var chunk = vectors.slice(i, i + CHUNK_SIZE);
+                ids.push.apply(ids, processChunk(chunk));
+            }
+        }
+        return ids;
+    };
+    /**
+     * Search for k-nearest neighbors
+     */
+    WasmBackend.prototype.search = function (queryEmbedding, k, metric, threshold) {
+        if (k === void 0) { k = 5; }
+        if (metric === void 0) { metric = 'cosine'; }
+        if (threshold === void 0) { threshold = 0.0; }
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var queryBuffer = this.serializeEmbedding(queryEmbedding);
+        var queryNorm = this.calculateNorm(queryEmbedding);
+        var sql;
+        var params;
+        switch (metric) {
+            case 'cosine':
+                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            cosine_similarity(embedding, ?, norm, ?) as score\n          FROM vectors\n          WHERE cosine_similarity(embedding, ?, norm, ?) >= ?\n          ORDER BY score DESC\n          LIMIT ?\n        ";
+                params = [queryBuffer, queryNorm, queryBuffer, queryNorm, threshold, k];
+                break;
+            case 'euclidean':
+                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            euclidean_distance(embedding, ?) as score\n          FROM vectors\n          WHERE euclidean_distance(embedding, ?) <= ? OR ? = 0\n          ORDER BY score ASC\n          LIMIT ?\n        ";
+                params = [queryBuffer, queryBuffer, threshold, threshold, k];
+                break;
+            case 'dot':
+                sql = "\n          SELECT\n            id,\n            embedding,\n            metadata,\n            dot_product(embedding, ?) as score\n          FROM vectors\n          WHERE dot_product(embedding, ?) >= ?\n          ORDER BY score DESC\n          LIMIT ?\n        ";
+                params = [queryBuffer, queryBuffer, threshold, k];
+                break;
+            default:
+                throw new Error("Unsupported similarity metric: ".concat(metric));
+        }
+        var stmt = this.db.prepare(sql);
+        stmt.bind(params);
+        var results = [];
+        while (stmt.step()) {
+            var row = stmt.getAsObject();
+            results.push({
+                id: row.id,
+                score: row.score,
+                embedding: this.deserializeEmbedding(row.embedding),
+                metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+            });
+        }
+        stmt.free();
+        return results;
+    };
+    /**
+     * Get vector by ID
+     */
+    WasmBackend.prototype.get = function (id) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var stmt = this.db.prepare('SELECT * FROM vectors WHERE id = ?');
+        stmt.bind([id]);
+        if (!stmt.step()) {
+            stmt.free();
+            return null;
+        }
+        var row = stmt.getAsObject();
+        stmt.free();
+        return {
+            id: row.id,
+            embedding: this.deserializeEmbedding(row.embedding),
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
+            norm: row.norm,
+            timestamp: row.timestamp
+        };
+    };
+    /**
+     * Delete vector by ID
+     */
+    WasmBackend.prototype.delete = function (id) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        var stmt = this.db.prepare('DELETE FROM vectors WHERE id = ?');
+        stmt.bind([id]);
+        stmt.step();
+        var changes = this.db.getRowsModified();
+        stmt.free();
+        return changes > 0;
+    };
+    /**
+     * Get database statistics
+     */
+    WasmBackend.prototype.stats = function () {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Get count
+        var countStmt = this.db.prepare('SELECT COUNT(*) as count FROM vectors');
+        countStmt.step();
+        var countRow = countStmt.getAsObject();
+        var count = countRow.count;
+        countStmt.free();
+        // Get size (approximate from exported binary)
+        var exported = this.db.export();
+        var size = exported.byteLength;
+        return { count: count, size: size };
+    };
+    /**
+     * Export database to binary format (for persistence)
+     */
+    WasmBackend.prototype.export = function () {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        return this.db.export();
+    };
+    /**
+     * Import database from binary format
+     */
+    WasmBackend.prototype.importAsync = function (data) {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                if (!this.SQL) {
+                    throw new Error('SQL.js not initialized');
+                }
+                // Close existing database
+                if (this.db) {
+                    this.db.close();
+                }
+                // Load from binary data
+                this.db = new this.SQL.Database(data);
+                this.registerCustomFunctions();
+                this.initialized = true;
+                return [2 /*return*/];
+            });
+        });
+    };
+    /**
+     * Close database and cleanup resources
+     */
+    WasmBackend.prototype.close = function () {
+        if (this.db) {
+            this.db.close();
+            this.db = null;
+        }
+        this.initialized = false;
+    };
+    /**
+     * Check if backend is initialized
+     */
+    WasmBackend.prototype.isInitialized = function () {
+        return this.initialized;
+    };
+    return WasmBackend;
+}());
+exports.WasmBackend = WasmBackend;

package/dist/cli/db-commands.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Database CLI Commands
+ * Full-featured CLI commands for database operations
+ */
+import { Command } from 'commander';
+/**
+ * Format options for import/export
+ */
+type FileFormat = 'json' | 'csv';
+/**
+ * Import vectors from file
+ */
+export declare function importVectors(dbPath: string, filePath: string, options?: {
+    format?: FileFormat;
+    batchSize?: number;
+    verbose?: boolean;
+}): Promise<void>;
+/**
+ * Export vectors to file
+ */
+export declare function exportVectors(dbPath: string, filePath: string, options?: {
+    format?: FileFormat;
+    limit?: number;
+    where?: string;
+    verbose?: boolean;
+}): Promise<void>;
+/**
+ * Query database with vector
+ */
+export declare function queryVectors(dbPath: string, embeddingInput: string, options?: {
+    k?: number;
+    metric?: 'cosine' | 'euclidean' | 'dot';
+    threshold?: number;
+    format?: 'table' | 'json';
+    verbose?: boolean;
+}): Promise<void>;
+/**
+ * Show database statistics
+ */
+export declare function showStats(dbPath: string, options?: {
+    detailed?: boolean;
+    format?: 'table' | 'json';
+}): Promise<void>;
+/**
+ * Setup database CLI commands
+ */
+export declare function setupDatabaseCommands(program: Command): void;
+export {};
+//# sourceMappingURL=db-commands.d.ts.map

package/dist/cli/db-commands.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"db-commands.d.ts","sourceRoot":"","sources":["../../src/cli/db-commands.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC;;GAEG;AACH,KAAK,UAAU,GAAG,MAAM,GAAG,KAAK,CAAC;AAiFjC;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IACP,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,GACL,OAAO,CAAC,IAAI,CAAC,CA6Jf;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IACP,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,GACL,OAAO,CAAC,IAAI,CAAC,CAsFf;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,OAAO,GAAE;IACP,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,KAAK,CAAC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,GACL,OAAO,CAAC,IAAI,CAAC,CAuFf;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CACtB,GACL,OAAO,CAAC,IAAI,CAAC,CA4Ff;AAeD;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAgE5D"}