@noyrax/5d-database-plugin 0.1.0

package/out/core/vss-vector-database.js

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VssVectorDatabase = void 0;
+/**
+ * VSS-based vector database implementation.
+ * Uses SQLite VSS Extension for optimized vector similarity search.
+ * Only available on macOS and Linux (Windows not supported).
+ */
+class VssVectorDatabase {
+    constructor(db, vssLoader) {
+        this.vssAvailable = false;
+        this.embeddingDimensions = 1536;
+        this.db = db;
+        this.vssLoader = vssLoader;
+    }
+    /**
+     * Checks if VSS extension is available.
+     */
+    isAvailable() {
+        return this.vssAvailable;
+    }
+    /**
+     * Initializes VSS Virtual Table.
+     * Must be called after VSS extension is loaded.
+     *
+     * @returns Promise that resolves when VSS is initialized
+     */
+    async initialize() {
+        // Check if VSS is available
+        this.vssAvailable = this.vssLoader.isAvailable();
+        if (!this.vssAvailable) {
+            console.warn('[VssVectorDatabase] VSS extension not available. Using fallback cosine similarity.');
+            return;
+        }
+        try {
+            // Check if virtual table already exists
+            const tableExists = await this.checkVirtualTableExists();
+            if (!tableExists) {
+                // Create VSS virtual table
+                await this.createVirtualTable();
+                console.log('[VssVectorDatabase] Created embeddings_vss virtual table.');
+            }
+            else {
+                console.log('[VssVectorDatabase] embeddings_vss virtual table already exists.');
+            }
+        }
+        catch (error) {
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            console.warn(`[VssVectorDatabase] Failed to initialize VSS virtual table: ${errorMsg}. Using fallback cosine similarity.`);
+            this.vssAvailable = false;
+        }
+    }
+    /**
+     * Checks if the VSS virtual table exists.
+     */
+    async checkVirtualTableExists() {
+        return new Promise((resolve, reject) => {
+            this.db.get(`SELECT name FROM sqlite_master WHERE type='virtual table' AND name='embeddings_vss'`, (err, row) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(row !== undefined);
+                }
+            });
+        });
+    }
+    /**
+     * Creates the VSS virtual table.
+     */
+    async createVirtualTable() {
+        return new Promise((resolve, reject) => {
+            this.db.run(`CREATE VIRTUAL TABLE embeddings_vss USING vss0(
+                    embedding_vector(${this.embeddingDimensions}),
+                    rowid HIDDEN
+                )`, (err) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
+    /**
+     * Gets the rowid for an embedding ID.
+     */
+    async getRowidFromEmbeddingId(embeddingId) {
+        return new Promise((resolve, reject) => {
+            this.db.get(`SELECT rowid FROM embeddings WHERE id = ?`, [embeddingId], (err, row) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(row ? row.rowid : null);
+                }
+            });
+        });
+    }
+    /**
+     * Gets the embedding ID from a rowid.
+     */
+    async getEmbeddingIdFromRowid(rowid) {
+        return new Promise((resolve, reject) => {
+            this.db.get(`SELECT id FROM embeddings WHERE rowid = ?`, [rowid], (err, row) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(row ? row.id : null);
+                }
+            });
+        });
+    }
+    /**
+     * Checks if a row exists in the VSS virtual table.
+     */
+    async checkRowExists(rowid) {
+        return new Promise((resolve, reject) => {
+            this.db.get(`SELECT count(*) as count FROM embeddings_vss WHERE rowid = ?`, [rowid], (err, row) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(row ? row.count > 0 : false);
+                }
+            });
+        });
+    }
+    /**
+     * Inserts an embedding into the VSS virtual table.
+     */
+    async insertEmbedding(rowid, vector) {
+        return new Promise((resolve, reject) => {
+            // VSS expects vector as JSON array
+            const vectorArray = Array.from(vector);
+            this.db.run(`INSERT INTO embeddings_vss(rowid, embedding_vector) VALUES (?, ?)`, [rowid, JSON.stringify(vectorArray)], (err) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
+    /**
+     * Deletes an embedding from the VSS virtual table.
+     */
+    async deleteEmbedding(rowid) {
+        return new Promise((resolve, reject) => {
+            this.db.run(`DELETE FROM embeddings_vss WHERE rowid = ?`, [rowid], (err) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
+    /**
+     * Upserts an embedding into the VSS virtual table.
+     */
+    async upsertEmbedding(embeddingId, embeddingVector) {
+        if (!this.vssAvailable) {
+            return;
+        }
+        try {
+            let vector;
+            if (embeddingVector instanceof Float32Array) {
+                vector = embeddingVector;
+            }
+            else if (embeddingVector instanceof Buffer) {
+                vector = new Float32Array(embeddingVector.buffer, embeddingVector.byteOffset, embeddingVector.length / 4);
+            }
+            else if (Array.isArray(embeddingVector)) {
+                vector = new Float32Array(embeddingVector);
+            }
+            else {
+                throw new Error(`Invalid embedding vector type: ${typeof embeddingVector}`);
+            }
+            const rowid = await this.getRowidFromEmbeddingId(embeddingId);
+            if (rowid === null) {
+                console.warn(`[VssVectorDatabase] Embedding ${embeddingId} not found in embeddings table`);
+                return;
+            }
+            const exists = await this.checkRowExists(rowid);
+            if (exists) {
+                await this.deleteEmbedding(rowid);
+            }
+            await this.insertEmbedding(rowid, vector);
+        }
+        catch (error) {
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            console.warn(`[VssVectorDatabase] Failed to upsert embedding in VSS: ${errorMsg}`);
+        }
+    }
+    /**
+     * Searches for similar embeddings using VSS.
+     */
+    async search(queryEmbedding, limit = 10, dimension, pluginId) {
+        if (!this.vssAvailable) {
+            console.warn('[VssVectorDatabase] VSS not available, cannot perform VSS search.');
+            return [];
+        }
+        // Convert query embedding to array format
+        let queryVectorArray;
+        if (queryEmbedding instanceof Float32Array) {
+            queryVectorArray = Array.from(queryEmbedding);
+        }
+        else if (queryEmbedding instanceof Buffer) {
+            const float32 = new Float32Array(queryEmbedding.buffer, queryEmbedding.byteOffset, queryEmbedding.length / 4);
+            queryVectorArray = Array.from(float32);
+        }
+        else {
+            queryVectorArray = Array.isArray(queryEmbedding) ? queryEmbedding : Array.from(queryEmbedding);
+        }
+        // Build query - VSS uses MATCH syntax
+        // Note: Dimension/pluginId filtering should be done via JOIN with embeddings table
+        // For simplicity, we return all results and let the caller filter
+        const query = `SELECT rowid, distance FROM embeddings_vss WHERE embedding_vector MATCH ? LIMIT ?`;
+        const params = [JSON.stringify(queryVectorArray), limit];
+        return new Promise((resolve, reject) => {
+            this.db.all(query, params, (err, rows) => {
+                if (err) {
+                    // Try alternative approach if MATCH fails
+                    this.tryAlternativeSearch(queryVectorArray, limit, dimension, pluginId)
+                        .then(resolve)
+                        .catch(reject);
+                }
+                else {
+                    // VSS returns distance (lower = more similar)
+                    // Convert to similarity score (higher = more similar)
+                    resolve(rows.map(row => ({
+                        rowid: row.rowid,
+                        distance: row.distance || 0,
+                        similarity: 1 / (1 + Math.max(row.distance || 0, 0)) // Convert distance to similarity
+                    })));
+                }
+            });
+        });
+    }
+    /**
+     * Alternative search method (fallback if MATCH syntax doesn't work).
+     */
+    async tryAlternativeSearch(queryVector, limit, dimension, pluginId) {
+        // Try with JOIN to embeddings table for filtering
+        let query = `
+            SELECT 
+                e.rowid,
+                vss_distance_l2(e.embedding_vector, ?) as distance
+            FROM embeddings_vss e
+        `;
+        const params = [JSON.stringify(queryVector)];
+        // Add dimension filter if provided
+        if (dimension && pluginId) {
+            query += ` JOIN embeddings em ON e.rowid = em.rowid
+                WHERE em.dimension = ? AND em.plugin_id = ?`;
+            params.push(dimension, pluginId);
+        }
+        query += ` ORDER BY distance LIMIT ?`;
+        params.push(limit);
+        return new Promise((resolve, reject) => {
+            this.db.all(query, params, (err, rows) => {
+                if (err) {
+                    // If this also fails, return empty array (fallback to cosine similarity)
+                    console.warn(`[VssVectorDatabase] VSS search failed: ${err.message}. Falling back to cosine similarity.`);
+                    resolve([]);
+                }
+                else {
+                    resolve(rows.map(row => ({
+                        rowid: row.rowid,
+                        distance: row.distance || 0,
+                        similarity: 1 / (1 + Math.max(row.distance || 0, 0))
+                    })));
+                }
+            });
+        });
+    }
+}
+exports.VssVectorDatabase = VssVectorDatabase;
+//# sourceMappingURL=vss-vector-database.js.map

package/out/core/vss-vector-database.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vss-vector-database.js","sourceRoot":"","sources":["../../src/core/vss-vector-database.ts"],"names":[],"mappings":";;;AAIA;;;;GAIG;AACH,MAAa,iBAAiB;IAM1B,YAAY,EAAoB,EAAE,SAAoB;QAH9C,iBAAY,GAAY,KAAK,CAAC;QACrB,wBAAmB,GAAW,IAAI,CAAC;QAGhD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU;QACZ,4BAA4B;QAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;YACnG,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,wCAAwC;YACxC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAEzD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACf,2BAA2B;gBAC3B,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;YAC7E,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC,+DAA+D,QAAQ,qCAAqC,CAAC,CAAC;YAC3H,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC9B,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB;QACjC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP,qFAAqF,EACrF,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBACT,IAAI,GAAG,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC;gBAC/B,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB;QAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP;uCACuB,IAAI,CAAC,mBAAmB;;kBAE7C,EACF,CAAC,GAAG,EAAE,EAAE;gBACJ,IAAI,GAAG,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,EAAE,CAAC;gBACd,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB,CAAC,WAAmB;QACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP,2CAA2C,EAC3C,CAAC,WAAW,CAAC,EACb,CAAC,GAAG,EAAE,GAAkC,EAAE,EAAE;gBACxC,IAAI,GAAG,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACpC,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,uBAAuB,CAAC,KAAa;QACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP,2CAA2C,EAC3C,CAAC,KAAK,CAAC,EACP,CAAC,GAAG,EAAE,GAA+B,EAAE,EAAE;gBACrC,IAAI,GAAG,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjC,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CAAC,KAAa;QACtC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP,8DAA8D,EAC9D,CAAC,KAAK,CAAC,EACP,CAAC,GAAG,EAAE,GAAkC,EAAE,EAAE;gBACxC,IAAI,GAAG,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACzC,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,MAAoB;QAC7D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,mCAAmC;YACnC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP,mEAAmE,EACnE,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,EACpC,CAAC,GAAG,EAAE,EAAE;gBACJ,IAAI,GAAG,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,EAAE,CAAC;gBACd,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,KAAa;QACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP,4CAA4C,EAC5C,CAAC,KAAK,CAAC,EACP,CAAC,GAAG,EAAE,EAAE;gBACJ,IAAI,GAAG,EAAE,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,EAAE,CAAC;gBACd,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,WAAmB,EAAE,eAAwC;QAC/E,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,IAAI,MAAoB,CAAC;YACzB,IAAI,eAAe,YAAY,YAAY,EAAE,CAAC;gBAC1C,MAAM,GAAG,eAAe,CAAC;YAC7B,CAAC;iBAAM,IAAI,eAAe,YAAY,MAAM,EAAE,CAAC;gBAC3C,MAAM,GAAG,IAAI,YAAY,CACrB,eAAe,CAAC,MAAM,EACtB,eAAe,CAAC,UAAU,EAC1B,eAAe,CAAC,MAAM,GAAG,CAAC,CAC7B,CAAC;YACN,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACxC,MAAM,GAAG,IAAI,YAAY,CAAC,eAAe,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACJ,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,eAAe,EAAE,CAAC,CAAC;YAChF,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC;YAE9D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,iCAAiC,WAAW,gCAAgC,CAAC,CAAC;gBAC3F,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAEhD,IAAI,MAAM,EAAE,CAAC;gBACT,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACtC,CAAC;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,OAAO,CAAC,IAAI,CAAC,0DAA0D,QAAQ,EAAE,CAAC,CAAC;QACvF,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACR,cAAuC,EACvC,QAAgB,EAAE,EAClB,SAAuC,EACvC,QAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;YAClF,OAAO,EAAE,CAAC;QACd,CAAC;QAED,0CAA0C;QAC1C,IAAI,gBAA0B,CAAC;QAC/B,IAAI,cAAc,YAAY,YAAY,EAAE,CAAC;YACzC,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,cAAc,YAAY,MAAM,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,IAAI,YAAY,CAC5B,cAAc,CAAC,MAAM,EACrB,cAAc,CAAC,UAAU,EACzB,cAAc,CAAC,MAAM,GAAG,CAAC,CAC5B,CAAC;YACF,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACJ,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACnG,CAAC;QAED,sCAAsC;QACtC,mFAAmF;QACnF,kEAAkE;QAClE,MAAM,KAAK,GAAG,mFAAmF,CAAC;QAClG,MAAM,MAAM,GAAU,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,EAAE,KAAK,CAAC,CAAC;QAEhE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CACP,KAAK,EACL,MAAM,EACN,CAAC,GAAG,EAAE,IAA2C,EAAE,EAAE;gBACjD,IAAI,GAAG,EAAE,CAAC;oBACN,0CAA0C;oBAC1C,IAAI,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC;yBAClE,IAAI,CAAC,OAAO,CAAC;yBACb,KAAK,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;qBAAM,CAAC;oBACJ,8CAA8C;oBAC9C,sDAAsD;oBACtD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACrB,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC;wBAC3B,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,iCAAiC;qBACzF,CAAC,CAAC,CAAC,CAAC;gBACT,CAAC;YACL,CAAC,CACJ,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,WAAqB,EACrB,KAAa,EACb,SAAuC,EACvC,QAAiB;QAEjB,kDAAkD;QAClD,IAAI,KAAK,GAAG;;;;;SAKX,CAAC;QAEF,MAAM,MAAM,GAAU,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;QAEpD,mCAAmC;QACnC,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;YACxB,KAAK,IAAI;4DACuC,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;QAED,KAAK,IAAI,4BAA4B,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,IAAW,EAAE,EAAE;gBAC5C,IAAI,GAAG,EAAE,CAAC;oBACN,yEAAyE;oBACzE,OAAO,CAAC,IAAI,CAAC,0CAA0C,GAAG,CAAC,OAAO,sCAAsC,CAAC,CAAC;oBAC1G,OAAO,CAAC,EAAE,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACrB,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC;wBAC3B,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;qBACvD,CAAC,CAAC,CAAC,CAAC;gBACT,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AA/UD,8CA+UC"}

package/out/embedding/embedding-generator.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Generates embeddings for entities across all 5 dimensions.
+ * Uses OpenAI text-embedding-3-small model (1536 dimensions).
+ */
+export declare class EmbeddingGenerator {
+    private openai;
+    private readonly model;
+    private readonly dimensions;
+    constructor(apiKey?: string);
+    /**
+     * Generates an embedding for a single entity.
+     *
+     * @param dimension The dimension (X, Y, Z, W, or T)
+     * @param entityId The entity ID (for logging)
+     * @param content The content to embed
+     * @returns Promise that resolves to the embedding vector (1536 dimensions)
+     * @throws Error if OpenAI API is not configured or API call fails
+     */
+    generateEmbedding(dimension: 'X' | 'Y' | 'Z' | 'W' | 'T', entityId: string, content: string): Promise<number[]>;
+    /**
+     * Generates embeddings for multiple entities in batch.
+     * Uses OpenAI batch API for better performance.
+     *
+     * @param items Array of items to embed
+     * @returns Promise that resolves to a Map of entityId -> embedding vector
+     */
+    generateBatch(items: Array<{
+        dimension: 'X' | 'Y' | 'Z' | 'W' | 'T';
+        entityId: string;
+        content: string;
+    }>): Promise<Map<string, number[]>>;
+    /**
+     * Gets the embedding model name.
+     */
+    getModel(): string;
+    /**
+     * Gets the embedding dimensions.
+     */
+    getDimensions(): number;
+    /**
+     * Checks if OpenAI API is configured.
+     */
+    isConfigured(): boolean;
+}
+//# sourceMappingURL=embedding-generator.d.ts.map

package/out/embedding/embedding-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-generator.d.ts","sourceRoot":"","sources":["../../src/embedding/embedding-generator.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,qBAAa,kBAAkB;IAC3B,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoC;IAC1D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgB;gBAE/B,MAAM,CAAC,EAAE,MAAM;IAc3B;;;;;;;;OAQG;IACG,iBAAiB,CACnB,SAAS,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,EACtC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC;IA2BpB;;;;;;OAMG;IACG,aAAa,CACf,KAAK,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAC5F,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IA+DjC;;OAEG;IACH,QAAQ,IAAI,MAAM;IAIlB;;OAEG;IACH,aAAa,IAAI,MAAM;IAIvB;;OAEG;IACH,YAAY,IAAI,OAAO;CAG1B"}

package/out/embedding/embedding-generator.js

@@ -0,0 +1,143 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EmbeddingGenerator = void 0;
+const openai_1 = __importDefault(require("openai"));
+/**
+ * Generates embeddings for entities across all 5 dimensions.
+ * Uses OpenAI text-embedding-3-small model (1536 dimensions).
+ */
+class EmbeddingGenerator {
+    constructor(apiKey) {
+        this.openai = null;
+        this.model = 'text-embedding-3-small';
+        this.dimensions = 1536;
+        if (apiKey) {
+            this.openai = new openai_1.default({ apiKey });
+        }
+        else {
+            // Try to get from environment variable
+            const envKey = process.env.OPENAI_API_KEY;
+            if (envKey) {
+                this.openai = new openai_1.default({ apiKey: envKey });
+            }
+            else {
+                console.warn('[EmbeddingGenerator] OpenAI API key not provided. Embedding generation will not work.');
+            }
+        }
+    }
+    /**
+     * Generates an embedding for a single entity.
+     *
+     * @param dimension The dimension (X, Y, Z, W, or T)
+     * @param entityId The entity ID (for logging)
+     * @param content The content to embed
+     * @returns Promise that resolves to the embedding vector (1536 dimensions)
+     * @throws Error if OpenAI API is not configured or API call fails
+     */
+    async generateEmbedding(dimension, entityId, content) {
+        if (!this.openai) {
+            throw new Error('OpenAI API key not configured. Set OPENAI_API_KEY environment variable.');
+        }
+        if (!content || content.trim().length === 0) {
+            throw new Error(`Empty content for entity ${entityId} in dimension ${dimension}`);
+        }
+        try {
+            const response = await this.openai.embeddings.create({
+                model: this.model,
+                input: content,
+                dimensions: this.dimensions
+            });
+            if (!response.data || response.data.length === 0) {
+                throw new Error(`No embedding returned for entity ${entityId}`);
+            }
+            return response.data[0].embedding;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to generate embedding for entity ${entityId} in dimension ${dimension}: ${errorMessage}`);
+        }
+    }
+    /**
+     * Generates embeddings for multiple entities in batch.
+     * Uses OpenAI batch API for better performance.
+     *
+     * @param items Array of items to embed
+     * @returns Promise that resolves to a Map of entityId -> embedding vector
+     */
+    async generateBatch(items) {
+        if (!this.openai) {
+            throw new Error('OpenAI API key not configured. Set OPENAI_API_KEY environment variable.');
+        }
+        if (items.length === 0) {
+            return new Map();
+        }
+        // Filter out empty content
+        const validItems = items.filter(item => item.content && item.content.trim().length > 0);
+        if (validItems.length === 0) {
+            return new Map();
+        }
+        const results = new Map();
+        // OpenAI embeddings API supports up to 2048 inputs per request
+        // We'll process in batches of 100 for safety
+        const batchSize = 100;
+        for (let i = 0; i < validItems.length; i += batchSize) {
+            const batch = validItems.slice(i, i + batchSize);
+            try {
+                const inputs = batch.map(item => item.content);
+                const response = await this.openai.embeddings.create({
+                    model: this.model,
+                    input: inputs,
+                    dimensions: this.dimensions
+                });
+                if (!response.data || response.data.length !== batch.length) {
+                    throw new Error(`Mismatch: expected ${batch.length} embeddings, got ${response.data?.length || 0}`);
+                }
+                // Map results back to entity IDs
+                for (let j = 0; j < batch.length; j++) {
+                    const item = batch[j];
+                    const embedding = response.data[j].embedding;
+                    results.set(item.entityId, embedding);
+                }
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                console.error(`[EmbeddingGenerator] Batch embedding failed for batch ${i / batchSize + 1}: ${errorMessage}`);
+                // Fallback: Try individual embeddings for this batch
+                for (const item of batch) {
+                    try {
+                        const embedding = await this.generateEmbedding(item.dimension, item.entityId, item.content);
+                        results.set(item.entityId, embedding);
+                    }
+                    catch (individualError) {
+                        console.error(`[EmbeddingGenerator] Failed to generate embedding for ${item.entityId}: ${individualError}`);
+                        // Continue with other items
+                    }
+                }
+            }
+        }
+        return results;
+    }
+    /**
+     * Gets the embedding model name.
+     */
+    getModel() {
+        return this.model;
+    }
+    /**
+     * Gets the embedding dimensions.
+     */
+    getDimensions() {
+        return this.dimensions;
+    }
+    /**
+     * Checks if OpenAI API is configured.
+     */
+    isConfigured() {
+        return this.openai !== null;
+    }
+}
+exports.EmbeddingGenerator = EmbeddingGenerator;
+//# sourceMappingURL=embedding-generator.js.map

package/out/embedding/embedding-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-generator.js","sourceRoot":"","sources":["../../src/embedding/embedding-generator.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B;;;GAGG;AACH,MAAa,kBAAkB;IAK3B,YAAY,MAAe;QAJnB,WAAM,GAAkB,IAAI,CAAC;QACpB,UAAK,GAAW,wBAAwB,CAAC;QACzC,eAAU,GAAW,IAAI,CAAC;QAGvC,IAAI,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACJ,uCAAuC;YACvC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC1C,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YACjD,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,IAAI,CAAC,uFAAuF,CAAC,CAAC;YAC1G,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,iBAAiB,CACnB,SAAsC,EACtC,QAAgB,EAChB,OAAe;QAEf,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,iBAAiB,SAAS,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACjD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,OAAO;gBACd,UAAU,EAAE,IAAI,CAAC,UAAU;aAC9B,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,EAAE,CAAC,CAAC;YACpE,CAAC;YAED,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,2CAA2C,QAAQ,iBAAiB,SAAS,KAAK,YAAY,EAAE,CAAC,CAAC;QACtH,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CACf,KAA2F;QAE3F,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO,IAAI,GAAG,EAAE,CAAC;QACrB,CAAC;QAED,2BAA2B;QAC3B,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAExF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,IAAI,GAAG,EAAE,CAAC;QACrB,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;QAE5C,+DAA+D;QAC/D,6CAA6C;QAC7C,MAAM,SAAS,GAAG,GAAG,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAEjD,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC/C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;oBACjD,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,KAAK,EAAE,MAAM;oBACb,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC9B,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;oBAC1D,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,MAAM,oBAAoB,QAAQ,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC,CAAC;gBACxG,CAAC;gBAED,iCAAiC;gBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtB,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;oBAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5E,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,GAAG,SAAS,GAAG,CAAC,KAAK,YAAY,EAAE,CAAC,CAAC;gBAE7G,qDAAqD;gBACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC;wBACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC5F,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;oBAC1C,CAAC;oBAAC,OAAO,eAAe,EAAE,CAAC;wBACvB,OAAO,CAAC,KAAK,CAAC,yDAAyD,IAAI,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC,CAAC;wBAC5G,4BAA4B;oBAChC,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,QAAQ;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC;IAChC,CAAC;CACJ;AAvJD,gDAuJC"}

package/out/embedding/embedding-pipeline.d.ts

@@ -0,0 +1,42 @@
+import { MultiDbManager } from '../core/multi-db-manager';
+import { EmbeddingGenerator } from './embedding-generator';
+/**
+ * Coordinates embedding generation for all 5 dimensions.
+ * Syncs embeddings with 5D-DBs using hash-based change detection.
+ */
+export declare class EmbeddingPipeline {
+    private dbManager;
+    private embeddingGenerator;
+    private moduleApi;
+    private symbolApi;
+    private dependencyApi;
+    private adrApi;
+    private changeApi;
+    constructor(dbManager: MultiDbManager, embeddingGenerator: EmbeddingGenerator);
+    /**
+     * Syncs embeddings for all dimensions.
+     * Only generates embeddings for changed/new entities.
+     */
+    syncEmbeddings(pluginId: string): Promise<void>;
+    /**
+     * Syncs embeddings for a specific dimension.
+     */
+    syncDimension(dimension: 'X' | 'Y' | 'Z' | 'W' | 'T', pluginId: string, model: string): Promise<void>;
+    /**
+     * Gets all entities for a dimension.
+     */
+    private getEntitiesForDimension;
+    /**
+     * Extracts content for embedding based on dimension.
+     */
+    private extractContentForEmbedding;
+    /**
+     * Gets external ID for an entity.
+     */
+    private getExternalId;
+    /**
+     * Computes content hash for change detection.
+     */
+    private computeContentHash;
+}
+//# sourceMappingURL=embedding-pipeline.d.ts.map

package/out/embedding/embedding-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-pipeline.d.ts","sourceRoot":"","sources":["../../src/embedding/embedding-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAe3D;;;GAGG;AACH,qBAAa,iBAAiB;IAC1B,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAY;gBAEjB,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,kBAAkB;IAU7E;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBrD;;OAEG;IACG,aAAa,CACf,SAAS,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,EACtC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC;IAkJhB;;OAEG;YACW,uBAAuB;IAoBrC;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAwClC;;OAEG;IACH,OAAO,CAAC,aAAa;IAgBrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;CAG7B"}