doc2vec 1.3.0 → 2.2.0

package/dist/database.js

@@ -59,12 +59,16 @@ class DatabaseManager {
                     embedding FLOAT[3072],
                     product_name TEXT,
                     version TEXT,
+                    branch TEXT,
+                    repo TEXT,
                     heading_hierarchy TEXT,
                     section TEXT,
                     chunk_id TEXT UNIQUE,
                     content TEXT,
                     url TEXT,
-                    hash TEXT
+                    hash TEXT,
+                    chunk_index INTEGER,
+                    total_chunks INTEGER
                 );
             `);
             logger.info(`SQLite database initialized successfully`);
@@ -139,6 +143,82 @@ class DatabaseManager {
             // Nothing special to initialize as we'll use the same collection
         }
     }
+    static async getMetadataValue(dbConnection, key, defaultValue, logger) {
+        try {
+            if (dbConnection.type === 'sqlite') {
+                const stmt = dbConnection.db.prepare('SELECT value FROM vec_metadata WHERE key = ?');
+                const result = stmt.get(key);
+                if (result) {
+                    logger.debug(`Retrieved metadata value for ${key}: ${result.value}`);
+                    return result.value;
+                }
+            }
+            else if (dbConnection.type === 'qdrant') {
+                const metadataUUID = utils_1.Utils.generateMetadataUUID(key);
+                logger.debug(`Looking up metadata with UUID: ${metadataUUID}`);
+                try {
+                    const response = await dbConnection.client.retrieve(dbConnection.collectionName, {
+                        ids: [metadataUUID],
+                        with_payload: true,
+                        with_vector: false
+                    });
+                    if (response.length > 0 && response[0].payload?.metadata_value) {
+                        const value = response[0].payload.metadata_value;
+                        logger.debug(`Retrieved metadata value for ${key}: ${value}`);
+                        return value;
+                    }
+                }
+                catch (error) {
+                    logger.warn(`Failed to retrieve metadata for ${key}:`, error);
+                }
+            }
+        }
+        catch (error) {
+            logger.warn(`Error retrieving metadata value for ${key}:`, error);
+        }
+        if (defaultValue !== undefined) {
+            logger.debug(`No metadata value found for ${key}, using default: ${defaultValue}`);
+        }
+        return defaultValue;
+    }
+    static async setMetadataValue(dbConnection, key, value, logger) {
+        try {
+            if (dbConnection.type === 'sqlite') {
+                const stmt = dbConnection.db.prepare(`
+                    INSERT INTO vec_metadata (key, value) VALUES (?, ?)
+                    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+                `);
+                stmt.run(key, value);
+                logger.debug(`Updated metadata value for ${key}`);
+            }
+            else if (dbConnection.type === 'qdrant') {
+                const metadataUUID = utils_1.Utils.generateMetadataUUID(key);
+                const dummyEmbeddingSize = 3072;
+                const dummyEmbedding = new Array(dummyEmbeddingSize).fill(0);
+                const metadataPoint = {
+                    id: metadataUUID,
+                    vector: dummyEmbedding,
+                    payload: {
+                        metadata_key: key,
+                        metadata_value: value,
+                        is_metadata: true,
+                        content: `Metadata: ${key}`,
+                        product_name: 'system',
+                        version: 'metadata',
+                        url: 'metadata://' + key
+                    }
+                };
+                await dbConnection.client.upsert(dbConnection.collectionName, {
+                    wait: true,
+                    points: [metadataPoint]
+                });
+                logger.debug(`Updated metadata value for ${key}`);
+            }
+        }
+        catch (error) {
+            logger.error(`Failed to update metadata value for ${key}:`, error);
+        }
+    }
     static async getLastRunDate(dbConnection, repo, defaultDate, logger) {
         const metadataKey = `last_run_${repo.replace('/', '_')}`;
         try {
@@ -224,40 +304,98 @@ class DatabaseManager {
         }
     }
     static prepareSQLiteStatements(db) {
+        let cached = this.columnCache.get(db);
+        if (!cached) {
+            cached = {
+                hasBranch: this.hasColumn(db, 'branch'),
+                hasRepo: this.hasColumn(db, 'repo')
+            };
+            this.columnCache.set(db, cached);
+        }
+        const hasBranchColumn = cached.hasBranch;
+        const hasRepoColumn = cached.hasRepo;
+        const insertColumns = [
+            'embedding',
+            'product_name',
+            'version',
+            ...(hasBranchColumn ? ['branch'] : []),
+            ...(hasRepoColumn ? ['repo'] : []),
+            'heading_hierarchy',
+            'section',
+            'chunk_id',
+            'content',
+            'url',
+            'hash',
+            'chunk_index',
+            'total_chunks'
+        ];
+        const insertPlaceholders = insertColumns.map(() => '?').join(', ');
+        const updateSet = [
+            'embedding = ?',
+            'product_name = ?',
+            'version = ?',
+            ...(hasBranchColumn ? ['branch = ?'] : []),
+            ...(hasRepoColumn ? ['repo = ?'] : []),
+            'heading_hierarchy = ?',
+            'section = ?',
+            'content = ?',
+            'url = ?',
+            'hash = ?',
+            'chunk_index = ?',
+            'total_chunks = ?'
+        ].join(', ');
         return {
             insertStmt: db.prepare(`
-                INSERT INTO vec_items (embedding, product_name, version, heading_hierarchy, section, chunk_id, content, url, hash)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                INSERT INTO vec_items (${insertColumns})
+                VALUES (${insertPlaceholders})
             `),
             checkHashStmt: db.prepare(`SELECT hash FROM vec_items WHERE chunk_id = ?`),
             updateStmt: db.prepare(`
-                UPDATE vec_items SET embedding = ?, product_name = ?, version = ?, heading_hierarchy = ?, section = ?, content = ?, url = ?, hash = ?
+                UPDATE vec_items SET ${updateSet}
                 WHERE chunk_id = ?
             `),
             getAllChunkIdsStmt: db.prepare(`SELECT chunk_id FROM vec_items`),
-            deleteChunkStmt: db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`)
+            deleteChunkStmt: db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`),
+            hasBranchColumn,
+            hasRepoColumn
         };
     }
     static insertVectorsSQLite(db, chunk, embedding, logger, chunkHash) {
-        const { insertStmt, updateStmt } = this.prepareSQLiteStatements(db);
+        const { insertStmt, updateStmt, hasBranchColumn, hasRepoColumn } = this.prepareSQLiteStatements(db);
         const hash = chunkHash || utils_1.Utils.generateHash(chunk.content);
         const transaction = db.transaction(() => {
-            const params = [
-                new Float32Array(embedding),
-                chunk.metadata.product_name,
-                chunk.metadata.version,
-                JSON.stringify(chunk.metadata.heading_hierarchy),
-                chunk.metadata.section,
-                chunk.metadata.chunk_id,
-                chunk.content,
-                chunk.metadata.url,
-                hash
-            ];
+            // Use BigInt for true integer representation in SQLite vec0
+            const chunkIndex = BigInt(chunk.metadata.chunk_index | 0);
+            const totalChunks = BigInt(chunk.metadata.total_chunks | 0);
             try {
-                insertStmt.run(params);
+                const insertValues = [
+                    new Float32Array(embedding),
+                    chunk.metadata.product_name,
+                    chunk.metadata.version
+                ];
+                if (hasBranchColumn) {
+                    insertValues.push(chunk.metadata.branch ?? null);
+                }
+                if (hasRepoColumn) {
+                    insertValues.push(chunk.metadata.repo ?? null);
+                }
+                insertValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.metadata.chunk_id, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks);
+                insertStmt.run(...insertValues);
             }
             catch (error) {
-                updateStmt.run([...params.slice(0, 8), chunk.metadata.chunk_id]);
+                const updateValues = [
+                    new Float32Array(embedding),
+                    chunk.metadata.product_name,
+                    chunk.metadata.version
+                ];
+                if (hasBranchColumn) {
+                    updateValues.push(chunk.metadata.branch ?? null);
+                }
+                if (hasRepoColumn) {
+                    updateValues.push(chunk.metadata.repo ?? null);
+                }
+                updateValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks, chunk.metadata.chunk_id);
+                updateStmt.run(...updateValues);
             }
         });
         transaction();
@@ -283,11 +421,15 @@ class DatabaseManager {
                     content: chunk.content,
                     product_name: chunk.metadata.product_name,
                     version: chunk.metadata.version,
+                    branch: chunk.metadata.branch,
+                    repo: chunk.metadata.repo,
                     heading_hierarchy: chunk.metadata.heading_hierarchy,
                     section: chunk.metadata.section,
                     url: chunk.metadata.url,
                     hash: hash,
                     original_chunk_id: chunk.metadata.chunk_id,
+                    chunk_index: chunk.metadata.chunk_index,
+                    total_chunks: chunk.metadata.total_chunks,
                 },
             };
             await client.upsert(collectionName, {
@@ -370,6 +512,49 @@ class DatabaseManager {
             logger.error(`Error removing obsolete chunks from Qdrant:`, error);
         }
     }
+    static hasColumn(db, columnName) {
+        try {
+            const columns = db.prepare('PRAGMA table_info(vec_items)').all();
+            return columns.some((column) => column.name === columnName);
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    static removeChunksByUrlSQLite(db, url, logger) {
+        const deleteStmt = db.prepare(`DELETE FROM vec_items WHERE url = ?`);
+        const result = deleteStmt.run(url);
+        logger.info(`Deleted ${result.changes} chunks from SQLite for URL ${url}`);
+    }
+    static async removeChunksByUrlQdrant(db, url, logger) {
+        const { client, collectionName } = db;
+        try {
+            await client.delete(collectionName, {
+                filter: {
+                    must: [
+                        {
+                            key: 'url',
+                            match: {
+                                text: url
+                            }
+                        }
+                    ],
+                    must_not: [
+                        {
+                            key: 'is_metadata',
+                            match: {
+                                value: true
+                            }
+                        }
+                    ]
+                }
+            });
+            logger.info(`Deleted chunks from Qdrant for URL ${url}`);
+        }
+        catch (error) {
+            logger.error(`Error deleting chunks from Qdrant for URL ${url}:`, error);
+        }
+    }
     static removeObsoleteFilesSQLite(db, processedFiles, pathConfig, logger) {
         const getChunksForPathStmt = db.prepare(`
             SELECT chunk_id, url FROM vec_items
@@ -505,3 +690,4 @@ class DatabaseManager {
     }
 }
 exports.DatabaseManager = DatabaseManager;
+DatabaseManager.columnCache = new WeakMap();