doc2vec 2.3.0 → 2.5.0

package/dist/database.js

@@ -44,7 +44,7 @@ const sqliteVec = __importStar(require("sqlite-vec"));
 const js_client_rest_1 = require("@qdrant/js-client-rest");
 const utils_1 = require("./utils");
 class DatabaseManager {
-    static async initDatabase(config, parentLogger) {
+    static async initDatabase(config, parentLogger, embeddingDimension) {
         const logger = parentLogger.child('database');
         const dbConfig = config.database_config;
         if (dbConfig.type === 'sqlite') {
@@ -53,10 +53,10 @@ class DatabaseManager {
             logger.info(`Opening SQLite database at ${dbPath}`);
             const db = new better_sqlite3_1.default(dbPath, { allowExtension: true });
             sqliteVec.load(db);
-            logger.debug(`Creating vec_items table if it doesn't exist`);
+            logger.debug(`Creating vec_items table if it doesn't exist (dimension: ${embeddingDimension})`);
             db.exec(`
                 CREATE VIRTUAL TABLE IF NOT EXISTS vec_items USING vec0(
-                    embedding FLOAT[3072],
+                    embedding FLOAT[${embeddingDimension}],
                     product_name TEXT,
                     version TEXT,
                     branch TEXT,
@@ -81,7 +81,7 @@ class DatabaseManager {
             const collectionName = params.collection_name || `${config.product_name.toLowerCase().replace(/\s+/g, '_')}_${config.version}`;
             logger.info(`Connecting to Qdrant at ${qdrantUrl}:${qdrantPort}, collection: ${collectionName}`);
             const qdrantClient = new js_client_rest_1.QdrantClient({ url: qdrantUrl, apiKey: process.env.QDRANT_API_KEY, port: qdrantPort });
-            await this.createCollectionQdrant(qdrantClient, collectionName, logger);
+            await this.createCollectionQdrant(qdrantClient, collectionName, logger, embeddingDimension);
             logger.info(`Qdrant connection established successfully`);
             return { client: qdrantClient, collectionName, type: 'qdrant' };
         }
@@ -91,7 +91,7 @@ class DatabaseManager {
             throw new Error(errMsg);
         }
     }
-    static async createCollectionQdrant(qdrantClient, collectionName, logger) {
+    static async createCollectionQdrant(qdrantClient, collectionName, logger, embeddingDimension) {
         try {
             logger.debug(`Checking if collection ${collectionName} exists`);
             const collections = await qdrantClient.getCollections();
@@ -100,10 +100,10 @@ class DatabaseManager {
                 logger.info(`Collection ${collectionName} already exists`);
                 return;
             }
-            logger.info(`Creating new collection ${collectionName}`);
+            logger.info(`Creating new collection ${collectionName} with dimension ${embeddingDimension}`);
             await qdrantClient.createCollection(collectionName, {
                 vectors: {
-                    size: 3072,
+                    size: embeddingDimension,
                     distance: "Cosine",
                 },
             });
@@ -181,7 +181,7 @@ class DatabaseManager {
         }
         return defaultValue;
     }
-    static async setMetadataValue(dbConnection, key, value, logger) {
+    static async setMetadataValue(dbConnection, key, value, logger, embeddingDimension) {
         try {
             if (dbConnection.type === 'sqlite') {
                 const stmt = dbConnection.db.prepare(`
@@ -193,8 +193,7 @@ class DatabaseManager {
             }
             else if (dbConnection.type === 'qdrant') {
                 const metadataUUID = utils_1.Utils.generateMetadataUUID(key);
-                const dummyEmbeddingSize = 3072;
-                const dummyEmbedding = new Array(dummyEmbeddingSize).fill(0);
+                const dummyEmbedding = new Array(embeddingDimension).fill(0);
                 const metadataPoint = {
                     id: metadataUUID,
                     vector: dummyEmbedding,
@@ -258,7 +257,7 @@ class DatabaseManager {
         logger.info(`No saved run date found for ${repo}, using default: ${defaultDate}`);
         return defaultDate;
     }
-    static async updateLastRunDate(dbConnection, repo, logger) {
+    static async updateLastRunDate(dbConnection, repo, logger, embeddingDimension) {
         const now = new Date().toISOString();
         try {
             if (dbConnection.type === 'sqlite') {
@@ -276,8 +275,7 @@ class DatabaseManager {
                 const metadataKey = `last_run_${repo.replace('/', '_')}`;
                 logger.debug(`Using UUID: ${metadataUUID} for metadata`);
                 // Generate a dummy embedding (all zeros)
-                const dummyEmbeddingSize = 3072; // Same size as your content embeddings
-                const dummyEmbedding = new Array(dummyEmbeddingSize).fill(0);
+                const dummyEmbedding = new Array(embeddingDimension).fill(0);
                 // Create a point with special metadata payload
                 const metadataPoint = {
                     id: metadataUUID,
@@ -535,7 +533,7 @@ class DatabaseManager {
                         {
                             key: 'url',
                             match: {
-                                text: url
+                                value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
                             }
                         }
                     ],
@@ -555,6 +553,96 @@ class DatabaseManager {
             logger.error(`Error deleting chunks from Qdrant for URL ${url}:`, error);
         }
     }
+    /**
+     * Fetch all distinct URLs stored under a given URL prefix.
+     * Used to pre-seed the crawl queue on re-runs so link discovery isn't lost
+     * when pages are skipped via ETag matching.
+     */
+    static getStoredUrlsByPrefixSQLite(db, urlPrefix) {
+        const stmt = db.prepare('SELECT DISTINCT url FROM vec_items WHERE url LIKE ? || \'%\'');
+        const rows = stmt.all(urlPrefix);
+        return rows.map(r => r.url);
+    }
+    static async getStoredUrlsByPrefixQdrant(db, urlPrefix) {
+        const { client, collectionName } = db;
+        try {
+            const response = await client.scroll(collectionName, {
+                limit: 10000,
+                with_payload: { include: ['url'] },
+                with_vector: false,
+                filter: {
+                    must: [
+                        {
+                            key: 'url',
+                            match: { text: urlPrefix }
+                        }
+                    ],
+                    must_not: [
+                        {
+                            key: 'is_metadata',
+                            match: { value: true }
+                        }
+                    ]
+                }
+            });
+            const urls = new Set();
+            for (const point of response.points) {
+                const url = point.payload?.url;
+                if (url && url.startsWith(urlPrefix)) {
+                    urls.add(url);
+                }
+            }
+            return Array.from(urls);
+        }
+        catch (error) {
+            return [];
+        }
+    }
+    /**
+     * Fetch all stored chunk content hashes for a given URL.
+     * Returns a sorted array of hash strings for comparison.
+     */
+    static getChunkHashesByUrlSQLite(db, url) {
+        const stmt = db.prepare('SELECT hash FROM vec_items WHERE url = ?');
+        const rows = stmt.all(url);
+        return rows.map(r => r.hash).sort();
+    }
+    /**
+     * Fetch all stored chunk content hashes for a given URL from Qdrant.
+     * Returns a sorted array of hash strings for comparison.
+     */
+    static async getChunkHashesByUrlQdrant(db, url) {
+        const { client, collectionName } = db;
+        try {
+            const response = await client.scroll(collectionName, {
+                limit: 10000,
+                with_payload: { include: ['hash'] },
+                with_vector: false,
+                filter: {
+                    must: [
+                        {
+                            key: 'url',
+                            match: { value: url }
+                        }
+                    ],
+                    must_not: [
+                        {
+                            key: 'is_metadata',
+                            match: { value: true }
+                        }
+                    ]
+                }
+            });
+            return response.points
+                .map((point) => point.payload?.hash)
+                .filter(Boolean)
+                .sort();
+        }
+        catch (error) {
+            // On error, return empty array so the caller treats it as "changed" (safe default)
+            return [];
+        }
+    }
     static removeObsoleteFilesSQLite(db, processedFiles, pathConfig, logger) {
         const getChunksForPathStmt = db.prepare(`
             SELECT chunk_id, url FROM vec_items