doc2vec 2.14.2 → 2.15.0

package/dist/content-processor.js

@@ -72,12 +72,12 @@ exports.BrowserLaunchError = BrowserLaunchError;
  *     Chrome for Testing is not published)
  *  4. undefined — let puppeteer decide
  */
-function resolveBrowserExecutablePath() {
+async function resolveBrowserExecutablePath() {
     const override = process.env.PUPPETEER_EXECUTABLE_PATH;
     if (override)
         return override;
     try {
-        const bundled = puppeteer_1.default.executablePath();
+        const bundled = await puppeteer_1.default.executablePath();
         if (bundled && fs.existsSync(bundled))
             return bundled;
     }
@@ -421,7 +421,7 @@ class ContentProcessor {
         let browser = null;
         let page = null;
         const launchBrowser = async () => {
-            const executablePath = resolveBrowserExecutablePath();
+            const executablePath = await resolveBrowserExecutablePath();
             const launchOptions = {
                 executablePath,
                 args: [
@@ -487,7 +487,7 @@ class ContentProcessor {
             // If protocol errors are persisting after page recreation, the browser
             // process itself is degraded — a full restart is needed.
             const needsBrowserRestart = pageNeedsRecreation && consecutiveProtocolErrors >= 1;
-            if (!browser || !browser.isConnected() || needsBrowserRestart) {
+            if (!browser || !browser.connected || needsBrowserRestart) {
                 const reason = needsBrowserRestart
                     ? `protocol errors persisting after page recreation (${consecutiveProtocolErrors} consecutive)`
                     : browser ? 'browser disconnected' : 'initial launch';
@@ -793,7 +793,9 @@ class ContentProcessor {
                     // A browser that cannot launch is fatal for the whole crawl —
                     // rethrow so the source (and the run) is reported as failed
                     // rather than logging an error per URL and finishing "green".
-                    if (error instanceof BrowserLaunchError) {
+                    // Same for a permanent embedding failure (bad API key):
+                    // every remaining page would fail identically.
+                    if (error instanceof BrowserLaunchError || error?.name === 'FatalEmbeddingError') {
                         logger.error(`Aborting crawl of ${baseUrl}: ${error.message}`);
                         throw error;
                     }
@@ -858,7 +860,7 @@ class ContentProcessor {
         finally {
             // Close the shared browser instance when the crawl is done
             const browserToClose = browser;
-            if (browserToClose && browserToClose.isConnected()) {
+            if (browserToClose && browserToClose.connected) {
                 await browserToClose.close();
                 logger.debug('Shared browser closed after crawl completed');
             }
@@ -962,7 +964,7 @@ class ContentProcessor {
             else {
                 // Standalone mode: launch a browser for this single page
                 browser = await puppeteer_1.default.launch({
-                    executablePath: resolveBrowserExecutablePath(),
+                    executablePath: await resolveBrowserExecutablePath(),
                     args: [
                         '--no-sandbox',
                         '--disable-setuid-sandbox',
@@ -1143,7 +1145,7 @@ class ContentProcessor {
         }
         finally {
             // Only close the browser if we launched it ourselves (standalone mode)
-            if (ownsTheBrowser && browser && browser.isConnected()) {
+            if (ownsTheBrowser && browser && browser.connected) {
                 await browser.close();
                 logger.debug(`Browser closed for ${url}`);
             }

package/dist/doc2vec.js

@@ -39,7 +39,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Doc2Vec = void 0;
 const axios_1 = __importDefault(require("axios"));
-const crypto_1 = __importDefault(require("crypto"));
 const yaml = __importStar(require("js-yaml"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
@@ -481,6 +480,30 @@ class Doc2Vec {
             };
             const chunks = await this.contentProcessor.chunkMarkdown(markdown, issueConfig, url);
             logger.info(`Issue #${issueNumber}: Created ${chunks.length} chunks`);
+            // Embed every chunk BEFORE touching stored data, so an embedding
+            // failure can't purge the issue's existing chunks and store nothing
+            // back. A FatalEmbeddingError (bad API key) propagates and aborts
+            // the source; transient failures skip this issue but count toward
+            // failedChunks so the last-run date is not advanced.
+            const embedded = [];
+            let failedEmbeddings = 0;
+            for (const chunk of chunks) {
+                const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
+                const embeddings = await this.createEmbeddings([chunk.content]);
+                if (embeddings.length) {
+                    embedded.push(embeddings[0]);
+                }
+                else {
+                    failedEmbeddings++;
+                    logger.error(`Embedding failed for chunk: ${chunkId}`);
+                }
+            }
+            if (failedEmbeddings > 0) {
+                failedChunks += failedEmbeddings;
+                logger.error(`Issue #${issueNumber}: ${failedEmbeddings}/${chunks.length} chunks failed to embed; ` +
+                    `keeping its existing chunks so the next run retries`);
+                return;
+            }
             // Purge the issue's existing chunks before inserting the fresh set.
             // chunk_id is a content hash, so when an issue changes (closed/reopened,
             // edited, new comments) the regenerated chunks get new ids and the old
@@ -501,57 +524,21 @@ class Doc2Vec {
             else {
                 this.counters.items_new++;
             }
-            // Process and store each chunk immediately
-            for (const chunk of chunks) {
+            // Store each chunk
+            for (let i = 0; i < chunks.length; i++) {
+                const chunk = chunks[i];
                 const chunkHash = utils_1.Utils.generateHash(chunk.content);
                 const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
                 if (dbConnection.type === 'sqlite') {
-                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                    const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                    if (existing && existing.hash === chunkHash) {
-                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                        continue;
-                    }
-                    const embeddings = await this.createEmbeddings([chunk.content]);
-                    if (embeddings.length) {
-                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
-                        this.counters.chunks_added++;
-                        logger.debug(`Stored chunk ${chunkId} in SQLite`);
-                    }
-                    else {
-                        logger.error(`Embedding failed for chunk: ${chunkId}`);
-                    }
+                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedded[i], logger, chunkHash);
+                    this.counters.chunks_added++;
+                    logger.debug(`Stored chunk ${chunkId} in SQLite`);
                 }
                 else if (dbConnection.type === 'qdrant') {
                     try {
-                        let pointId;
-                        try {
-                            pointId = chunk.metadata.chunk_id;
-                            if (!utils_1.Utils.isValidUuid(pointId)) {
-                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                            }
-                        }
-                        catch (e) {
-                            pointId = crypto_1.default.randomUUID();
-                        }
-                        const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                            ids: [pointId],
-                            with_payload: true,
-                            with_vector: false,
-                        });
-                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                            logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            continue;
-                        }
-                        const embeddings = await this.createEmbeddings([chunk.content]);
-                        if (embeddings.length) {
-                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
-                            this.counters.chunks_added++;
-                            logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                        }
-                        else {
-                            logger.error(`Embedding failed for chunk: ${chunkId}`);
-                        }
+                        await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedded[i], chunkHash);
+                        this.counters.chunks_added++;
+                        logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
                     }
                     catch (error) {
                         // Keep going so one bad chunk can't cost the whole repo,
@@ -680,6 +667,8 @@ class Doc2Vec {
                 return true;
             }
             catch (error) {
+                if (error instanceof types_1.FatalEmbeddingError)
+                    throw error;
                 logger.error(`Error during chunking or embedding for ${url}:`, error);
                 return false;
             }
@@ -829,6 +818,8 @@ class Doc2Vec {
                 await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
             }
             catch (error) {
+                if (error instanceof types_1.FatalEmbeddingError)
+                    throw error;
                 logger.error(`Error during chunking or embedding for ${filePath}:`, error);
             }
         }, logger);
@@ -999,6 +990,8 @@ class Doc2Vec {
                 processedCount++;
             }
             catch (error) {
+                if (error instanceof types_1.FatalEmbeddingError)
+                    throw error;
                 logger.error(`Error processing S3 object ${obj.key}:`, error);
             }
         }
@@ -1155,6 +1148,8 @@ class Doc2Vec {
                     await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
                 }
                 catch (error) {
+                    if (error instanceof types_1.FatalEmbeddingError)
+                        throw error;
                     logger.error(`Error during code chunking or embedding for ${filePath}:`, error);
                 }
             }, logger, undefined, {
@@ -1594,6 +1589,8 @@ class Doc2Vec {
                     totalTickets++;
                 }
                 catch (error) {
+                    if (error instanceof types_1.FatalEmbeddingError)
+                        throw error;
                     failedTickets++;
                     logger.error(`Failed to process ticket #${ticket.id}, will retry next run: ${error.message}`);
                 }
@@ -1781,9 +1778,36 @@ class Doc2Vec {
             this.counters.items_unchanged++;
             return 0;
         }
-        // 4. Content changed — delete all existing chunks for this URL
+        // 4. Content changed — embed all new chunks BEFORE touching stored
+        // data, so an embedding failure (e.g. a bad API key) can't delete the
+        // existing chunks and then store nothing back. Throwing here also
+        // signals the caller that the item failed: for websites that means the
+        // ETag/lastmod is not stored and the page is retried next run.
+        const chunkProgress = logger.progress(`Embedding chunks for ${url}`, chunks.length);
+        const embedded = [];
+        let failedEmbeddings = 0;
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
+            const embeddings = await this.createEmbeddings([chunk.content]);
+            if (embeddings.length > 0) {
+                embedded[i] = embeddings[0];
+                chunkProgress.update(1, `Embedded chunk ${chunkId}`);
+            }
+            else {
+                failedEmbeddings++;
+                logger.error(`Embedding failed for chunk: ${chunkId}`);
+                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
+            }
+        }
+        if (failedEmbeddings > 0) {
+            chunkProgress.complete();
+            throw new Error(`embedding failed for ${failedEmbeddings}/${chunks.length} chunks of ${url}; ` +
+                `keeping the ${existingHashesSorted.length} existing chunks so the next run retries`);
+        }
+        // 5. All embeddings succeeded — delete the old chunks and insert the new set
         if (existingHashesSorted.length > 0) {
-            logger.info(`Content changed for ${url} (${existingHashesSorted.length} old chunks → ${chunks.length} new chunks), re-embedding all`);
+            logger.info(`Content changed for ${url} (${existingHashesSorted.length} old chunks → ${chunks.length} new chunks), replacing all`);
             if (dbConnection.type === 'sqlite') {
                 database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
             }
@@ -1796,34 +1820,22 @@ class Doc2Vec {
         else {
             this.counters.items_new++;
         }
-        // 5. Embed and insert all new chunks
-        let embeddedCount = 0;
-        const chunkProgress = logger.progress(`Embedding chunks for ${url}`, chunks.length);
         for (let i = 0; i < chunks.length; i++) {
             const chunk = chunks[i];
             const chunkHash = newHashes[i];
             const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-            const embeddings = await this.createEmbeddings([chunk.content]);
-            if (embeddings.length > 0) {
-                const embedding = embeddings[0];
-                if (dbConnection.type === 'sqlite') {
-                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                    chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
-                }
-                else if (dbConnection.type === 'qdrant') {
-                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
-                    chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                }
-                embeddedCount++;
+            if (dbConnection.type === 'sqlite') {
+                database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedded[i], logger, chunkHash);
+                logger.debug(`Stored chunk ${chunkId} in SQLite`);
             }
-            else {
-                logger.error(`Embedding failed for chunk: ${chunkId}`);
-                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
+            else if (dbConnection.type === 'qdrant') {
+                await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedded[i], chunkHash);
+                logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
             }
         }
         chunkProgress.complete();
-        this.counters.chunks_added += embeddedCount;
-        return embeddedCount;
+        this.counters.chunks_added += chunks.length;
+        return chunks.length;
     }
     async createEmbeddings(texts) {
         const logger = this.logger.child('embeddings');
@@ -1852,6 +1864,12 @@ class Doc2Vec {
         }
         catch (error) {
             logger.error('Failed to create embeddings:', error);
+            // An auth rejection is permanent: every later request in this run
+            // fails the same way, so abort the source instead of returning []
+            // and letting each chunk degrade into a logged-and-skipped failure.
+            if (error?.status === 401 || error?.status === 403) {
+                throw new types_1.FatalEmbeddingError(`embedding request rejected with HTTP ${error.status}: ${error?.message ?? error}`);
+            }
             return [];
         }
     }

package/dist/types.js

@@ -1,6 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.FatalEmbeddingError = void 0;
 exports.newSourceRunCounters = newSourceRunCounters;
+// A permanent embedding failure (e.g. an invalid API key): every subsequent
+// request in the run would fail identically, so callers must abort the source
+// instead of degrading each chunk into a logged-and-skipped failure.
+class FatalEmbeddingError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'FatalEmbeddingError';
+        Object.setPrototypeOf(this, FatalEmbeddingError.prototype);
+    }
+}
+exports.FatalEmbeddingError = FatalEmbeddingError;
 function newSourceRunCounters(itemsKind) {
     return {
         items_kind: itemsKind,

package/package.json

@@ -1,7 +1,10 @@
 {
     "name": "doc2vec",
-    "version": "2.14.2",
+    "version": "2.15.0",
     "type": "commonjs",
+    "engines": {
+        "node": ">=22"
+    },
     "description": "",
     "main": "dist/doc2vec.js",
     "private": false,
@@ -33,23 +36,23 @@
         "@aws-sdk/client-s3": "^3.1004.0",
         "@chonkiejs/core": "^0.0.7",
         "@mozilla/readability": "^0.4.4",
-        "@qdrant/js-client-rest": "^1.13.0",
-        "@qdrant/qdrant-js": "^1.13.0",
-        "axios": "^1.6.2",
-        "better-sqlite3": "^11.9.1",
+        "@qdrant/js-client-rest": "^1.19.0",
+        "@qdrant/qdrant-js": "^1.19.0",
+        "axios": "^1.19.0",
+        "better-sqlite3": "^13.0.3",
         "chalk": "^5.4.1",
         "cheerio": "^1.0.0-rc.12",
         "commander": "^15.0.0",
         "croner": "^10.0.1",
         "dotenv": "^16.3.1",
         "express": "^5.2.1",
-        "js-yaml": "^4.1.0",
+        "js-yaml": "^4.3.1",
         "jsdom": "^26.0.0",
         "mammoth": "^1.11.0",
         "openai": "^4.20.1",
         "pdfjs-dist": "^5.3.31",
         "pg": "^8.18.0",
-        "puppeteer": "^24.1.1",
+        "puppeteer": "^25.8.0",
         "sanitize-html": "^2.11.0",
         "sqlite-vec": "0.1.7-alpha.2",
         "tree-sitter-wasms": "^0.1.0",