doc2vec 2.10.3 → 2.10.5

package/dist/content-processor.js

@@ -576,6 +576,11 @@ class ContentProcessor {
                             logger.debug(`HEAD returned ${headStatus} for ${url}, skipping`);
                             if (headStatus === 404) {
                                 notFoundUrls.add(url);
+                                // The URL was optimistically added to visitedUrls when
+                                // dequeued (before we knew it was dead). Remove it so the
+                                // obsolete-chunk cleanup treats it as gone and purges its
+                                // chunks — otherwise deleted pages linger forever.
+                                visitedUrls.delete(normalizedUrl);
                             }
                             skippedCount++;
                             continue;
@@ -717,6 +722,11 @@ class ContentProcessor {
                         consecutiveProtocolErrors = 0;
                     }
                     if (status === 404) {
+                        notFoundUrls.add(url);
+                        // The URL was optimistically added to visitedUrls when dequeued
+                        // (before this GET revealed it's a 404). Remove it so the
+                        // obsolete-chunk cleanup treats it as gone and purges its chunks.
+                        visitedUrls.delete(normalizedUrl);
                         const sources = referrers.get(url) ?? new Set([baseUrl]);
                         for (const source of sources) {
                             addBrokenLink(source, url);

package/dist/database.js

@@ -215,7 +215,13 @@ class DatabaseManager {
             }
         }
         catch (error) {
-            logger.error(`Failed to update metadata value for ${key}:`, error);
+            // Best-effort by design: a failed metadata write errs toward the
+            // safe direction (redundant reprocessing next run — etag/lastmod/
+            // watermark stay behind, never skip ahead), so we warn rather than
+            // fail the job. Genuine disk/connection failures surface earlier at
+            // the chunk-write stage (storeChunkInQdrant), which shares this
+            // collection and throws.
+            logger.warn(`Failed to update metadata value for ${key} (continuing; will retry next run):`, error);
         }
     }
     static async getLastRunDate(dbConnection, repo, defaultDate, logger) {
@@ -436,7 +442,11 @@ class DatabaseManager {
             });
         }
         catch (error) {
-            console.error("Error storing chunk in Qdrant:", error);
+            // Rethrow so the failure propagates to the top-level handler and the
+            // job exits non-zero. Swallowing here previously let a sync "succeed"
+            // while chunks silently failed to store (e.g. Qdrant disk full).
+            console.error(`Error storing chunk in Qdrant (collection ${collectionName}, url ${chunk.metadata.url}):`, error);
+            throw error;
         }
     }
     static removeObsoleteChunksSQLite(db, visitedUrls, urlPrefix, logger) {
@@ -462,40 +472,51 @@ class DatabaseManager {
     static async removeObsoleteChunksQdrant(db, visitedUrls, urlPrefix, logger) {
         const { client, collectionName } = db;
         try {
-            // Get all points that match the URL prefix but are not metadata points
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: urlPrefix
-                            }
+            // Scroll through ALL points that match the URL prefix but are not
+            // metadata points. Qdrant caps a single scroll at its page limit, so
+            // we must paginate via next_page_offset — otherwise large collections
+            // (e.g. tens of thousands of chunks) would only have their first page
+            // examined and obsolete chunks beyond it would never be removed.
+            const filter = {
+                must: [
+                    {
+                        key: "url",
+                        match: {
+                            text: urlPrefix
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: "is_metadata",
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
-            const obsoletePointIds = response.points
-                .filter((point) => {
-                const url = point.payload?.url;
-                // Double check it's not a metadata record
-                if (point.payload?.is_metadata === true) {
-                    return false;
+                    }
+                ]
+            };
+            const obsoletePointIds = [];
+            let offset = undefined;
+            do {
+                const response = await client.scroll(collectionName, {
+                    limit: 1000,
+                    offset,
+                    with_payload: true,
+                    with_vector: false,
+                    filter
+                });
+                for (const point of response.points) {
+                    const url = point.payload?.url;
+                    // Double check it's not a metadata record
+                    if (point.payload?.is_metadata === true) {
+                        continue;
+                    }
+                    if (url && !visitedUrls.has(url)) {
+                        obsoletePointIds.push(point.id);
+                    }
                 }
-                return url && !visitedUrls.has(url);
-            })
-                .map((point) => point.id);
+                offset = response.next_page_offset;
+            } while (offset !== null && offset !== undefined);
             if (obsoletePointIds.length > 0) {
                 await client.delete(collectionName, {
                     points: obsoletePointIds,
@@ -507,7 +528,11 @@ class DatabaseManager {
             }
         }
         catch (error) {
+            // Rethrow so a failed cleanup fails the job instead of silently
+            // leaving orphans behind. This is idempotent and recomputed each
+            // run, so a rerun retries it cleanly.
             logger.error(`Error removing obsolete chunks from Qdrant:`, error);
+            throw error;
         }
     }
     static hasColumn(db, columnName) {
@@ -550,7 +575,12 @@ class DatabaseManager {
             logger.info(`Deleted chunks from Qdrant for URL ${url}`);
         }
         catch (error) {
+            // Rethrow so the failure propagates and the job exits non-zero.
+            // A swallowed delete leaves stale chunks behind while the sync
+            // reports success. Callers that treat deletion as best-effort
+            // (e.g. 404 cleanup) wrap this call in their own try/catch.
             logger.error(`Error deleting chunks from Qdrant for URL ${url}:`, error);
+            throw error;
         }
     }
     /**
@@ -715,53 +745,64 @@ class DatabaseManager {
                 urlPrefix = `file://${cleanedDirPrefix}`;
             }
             logger.debug(`Checking for obsolete chunks with URL prefix: ${urlPrefix}`);
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: urlPrefix
-                            }
+            // Scroll through ALL matching points via next_page_offset. A single
+            // capped scroll would only examine the first page, so obsolete
+            // chunks beyond it (in collections larger than the page limit) would
+            // never be removed.
+            const filter = {
+                must: [
+                    {
+                        key: "url",
+                        match: {
+                            text: urlPrefix
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: "is_metadata",
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
-            const obsoletePointIds = response.points
-                .filter((point) => {
-                const url = point.payload?.url;
-                // Double check it's not a metadata record
-                if (point.payload?.is_metadata === true) {
-                    return false;
-                }
-                if (!url || !url.startsWith(urlPrefix)) {
-                    return false;
-                }
-                let filePath;
-                if (isRewriteMode) {
-                    // URL rewrite mode: extract relative path and construct full file path
-                    const config = pathConfig;
-                    const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
-                    filePath = path.join(config.path, relativePath);
-                }
-                else {
-                    // Direct file path mode: remove file:// prefix to match with processedFiles
-                    filePath = url.startsWith('file://') ? url.substring(7) : '';
+                    }
+                ]
+            };
+            const obsoletePointIds = [];
+            let offset = undefined;
+            do {
+                const response = await client.scroll(collectionName, {
+                    limit: 1000,
+                    offset,
+                    with_payload: true,
+                    with_vector: false,
+                    filter
+                });
+                for (const point of response.points) {
+                    const url = point.payload?.url;
+                    // Double check it's not a metadata record
+                    if (point.payload?.is_metadata === true) {
+                        continue;
+                    }
+                    if (!url || !url.startsWith(urlPrefix)) {
+                        continue;
+                    }
+                    let filePath;
+                    if (isRewriteMode) {
+                        // URL rewrite mode: extract relative path and construct full file path
+                        const config = pathConfig;
+                        const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
+                        filePath = path.join(config.path, relativePath);
+                    }
+                    else {
+                        // Direct file path mode: remove file:// prefix to match with processedFiles
+                        filePath = url.startsWith('file://') ? url.substring(7) : '';
+                    }
+                    if (filePath && !processedFiles.has(filePath)) {
+                        obsoletePointIds.push(point.id);
+                    }
                 }
-                return filePath && !processedFiles.has(filePath);
-            })
-                .map((point) => point.id);
+                offset = response.next_page_offset;
+            } while (offset !== null && offset !== undefined);
             if (obsoletePointIds.length > 0) {
                 await client.delete(collectionName, {
                     points: obsoletePointIds,
@@ -773,7 +814,10 @@ class DatabaseManager {
             }
         }
         catch (error) {
+            // Rethrow so a failed cleanup fails the job instead of silently
+            // leaving orphans behind (idempotent — retried cleanly next run).
             logger.error(`Error removing obsolete chunks from Qdrant:`, error);
+            throw error;
         }
     }
 }

package/dist/doc2vec.js

@@ -522,15 +522,33 @@ class Doc2Vec {
         this.writeBrokenLinksReport();
         logger.info(`Found ${validChunkIds.size} valid chunks across processed pages for ${config.url}`);
         logger.section('CLEANUP');
-        // Remove 404 URLs from the Postgres markdown store
-        if (useMarkdownStore && crawlResult.notFoundUrls && crawlResult.notFoundUrls.size > 0) {
-            logger.info(`Removing ${crawlResult.notFoundUrls.size} not-found URLs from markdown store`);
+        // Remove URLs that returned 404 during the crawl. A 404 is a definitive
+        // "this page is gone" signal (network errors fall through to full
+        // processing and never land here), so we purge these unconditionally —
+        // even when hasNetworkErrors below skips the broad obsolete cleanup.
+        if (crawlResult.notFoundUrls && crawlResult.notFoundUrls.size > 0) {
+            logger.info(`Removing ${crawlResult.notFoundUrls.size} not-found (404) URLs`);
             for (const url of crawlResult.notFoundUrls) {
+                // Postgres markdown store
+                if (useMarkdownStore) {
+                    try {
+                        await this.markdownStore.deleteMarkdown(url);
+                    }
+                    catch (pgError) {
+                        logger.error(`Failed to delete markdown from Postgres for ${url}:`, pgError);
+                    }
+                }
+                // Vector DB chunks
                 try {
-                    await this.markdownStore.deleteMarkdown(url);
+                    if (dbConnection.type === 'sqlite') {
+                        database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                    }
+                    else if (dbConnection.type === 'qdrant') {
+                        await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                    }
                 }
-                catch (pgError) {
-                    logger.error(`Failed to delete markdown from Postgres for ${url}:`, pgError);
+                catch (dbError) {
+                    logger.error(`Failed to delete chunks for 404 URL ${url}:`, dbError);
                 }
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.10.3",
+    "version": "2.10.5",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",