doc2vec 2.10.3 → 2.10.4

package/dist/content-processor.js

@@ -576,6 +576,11 @@ class ContentProcessor {
                             logger.debug(`HEAD returned ${headStatus} for ${url}, skipping`);
                             if (headStatus === 404) {
                                 notFoundUrls.add(url);
+                                // The URL was optimistically added to visitedUrls when
+                                // dequeued (before we knew it was dead). Remove it so the
+                                // obsolete-chunk cleanup treats it as gone and purges its
+                                // chunks — otherwise deleted pages linger forever.
+                                visitedUrls.delete(normalizedUrl);
                             }
                             skippedCount++;
                             continue;
@@ -717,6 +722,11 @@ class ContentProcessor {
                         consecutiveProtocolErrors = 0;
                     }
                     if (status === 404) {
+                        notFoundUrls.add(url);
+                        // The URL was optimistically added to visitedUrls when dequeued
+                        // (before this GET revealed it's a 404). Remove it so the
+                        // obsolete-chunk cleanup treats it as gone and purges its chunks.
+                        visitedUrls.delete(normalizedUrl);
                         const sources = referrers.get(url) ?? new Set([baseUrl]);
                         for (const source of sources) {
                             addBrokenLink(source, url);

package/dist/database.js

@@ -462,40 +462,51 @@ class DatabaseManager {
     static async removeObsoleteChunksQdrant(db, visitedUrls, urlPrefix, logger) {
         const { client, collectionName } = db;
         try {
-            // Get all points that match the URL prefix but are not metadata points
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: urlPrefix
-                            }
+            // Scroll through ALL points that match the URL prefix but are not
+            // metadata points. Qdrant caps a single scroll at its page limit, so
+            // we must paginate via next_page_offset — otherwise large collections
+            // (e.g. tens of thousands of chunks) would only have their first page
+            // examined and obsolete chunks beyond it would never be removed.
+            const filter = {
+                must: [
+                    {
+                        key: "url",
+                        match: {
+                            text: urlPrefix
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: "is_metadata",
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
-            const obsoletePointIds = response.points
-                .filter((point) => {
-                const url = point.payload?.url;
-                // Double check it's not a metadata record
-                if (point.payload?.is_metadata === true) {
-                    return false;
+                    }
+                ]
+            };
+            const obsoletePointIds = [];
+            let offset = undefined;
+            do {
+                const response = await client.scroll(collectionName, {
+                    limit: 1000,
+                    offset,
+                    with_payload: true,
+                    with_vector: false,
+                    filter
+                });
+                for (const point of response.points) {
+                    const url = point.payload?.url;
+                    // Double check it's not a metadata record
+                    if (point.payload?.is_metadata === true) {
+                        continue;
+                    }
+                    if (url && !visitedUrls.has(url)) {
+                        obsoletePointIds.push(point.id);
+                    }
                 }
-                return url && !visitedUrls.has(url);
-            })
-                .map((point) => point.id);
+                offset = response.next_page_offset;
+            } while (offset !== null && offset !== undefined);
             if (obsoletePointIds.length > 0) {
                 await client.delete(collectionName, {
                     points: obsoletePointIds,

package/dist/doc2vec.js

@@ -522,15 +522,33 @@ class Doc2Vec {
         this.writeBrokenLinksReport();
         logger.info(`Found ${validChunkIds.size} valid chunks across processed pages for ${config.url}`);
         logger.section('CLEANUP');
-        // Remove 404 URLs from the Postgres markdown store
-        if (useMarkdownStore && crawlResult.notFoundUrls && crawlResult.notFoundUrls.size > 0) {
-            logger.info(`Removing ${crawlResult.notFoundUrls.size} not-found URLs from markdown store`);
+        // Remove URLs that returned 404 during the crawl. A 404 is a definitive
+        // "this page is gone" signal (network errors fall through to full
+        // processing and never land here), so we purge these unconditionally —
+        // even when hasNetworkErrors below skips the broad obsolete cleanup.
+        if (crawlResult.notFoundUrls && crawlResult.notFoundUrls.size > 0) {
+            logger.info(`Removing ${crawlResult.notFoundUrls.size} not-found (404) URLs`);
             for (const url of crawlResult.notFoundUrls) {
+                // Postgres markdown store
+                if (useMarkdownStore) {
+                    try {
+                        await this.markdownStore.deleteMarkdown(url);
+                    }
+                    catch (pgError) {
+                        logger.error(`Failed to delete markdown from Postgres for ${url}:`, pgError);
+                    }
+                }
+                // Vector DB chunks
                 try {
-                    await this.markdownStore.deleteMarkdown(url);
+                    if (dbConnection.type === 'sqlite') {
+                        database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                    }
+                    else if (dbConnection.type === 'qdrant') {
+                        await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                    }
                 }
-                catch (pgError) {
-                    logger.error(`Failed to delete markdown from Postgres for ${url}:`, pgError);
+                catch (dbError) {
+                    logger.error(`Failed to delete chunks for 404 URL ${url}:`, dbError);
                 }
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.10.3",
+    "version": "2.10.4",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",