doc2vec 2.10.2 → 2.10.4

package/README.md

@@ -151,6 +151,16 @@ Configuration is managed through two files:
     OPENAI_API_KEY="sk-..."
     OPENAI_MODEL="text-embedding-3-large"  # Optional, defaults to text-embedding-3-large
 
+    # Optional: Override the OpenAI API base URL. Useful for pointing the
+    # OpenAI SDK at an OpenAI-compatible endpoint such as Ollama, LM Studio,
+    # llama.cpp, vLLM, or a proxy gateway. When unset, the SDK uses the
+    # default OpenAI endpoint. Equivalent to the `embedding.openai.base_url`
+    # field in config.yaml; the env var wins if both are set.
+    # Example values:
+    #   OPENAI_BASE_URL="http://localhost:11434/v1"        # Ollama
+    #   OPENAI_BASE_URL="https://gateway.example.com/v1"   # proxy / gateway
+    OPENAI_BASE_URL="http://localhost:11434/v1"
+
     # Optional: Embedding dimension size (defaults to 3072)
     EMBEDDING_DIMENSION="3072"
 
@@ -277,6 +287,7 @@ Configuration is managed through two files:
       openai:
         api_key: '${OPENAI_API_KEY}'  # Optional, uses env var by default
         model: 'text-embedding-3-large'  # Optional, defaults to text-embedding-3-large
+        # base_url: 'http://localhost:11434/v1'  # Optional, override OpenAI API base URL for Ollama / other OpenAI-compatible endpoints. Falls back to OPENAI_BASE_URL env var.
       # For Azure OpenAI, use this instead:
       # azure:
       #   api_key: '${AZURE_OPENAI_KEY}'

package/dist/content-processor.js

@@ -576,6 +576,11 @@ class ContentProcessor {
                             logger.debug(`HEAD returned ${headStatus} for ${url}, skipping`);
                             if (headStatus === 404) {
                                 notFoundUrls.add(url);
+                                // The URL was optimistically added to visitedUrls when
+                                // dequeued (before we knew it was dead). Remove it so the
+                                // obsolete-chunk cleanup treats it as gone and purges its
+                                // chunks — otherwise deleted pages linger forever.
+                                visitedUrls.delete(normalizedUrl);
                             }
                             skippedCount++;
                             continue;
@@ -717,6 +722,11 @@ class ContentProcessor {
                         consecutiveProtocolErrors = 0;
                     }
                     if (status === 404) {
+                        notFoundUrls.add(url);
+                        // The URL was optimistically added to visitedUrls when dequeued
+                        // (before this GET revealed it's a 404). Remove it so the
+                        // obsolete-chunk cleanup treats it as gone and purges its chunks.
+                        visitedUrls.delete(normalizedUrl);
                         const sources = referrers.get(url) ?? new Set([baseUrl]);
                         for (const source of sources) {
                             addBrokenLink(source, url);

package/dist/database.js

@@ -462,40 +462,51 @@ class DatabaseManager {
     static async removeObsoleteChunksQdrant(db, visitedUrls, urlPrefix, logger) {
         const { client, collectionName } = db;
         try {
-            // Get all points that match the URL prefix but are not metadata points
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: urlPrefix
-                            }
+            // Scroll through ALL points that match the URL prefix but are not
+            // metadata points. Qdrant caps a single scroll at its page limit, so
+            // we must paginate via next_page_offset — otherwise large collections
+            // (e.g. tens of thousands of chunks) would only have their first page
+            // examined and obsolete chunks beyond it would never be removed.
+            const filter = {
+                must: [
+                    {
+                        key: "url",
+                        match: {
+                            text: urlPrefix
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: "is_metadata",
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
-            const obsoletePointIds = response.points
-                .filter((point) => {
-                const url = point.payload?.url;
-                // Double check it's not a metadata record
-                if (point.payload?.is_metadata === true) {
-                    return false;
+                    }
+                ]
+            };
+            const obsoletePointIds = [];
+            let offset = undefined;
+            do {
+                const response = await client.scroll(collectionName, {
+                    limit: 1000,
+                    offset,
+                    with_payload: true,
+                    with_vector: false,
+                    filter
+                });
+                for (const point of response.points) {
+                    const url = point.payload?.url;
+                    // Double check it's not a metadata record
+                    if (point.payload?.is_metadata === true) {
+                        continue;
+                    }
+                    if (url && !visitedUrls.has(url)) {
+                        obsoletePointIds.push(point.id);
+                    }
                 }
-                return url && !visitedUrls.has(url);
-            })
-                .map((point) => point.id);
+                offset = response.next_page_offset;
+            } while (offset !== null && offset !== undefined);
             if (obsoletePointIds.length > 0) {
                 await client.delete(collectionName, {
                     points: obsoletePointIds,

package/dist/doc2vec.js

@@ -96,13 +96,17 @@ class Doc2Vec {
         else {
             const openaiApiKey = embeddingConfig.openai?.api_key || process.env.OPENAI_API_KEY;
             const openaiModel = embeddingConfig.openai?.model || process.env.OPENAI_MODEL || 'text-embedding-3-large';
+            const openaiBaseURL = embeddingConfig.openai?.base_url || process.env.OPENAI_BASE_URL;
             if (!openaiApiKey) {
                 this.logger.error('OpenAI requires api_key to be configured');
                 process.exit(1);
             }
-            this.openai = new openai_1.OpenAI({ apiKey: openaiApiKey });
+            this.openai = new openai_1.OpenAI({
+                apiKey: openaiApiKey,
+                ...(openaiBaseURL && { baseURL: openaiBaseURL }),
+            });
             this.embeddingModel = openaiModel;
-            this.logger.info(`Using OpenAI with model: ${openaiModel} (${this.embeddingDimension} dimensions)`);
+            this.logger.info(`Using OpenAI with model: ${openaiModel} (${this.embeddingDimension} dimensions)${openaiBaseURL ? ` via ${openaiBaseURL}` : ''}`);
         }
         this.contentProcessor = new content_processor_1.ContentProcessor(this.logger);
         // Initialize Postgres markdown store if configured
@@ -518,15 +522,33 @@ class Doc2Vec {
         this.writeBrokenLinksReport();
         logger.info(`Found ${validChunkIds.size} valid chunks across processed pages for ${config.url}`);
         logger.section('CLEANUP');
-        // Remove 404 URLs from the Postgres markdown store
-        if (useMarkdownStore && crawlResult.notFoundUrls && crawlResult.notFoundUrls.size > 0) {
-            logger.info(`Removing ${crawlResult.notFoundUrls.size} not-found URLs from markdown store`);
+        // Remove URLs that returned 404 during the crawl. A 404 is a definitive
+        // "this page is gone" signal (network errors fall through to full
+        // processing and never land here), so we purge these unconditionally —
+        // even when hasNetworkErrors below skips the broad obsolete cleanup.
+        if (crawlResult.notFoundUrls && crawlResult.notFoundUrls.size > 0) {
+            logger.info(`Removing ${crawlResult.notFoundUrls.size} not-found (404) URLs`);
             for (const url of crawlResult.notFoundUrls) {
+                // Postgres markdown store
+                if (useMarkdownStore) {
+                    try {
+                        await this.markdownStore.deleteMarkdown(url);
+                    }
+                    catch (pgError) {
+                        logger.error(`Failed to delete markdown from Postgres for ${url}:`, pgError);
+                    }
+                }
+                // Vector DB chunks
                 try {
-                    await this.markdownStore.deleteMarkdown(url);
+                    if (dbConnection.type === 'sqlite') {
+                        database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                    }
+                    else if (dbConnection.type === 'qdrant') {
+                        await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                    }
                 }
-                catch (pgError) {
-                    logger.error(`Failed to delete markdown from Postgres for ${url}:`, pgError);
+                catch (dbError) {
+                    logger.error(`Failed to delete chunks for 404 URL ${url}:`, dbError);
                 }
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.10.2",
+    "version": "2.10.4",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",