doc2vec 2.2.0 → 2.3.0

package/dist/content-processor.js

@@ -247,83 +247,121 @@ class ContentProcessor {
         let pdfProcessedCount = 0;
         let errorCount = 0;
         let hasNetworkErrors = false;
-        while (queue.length > 0) {
-            const url = queue.shift();
-            if (!url)
-                continue;
-            const normalizedUrl = utils_1.Utils.normalizeUrl(url);
-            if (visitedUrls.has(normalizedUrl))
-                continue;
-            visitedUrls.add(normalizedUrl);
-            if (!utils_1.Utils.shouldProcessUrl(url)) {
-                logger.debug(`Skipping URL with unsupported extension: ${url}`);
-                skippedCount++;
-                continue;
+        // Launch a single browser instance and reuse one page (tab) for all URLs
+        let browser = null;
+        let page = null;
+        const launchBrowser = async () => {
+            let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+            if (!executablePath) {
+                if (fs.existsSync('/usr/bin/chromium')) {
+                    executablePath = '/usr/bin/chromium';
+                }
+                else if (fs.existsSync('/usr/bin/chromium-browser')) {
+                    executablePath = '/usr/bin/chromium-browser';
+                }
             }
-            try {
-                logger.info(`Crawling: ${url}`);
-                const sources = referrers.get(url) ?? new Set([baseUrl]);
-                const content = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
-                    if (status === 404) {
-                        for (const source of sources) {
-                            addBrokenLink(source, reportedUrl);
+            const b = await puppeteer_1.default.launch({
+                executablePath,
+                args: ['--no-sandbox', '--disable-setuid-sandbox'],
+            });
+            const p = await b.newPage();
+            return { browser: b, page: p };
+        };
+        const ensureBrowser = async () => {
+            if (!browser || !browser.isConnected()) {
+                logger.info(browser ? 'Browser disconnected, relaunching...' : 'Launching browser...');
+                const launched = await launchBrowser();
+                browser = launched.browser;
+                page = launched.page;
+            }
+            return page;
+        };
+        try {
+            while (queue.length > 0) {
+                const url = queue.shift();
+                if (!url)
+                    continue;
+                const normalizedUrl = utils_1.Utils.normalizeUrl(url);
+                if (visitedUrls.has(normalizedUrl))
+                    continue;
+                visitedUrls.add(normalizedUrl);
+                if (!utils_1.Utils.shouldProcessUrl(url)) {
+                    logger.debug(`Skipping URL with unsupported extension: ${url}`);
+                    skippedCount++;
+                    continue;
+                }
+                try {
+                    logger.info(`Crawling: ${url}`);
+                    const sources = referrers.get(url) ?? new Set([baseUrl]);
+                    // For HTML pages, ensure the browser is running and pass the shared page
+                    // For PDFs, processPage handles them without Puppeteer
+                    const currentPage = utils_1.Utils.isPdfUrl(url) ? undefined : await ensureBrowser();
+                    const result = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
+                        if (status === 404) {
+                            for (const source of sources) {
+                                addBrokenLink(source, reportedUrl);
+                            }
+                        }
+                    }, currentPage);
+                    if (result.content !== null) {
+                        await processPageContent(url, result.content);
+                        if (utils_1.Utils.isPdfUrl(url)) {
+                            pdfProcessedCount++;
+                        }
+                        else {
+                            processedCount++;
                         }
-                    }
-                });
-                if (content !== null) {
-                    await processPageContent(url, content);
-                    if (utils_1.Utils.isPdfUrl(url)) {
-                        pdfProcessedCount++;
                     }
                     else {
-                        processedCount++;
+                        skippedSizeCount++;
                     }
-                }
-                else {
-                    skippedSizeCount++;
-                }
-                // Only try to extract links from HTML pages, not PDFs
-                if (!utils_1.Utils.isPdfUrl(url)) {
-                    const response = await axios_1.default.get(url);
-                    const $ = (0, cheerio_1.load)(response.data);
-                    const pageUrlForLinks = response?.request?.res?.responseUrl || url;
-                    logger.debug(`Finding links on page ${url}`);
-                    let newLinksFound = 0;
-                    $('a[href]').each((_, element) => {
-                        const href = $(element).attr('href');
-                        if (!href || href.startsWith('#') || href.startsWith('mailto:'))
-                            return;
-                        const fullUrl = utils_1.Utils.buildUrl(href, pageUrlForLinks);
-                        if (fullUrl.startsWith(sourceConfig.url)) {
-                            addReferrer(fullUrl, pageUrlForLinks);
-                            if (!visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
-                                if (!queue.includes(fullUrl)) {
-                                    queue.push(fullUrl);
-                                    newLinksFound++;
+                    // Use links extracted from the full rendered DOM by processPage
+                    // (no separate axios request needed)
+                    if (result.links.length > 0) {
+                        const pageUrlForLinks = result.finalUrl || url;
+                        logger.debug(`Finding links on page ${url}`);
+                        let newLinksFound = 0;
+                        for (const href of result.links) {
+                            const fullUrl = utils_1.Utils.buildUrl(href, pageUrlForLinks);
+                            if (fullUrl.startsWith(sourceConfig.url)) {
+                                addReferrer(fullUrl, pageUrlForLinks);
+                                if (!visitedUrls.has(utils_1.Utils.normalizeUrl(fullUrl))) {
+                                    if (!queue.includes(fullUrl)) {
+                                        queue.push(fullUrl);
+                                        newLinksFound++;
+                                    }
                                 }
                             }
                         }
-                    });
-                    logger.debug(`Found ${newLinksFound} new links on ${url}`);
-                }
-            }
-            catch (error) {
-                logger.error(`Failed during processing or link discovery for ${url}:`, error);
-                errorCount++;
-                const status = this.getHttpStatus(error);
-                if (status === 404) {
-                    const sources = referrers.get(url) ?? new Set([baseUrl]);
-                    for (const source of sources) {
-                        addBrokenLink(source, url);
+                        logger.debug(`Found ${newLinksFound} new links on ${url}`);
                     }
                 }
-                // Check if this is a network error (DNS resolution, connection issues, etc.)
-                if (this.isNetworkError(error)) {
-                    hasNetworkErrors = true;
-                    logger.warn(`Network error detected for ${url}, this may affect cleanup decisions`);
+                catch (error) {
+                    logger.error(`Failed during processing or link discovery for ${url}:`, error);
+                    errorCount++;
+                    const status = this.getHttpStatus(error);
+                    if (status === 404) {
+                        const sources = referrers.get(url) ?? new Set([baseUrl]);
+                        for (const source of sources) {
+                            addBrokenLink(source, url);
+                        }
+                    }
+                    // Check if this is a network error (DNS resolution, connection issues, etc.)
+                    if (this.isNetworkError(error)) {
+                        hasNetworkErrors = true;
+                        logger.warn(`Network error detected for ${url}, this may affect cleanup decisions`);
+                    }
                 }
             }
         }
+        finally {
+            // Close the shared browser instance when the crawl is done
+            const browserToClose = browser;
+            if (browserToClose && browserToClose.isConnected()) {
+                await browserToClose.close();
+                logger.debug('Shared browser closed after crawl completed');
+            }
+        }
         logger.info(`Crawl completed. HTML Pages: ${processedCount}, PDFs: ${pdfProcessedCount}, Skipped (Extension): ${skippedCount}, Skipped (Size): ${skippedSizeCount}, Errors: ${errorCount}`);
         if (hasNetworkErrors) {
             logger.warn('Network errors were encountered during crawling. Cleanup may be skipped to avoid removing valid chunks.');
@@ -369,7 +407,7 @@ class ContentProcessor {
         }
         return false;
     }
-    async processPage(url, sourceConfig, onHttpStatus) {
+    async processPage(url, sourceConfig, onHttpStatus, existingPage) {
         const logger = this.logger.child('page-processor');
         logger.debug(`Processing content from ${url}`);
         // Check if this is a PDF URL
@@ -380,9 +418,9 @@ class ContentProcessor {
                 // Check size limit for PDF content
                 if (markdown.length > sourceConfig.max_size) {
                     logger.warn(`PDF content (${markdown.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping ${url}.`);
-                    return null;
+                    return { content: null, links: [], finalUrl: url };
                 }
-                return markdown;
+                return { content: markdown, links: [], finalUrl: url };
             }
             catch (error) {
                 const status = this.getHttpStatus(error);
@@ -393,27 +431,35 @@ class ContentProcessor {
                     throw error;
                 }
                 logger.error(`Failed to process PDF ${url}:`, error);
-                return null;
+                return { content: null, links: [], finalUrl: url };
             }
         }
-        // Original HTML page processing logic
+        // HTML page processing logic
+        // If an existing page (tab) is provided, reuse it; otherwise launch a standalone browser
         let browser = null;
+        let page;
+        const ownsTheBrowser = !existingPage;
         try {
-            // Use system Chromium if available (for Docker environments)
-            let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
-            if (!executablePath) {
-                if (fs.existsSync('/usr/bin/chromium')) {
-                    executablePath = '/usr/bin/chromium';
-                }
-                else if (fs.existsSync('/usr/bin/chromium-browser')) {
-                    executablePath = '/usr/bin/chromium-browser';
+            if (existingPage) {
+                page = existingPage;
+            }
+            else {
+                // Standalone mode: launch a browser for this single page
+                let executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+                if (!executablePath) {
+                    if (fs.existsSync('/usr/bin/chromium')) {
+                        executablePath = '/usr/bin/chromium';
+                    }
+                    else if (fs.existsSync('/usr/bin/chromium-browser')) {
+                        executablePath = '/usr/bin/chromium-browser';
+                    }
                 }
+                browser = await puppeteer_1.default.launch({
+                    executablePath,
+                    args: ['--no-sandbox', '--disable-setuid-sandbox'],
+                });
+                page = await browser.newPage();
             }
-            browser = await puppeteer_1.default.launch({
-                executablePath,
-                args: ['--no-sandbox', '--disable-setuid-sandbox'],
-            });
-            const page = await browser.newPage();
             logger.debug(`Navigating to ${url}`);
             const response = await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
             const status = response?.status();
@@ -422,8 +468,24 @@ class ContentProcessor {
                 error.status = status;
                 throw error;
             }
+            // Get the final URL after any redirects
+            const finalUrl = page.url();
+            // Extract ALL links from the full rendered DOM before any content filtering
+            // This searches the entire document, not just the main content area
+            const links = await page.evaluate(() => {
+                const anchors = document.querySelectorAll('a[href]');
+                const hrefs = [];
+                anchors.forEach(a => {
+                    const href = a.getAttribute('href');
+                    if (href && !href.startsWith('#') && !href.startsWith('mailto:')) {
+                        hrefs.push(href);
+                    }
+                });
+                return hrefs;
+            });
+            logger.debug(`Extracted ${links.length} links from full DOM of ${url}`);
             const htmlContent = await page.evaluate(() => {
-                // 💡 Try specific content selectors first, then fall back to broader ones
+                // Try specific content selectors first, then fall back to broader ones
                 const mainContentElement = document.querySelector('.docs-content') || // Common docs pattern
                     document.querySelector('.doc-content') || // Alternative docs pattern
                     document.querySelector('.markdown-body') || // GitHub-style
@@ -435,8 +497,7 @@ class ContentProcessor {
             });
             if (htmlContent.length > sourceConfig.max_size) {
                 logger.warn(`Raw HTML content (${htmlContent.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping detailed processing for ${url}.`);
-                await browser.close();
-                return null;
+                return { content: null, links, finalUrl };
             }
             logger.debug(`Got HTML content (${htmlContent.length} chars), creating DOM`);
             const dom = new jsdom_1.JSDOM(htmlContent);
@@ -446,7 +507,7 @@ class ContentProcessor {
                 pre.setAttribute('data-readable-content-score', '100');
                 this.markCodeParents(pre.parentElement);
             });
-            // 💡 Extract H1s BEFORE Readability - it often strips them as "chrome"
+            // Extract H1s BEFORE Readability - it often strips them as "chrome"
             // We'll inject them back after Readability processing
             const h1Elements = document.querySelectorAll('h1');
             const extractedH1s = [];
@@ -468,8 +529,7 @@ class ContentProcessor {
             const article = reader.parse();
             if (!article) {
                 logger.warn(`Failed to parse article content with Readability for ${url}`);
-                await browser.close();
-                return null;
+                return { content: null, links, finalUrl };
             }
             // Debug: Log what Readability extracted
             logger.debug(`[Readability Debug] article.title: "${article.title}"`);
@@ -478,7 +538,7 @@ class ContentProcessor {
             logger.debug(`[Readability Debug] Contains H1 tag: ${article.content?.includes('<h1')}`);
             logger.debug(`[Readability Debug] Contains H2 tag: ${article.content?.includes('<h2')}`);
             logger.debug(`[Readability Debug] Contains original-h1 class: ${article.content?.includes('original-h1')}`);
-            // 💡 Restore H1s: find elements with our marker class and convert back from H2
+            // Restore H1s: find elements with our marker class and convert back from H2
             const articleDom = new jsdom_1.JSDOM(article.content);
             const articleDoc = articleDom.window.document;
             const originalH1Elements = articleDoc.querySelectorAll('.original-h1');
@@ -515,7 +575,7 @@ class ContentProcessor {
             });
             logger.debug(`Converting HTML to Markdown`);
             let markdown = this.turndownService.turndown(cleanHtml);
-            // 💡 Inject extracted H1s back if they're not in the markdown
+            // Inject extracted H1s back if they're not in the markdown
             // Readability often strips them as "page chrome"
             // Use article.title as fallback if no H1 was extracted
             const pageTitle = extractedH1s.length > 0 ? extractedH1s[0] : (article.title?.trim() || '');
@@ -535,7 +595,7 @@ class ContentProcessor {
                 }
             }
             logger.debug(`Markdown conversion complete (${markdown.length} chars)`);
-            return markdown;
+            return { content: markdown, links, finalUrl };
         }
         catch (error) {
             const status = this.getHttpStatus(error);
@@ -546,10 +606,11 @@ class ContentProcessor {
                 throw error;
             }
             logger.error(`Error processing page ${url}:`, error);
-            return null;
+            return { content: null, links: [], finalUrl: url };
         }
         finally {
-            if (browser && browser.isConnected()) {
+            // Only close the browser if we launched it ourselves (standalone mode)
+            if (ownsTheBrowser && browser && browser.isConnected()) {
                 await browser.close();
                 logger.debug(`Browser closed for ${url}`);
             }

package/dist/database.js

@@ -374,10 +374,10 @@ class DatabaseManager {
                     chunk.metadata.version
                 ];
                 if (hasBranchColumn) {
-                    insertValues.push(chunk.metadata.branch ?? null);
+                    insertValues.push(chunk.metadata.branch ?? '');
                 }
                 if (hasRepoColumn) {
-                    insertValues.push(chunk.metadata.repo ?? null);
+                    insertValues.push(chunk.metadata.repo ?? '');
                 }
                 insertValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.metadata.chunk_id, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks);
                 insertStmt.run(...insertValues);
@@ -389,10 +389,10 @@ class DatabaseManager {
                     chunk.metadata.version
                 ];
                 if (hasBranchColumn) {
-                    updateValues.push(chunk.metadata.branch ?? null);
+                    updateValues.push(chunk.metadata.branch ?? '');
                 }
                 if (hasRepoColumn) {
-                    updateValues.push(chunk.metadata.repo ?? null);
+                    updateValues.push(chunk.metadata.repo ?? '');
                 }
                 updateValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks, chunk.metadata.chunk_id);
                 updateStmt.run(...updateValues);

package/dist/doc2vec.js

@@ -402,7 +402,7 @@ class Doc2Vec {
                 const chunks = await this.contentProcessor.chunkMarkdown(content, config, url);
                 logger.info(`Created ${chunks.length} chunks`);
                 if (chunks.length > 0) {
-                    const chunkProgress = logger.progress(`Embedding chunks for ${url}`, chunks.length);
+                    const chunkProgress = logger.progress(`Processing chunks for ${url}`, chunks.length);
                     for (let i = 0; i < chunks.length; i++) {
                         const chunk = chunks[i];
                         validChunkIds.add(chunk.metadata.chunk_id);
@@ -561,7 +561,7 @@ class Doc2Vec {
                 const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
                 logger.info(`Created ${chunks.length} chunks`);
                 if (chunks.length > 0) {
-                    const chunkProgress = logger.progress(`Embedding chunks for ${filePath}`, chunks.length);
+                    const chunkProgress = logger.progress(`Processing chunks for ${filePath}`, chunks.length);
                     for (let i = 0; i < chunks.length; i++) {
                         const chunk = chunks[i];
                         validChunkIds.add(chunk.metadata.chunk_id);
@@ -729,7 +729,7 @@ class Doc2Vec {
                     const chunks = await this.contentProcessor.chunkCode(content, config, fileUrl, relativePath || filePath, repoBranch || config.branch, config.repo);
                     logger.info(`Created ${chunks.length} chunks`);
                     if (chunks.length > 0) {
-                        const chunkProgress = logger.progress(`Embedding chunks for ${relativePath || filePath}`, chunks.length);
+                        const chunkProgress = logger.progress(`Processing chunks for ${relativePath || filePath}`, chunks.length);
                         for (let i = 0; i < chunks.length; i++) {
                             const chunk = chunks[i];
                             validChunkIds.add(chunk.metadata.chunk_id);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.2.0",
+    "version": "2.3.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",