doc2vec 2.17.3 → 2.17.4

package/README.md

@@ -257,7 +257,7 @@ Configuration is managed through two files:
         Common configuration for all types:
         *   `product_name`: A string identifying the product (used in metadata).
         *   `version`: A string identifying the product version (used in metadata).
-        *   `max_size`: Maximum raw content size (in characters). For websites, this limits the raw HTML fetched by Puppeteer. Recommending 1MB (1048576).
+        *   `max_size`: Maximum raw content size (in characters). For websites, this limits the raw HTML fetched by Puppeteer. Recommending 1MB (1048576). Pages whose uncompressed HTML is more than 4× `max_size` (checked with a HEAD request) are skipped without being rendered, so a multi-MB page can't exhaust the browser's memory; their links are still followed.
         *   `database_config`: Configuration for the database.
             *   `type`: Specifies the storage backend (`'sqlite'` or `'qdrant'`).
             *   `params`: Parameters specific to the chosen database type.
@@ -548,6 +548,8 @@ Notes:
 - A crawl holds a browser for as long as it runs, so disable or raise the browser service's session timeout (browserless: `TIMEOUT=0`).
 - Each connection gets its own browser session; doc2vec closes it when the crawl finishes and every 50 pages.
 - Launch flags such as `--no-sandbox` belong to the process that starts the browser and are not sent to a remote one.
+- If the browser service is unreachable (e.g. its pod was OOMKilled and is restarting), doc2vec retries the connection after 5s, 15s, 30s and 60s before failing the source, and reports the actual connection error (`Unexpected server response: 429`, `ECONNREFUSED`, …).
+- Size the browser service's memory for the largest pages you crawl. doc2vec skips pages over 4× `max_size` before rendering, but pages just under that can still be heavy. A memory-backed `/dev/shm` (`emptyDir: { medium: Memory }`) counts against the same limit.
 - With `BROWSER_WS_ENDPOINT` unset, doc2vec launches a local Chrome: `PUPPETEER_EXECUTABLE_PATH` if set, else Puppeteer's downloaded Chrome for Testing, else a system `chromium`.
 
 ## Controller Mode

package/dist/content-processor.js

@@ -36,9 +36,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContentProcessor = exports.BROWSER_WS_ENDPOINT_ENV = exports.BrowserLaunchError = void 0;
+exports.ContentProcessor = exports.OVERSIZED_PAGE_FACTOR = exports.BROWSER_WS_ENDPOINT_ENV = exports.BrowserLaunchError = void 0;
 exports.remoteBrowserEndpoint = remoteBrowserEndpoint;
 exports.openBrowser = openBrowser;
+exports.describeError = describeError;
+exports.browserLaunchRetryDelaysMs = browserLaunchRetryDelaysMs;
 exports.resolveBrowserExecutablePath = resolveBrowserExecutablePath;
 const readability_1 = require("@mozilla/readability");
 const axios_1 = __importDefault(require("axios"));
@@ -94,6 +96,14 @@ const LOCAL_LAUNCH_ARGS = [
     '--disable-extensions',
 ];
 const BROWSER_PROTOCOL_TIMEOUT_MS = 60000;
+/**
+ * Pages whose uncompressed HTML exceeds this multiple of a source's max_size are
+ * skipped before rendering. The max_size check after rendering measures only
+ * the main content element, so the whole page is always somewhat larger. The
+ * margin keeps pages with a lot of chrome around valid content from being lost.
+ * The pages that OOM-killed browserless were 5.6–12.7 MB, against a 1 MB max_size.
+ */
+exports.OVERSIZED_PAGE_FACTOR = 4;
 /**
  * Open a browser: connect to the remote one named by BROWSER_WS_ENDPOINT, or
  * launch a local one. Callers close it with browser.close() in both cases —
@@ -103,10 +113,19 @@ const BROWSER_PROTOCOL_TIMEOUT_MS = 60000;
 async function openBrowser() {
     const endpoint = remoteBrowserEndpoint();
     if (endpoint) {
-        return puppeteer_1.default.connect({
-            browserWSEndpoint: endpoint,
-            protocolTimeout: BROWSER_PROTOCOL_TIMEOUT_MS,
-        });
+        try {
+            return await puppeteer_1.default.connect({
+                browserWSEndpoint: endpoint,
+                protocolTimeout: BROWSER_PROTOCOL_TIMEOUT_MS,
+            });
+        }
+        catch (error) {
+            // puppeteer rejects a failed WebSocket handshake with the raw `ws`
+            // ErrorEvent (not an Error), which stringifies to "[object Object]".
+            // Re-throw a real Error naming the browser host (never the full
+            // URL: it carries the token).
+            throw new Error(`cannot connect to remote browser at ${describeEndpoint(endpoint)}: ${describeError(error)}`);
+        }
     }
     return puppeteer_1.default.launch({
         executablePath: await resolveBrowserExecutablePath(),
@@ -114,6 +133,47 @@ async function openBrowser() {
         protocolTimeout: BROWSER_PROTOCOL_TIMEOUT_MS,
     });
 }
+/** `ws://host:port` of a CDP endpoint, without path, query (token) or credentials. */
+function describeEndpoint(endpoint) {
+    try {
+        const url = new URL(endpoint);
+        return `${url.protocol}//${url.host}`;
+    }
+    catch {
+        return '<unparseable endpoint>';
+    }
+}
+/**
+ * Human-readable message for anything thrown or rejected. Handles Error, the
+ * `ws` ErrorEvent shape ({ message, error }) and other plain objects, which
+ * String() would render as "[object Object]".
+ */
+function describeError(error) {
+    if (error instanceof Error)
+        return error.message || error.name;
+    if (error && typeof error === 'object') {
+        const obj = error;
+        if (obj.error instanceof Error && obj.error.message)
+            return obj.error.message;
+        if (typeof obj.message === 'string' && obj.message)
+            return obj.message;
+        if (typeof obj.type === 'string' && obj.type)
+            return `${obj.type} event`;
+        // Deliberately not JSON-serialized: event objects reference the socket,
+        // whose URL may carry a token.
+        return `non-Error rejection (${error.constructor?.name ?? 'Object'})`;
+    }
+    return String(error);
+}
+/**
+ * Pauses between browser launch/connect attempts. A remote browser service
+ * (browserless) gets a longer window: when its pod is OOMKilled and restarts,
+ * it refuses connections for ~30s, and a single 5s retry turned that into
+ * several consecutive sources failing outright.
+ */
+function browserLaunchRetryDelaysMs() {
+    return remoteBrowserEndpoint() ? [5000, 15000, 30000, 60000] : [5000];
+}
 /**
  * Resolve which browser binary to launch, in order of preference:
  *  1. PUPPETEER_EXECUTABLE_PATH (explicit override)
@@ -473,26 +533,39 @@ class ContentProcessor {
         let browser = null;
         let page = null;
         const launchBrowser = async () => {
-            let b;
-            try {
-                b = await openBrowser();
-            }
-            catch (firstError) {
-                // One retry after a pause — the failure may be transient (e.g.
-                // memory pressure from a page that just closed). If the browser
-                // cannot launch at all, no page in this crawl can be processed,
-                // so escalate as fatal instead of failing every URL one by one.
-                logger.warn(`Browser launch failed (${memoryDiagnostics()}), retrying once in 5s...`, firstError);
-                await new Promise(resolve => setTimeout(resolve, 5000));
+            // Retry after pauses — the failure may be transient (memory
+            // pressure from a page that just closed, or a remote browser
+            // service restarting). If the browser cannot launch at all, no page
+            // in this crawl can be processed, so escalate as fatal instead of
+            // failing every URL one by one.
+            const delays = browserLaunchRetryDelaysMs();
+            const attempts = delays.length + 1;
+            for (let attempt = 1;; attempt++) {
+                let b;
                 try {
                     b = await openBrowser();
+                    const p = await b.newPage();
+                    if (attempt > 1)
+                        logger.info(`Browser launched on attempt ${attempt}/${attempts}`);
+                    return { browser: b, page: p };
                 }
-                catch (secondError) {
-                    throw new BrowserLaunchError(`browser failed to launch twice (${memoryDiagnostics()}): ${secondError instanceof Error ? secondError.message : String(secondError)}`);
+                catch (error) {
+                    // Don't leak a remote session whose first page failed to open
+                    if (b) {
+                        try {
+                            await b.close();
+                        }
+                        catch { /* already gone */ }
+                    }
+                    const reason = describeError(error);
+                    if (attempt >= attempts) {
+                        throw new BrowserLaunchError(`browser failed to launch after ${attempts} attempts (${memoryDiagnostics()}): ${reason}`);
+                    }
+                    const delayMs = delays[attempt - 1];
+                    logger.warn(`Browser launch failed (attempt ${attempt}/${attempts}, ${memoryDiagnostics()}): ${reason} — retrying in ${Math.round(delayMs / 1000)}s...`);
+                    await new Promise(resolve => setTimeout(resolve, delayMs));
                 }
             }
-            const p = await b.newPage();
-            return { browser: b, page: p };
         };
         // Track whether the page needs to be recreated (e.g., after a timeout error)
         let pageNeedsRecreation = false;
@@ -744,16 +817,33 @@ class ContentProcessor {
                 try {
                     logger.info(`Crawling: ${url}`);
                     const sources = referrers.get(url) ?? new Set([baseUrl]);
+                    // A page far larger than max_size would be discarded after
+                    // rendering anyway, and rendering multi-MB pages is what
+                    // OOM-kills the browser. Skip it before Chromium sees it, but
+                    // still harvest its links from the raw HTML so the crawl
+                    // discovers the same pages as before.
+                    const oversized = utils_1.Utils.isPdfUrl(url)
+                        ? null
+                        : await this.probeOversizedPage(url, sourceConfig.max_size, logger);
                     // For HTML pages, ensure the browser is running and pass the shared page
                     // For PDFs, processPage handles them without Puppeteer
-                    const currentPage = utils_1.Utils.isPdfUrl(url) ? undefined : await ensureBrowser();
-                    const result = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
-                        if (status === 404) {
-                            for (const source of sources) {
-                                addBrokenLink(source, reportedUrl);
+                    let currentPage;
+                    let result;
+                    if (oversized) {
+                        logger.warn(`Page is ${oversized.bytes} bytes, over ${exports.OVERSIZED_PAGE_FACTOR}x max size (${sourceConfig.max_size}). Skipping without rendering: ${url}`);
+                        const raw = await this.extractLinksWithoutBrowser(oversized.finalUrl, logger);
+                        result = { content: null, links: raw.links, finalUrl: raw.finalUrl };
+                    }
+                    else {
+                        currentPage = utils_1.Utils.isPdfUrl(url) ? undefined : await ensureBrowser();
+                        result = await this.processPage(url, sourceConfig, (reportedUrl, status) => {
+                            if (status === 404) {
+                                for (const source of sources) {
+                                    addBrokenLink(source, reportedUrl);
+                                }
                             }
-                        }
-                    }, currentPage);
+                        }, currentPage);
+                    }
                     let contentProcessedSuccessfully = false;
                     if (result.content !== null) {
                         contentProcessedSuccessfully = await processPageContent(url, result.content);
@@ -814,7 +904,7 @@ class ContentProcessor {
                     // Page processed successfully — reset protocol error counter and
                     // increment the periodic restart counter
                     consecutiveProtocolErrors = 0;
-                    if (!utils_1.Utils.isPdfUrl(url)) {
+                    if (currentPage) {
                         pagesSinceRestart++;
                     }
                     // Navigate to about:blank to clear any lingering JS, timers, or
@@ -965,6 +1055,77 @@ class ContentProcessor {
         }
         return false;
     }
+    /**
+     * HEAD the page asking for an UNCOMPRESSED body and report its size when it
+     * is more than OVERSIZED_PAGE_FACTOR x maxSize. Compressed lengths are
+     * useless here (a 7 MB docs page is 27 KB of brotli), so a response that is
+     * still encoded, has no Content-Length, fails, or isn't 2xx returns null and
+     * the page is rendered as usual.
+     */
+    async probeOversizedPage(url, maxSize, logger) {
+        try {
+            const response = await axios_1.default.request({
+                method: 'HEAD',
+                url,
+                timeout: 10000,
+                maxRedirects: 5,
+                headers: {
+                    'User-Agent': 'Mozilla/5.0 (compatible; doc2vec crawler)',
+                    'Accept-Encoding': 'identity',
+                },
+                validateStatus: () => true,
+                // Otherwise axios deletes Content-Encoding from HEAD responses,
+                // hiding that a server ignored `identity` and sent a compressed length
+                decompress: false,
+            });
+            if (response.status < 200 || response.status >= 300)
+                return null;
+            const encoding = String(response.headers['content-encoding'] ?? 'identity').toLowerCase();
+            if (encoding !== 'identity')
+                return null;
+            const bytes = Number(response.headers['content-length']);
+            if (!Number.isFinite(bytes) || bytes <= maxSize * exports.OVERSIZED_PAGE_FACTOR)
+                return null;
+            const finalUrl = response.request?.res?.responseUrl || url;
+            return { bytes, finalUrl };
+        }
+        catch (error) {
+            logger.debug(`Size probe failed for ${url}: ${describeError(error)}. Rendering normally.`);
+            return null;
+        }
+    }
+    /**
+     * Collect a page's links from its raw HTML, without a browser. Used for
+     * pages too large to render, so skipping them doesn't cut off the pages
+     * only they link to. JS-inserted links are missed — acceptable for the
+     * static documentation pages that reach this size.
+     */
+    async extractLinksWithoutBrowser(url, logger) {
+        try {
+            const response = await axios_1.default.get(url, {
+                timeout: 60000,
+                maxRedirects: 5,
+                responseType: 'text',
+                maxContentLength: 64 * 1024 * 1024,
+                headers: { 'User-Agent': 'Mozilla/5.0 (compatible; doc2vec crawler)' },
+            });
+            const $ = (0, cheerio_1.load)(response.data);
+            const links = [];
+            $('a[href]').each((_, el) => {
+                const href = $(el).attr('href');
+                if (href && !href.startsWith('#') && !href.startsWith('mailto:')) {
+                    links.push(href);
+                }
+            });
+            logger.debug(`Extracted ${links.length} links from raw HTML of ${url}`);
+            const finalUrl = response.request?.res?.responseUrl || url;
+            return { links, finalUrl };
+        }
+        catch (error) {
+            logger.warn(`Could not fetch ${url} to extract its links: ${describeError(error)}`);
+            return { links: [], finalUrl: url };
+        }
+    }
     async processPage(url, sourceConfig, onHttpStatus, existingPage) {
         const logger = this.logger.child('page-processor');
         logger.debug(`Processing content from ${url}`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.17.3",
+    "version": "2.17.4",
     "type": "commonjs",
     "engines": {
         "node": ">=22"