doc2vec 2.15.0 → 2.16.0

package/README.md

@@ -485,6 +485,42 @@ The script will then:
 7.  Cleanup obsolete chunks.
 8.  Output detailed logs.
 
+## Docker Images and the Browser
+
+Website sources are rendered with a real browser (Puppeteer). The `Dockerfile` builds two images so you can choose where that browser runs:
+
+| Target | Contents | Use it when |
+|---|---|---|
+| `runtime` | doc2vec only, no browser | You run a browser elsewhere and point doc2vec at it (see below). Recommended for clusters: the browser is the one component that renders untrusted input, and it can be updated on its own schedule. |
+| `runtime-chrome` | doc2vec plus a bundled, version-matched Chrome | You want a single self-contained container. `linux/amd64` only. |
+
+```bash
+docker build --target runtime        -t doc2vec .
+docker build --target runtime-chrome -t doc2vec:chrome .
+```
+
+Both images are based on Wolfi (Chainguard) and run as the non-root user `doc2vec` (UID 14000).
+
+### Using a remote browser (`BROWSER_WS_ENDPOINT`)
+
+Set `BROWSER_WS_ENDPOINT` to a Chrome DevTools Protocol WebSocket URL and doc2vec connects to that browser instead of launching one. Any CDP-speaking browser works; [browserless](https://github.com/browserless/browserless) is a convenient packaged one:
+
+```bash
+docker run -d --name browser -e TOKEN=secret -e TIMEOUT=0 --shm-size=1g ghcr.io/browserless/chromium
+docker run --rm --link browser \
+  -e BROWSER_WS_ENDPOINT='ws://browser:3000?token=secret' \
+  -e OPENAI_API_KEY \
+  -v "$PWD/config.yaml:/config/config.yaml:ro" \
+  doc2vec node dist/doc2vec.js /config/config.yaml
+```
+
+Notes:
+
+- A crawl holds a browser for as long as it runs, so disable or raise the browser service's session timeout (browserless: `TIMEOUT=0`).
+- Each connection gets its own browser session; doc2vec closes it when the crawl finishes and every 50 pages.
+- Launch flags such as `--no-sandbox` belong to the process that starts the browser and are not sent to a remote one.
+- With `BROWSER_WS_ENDPOINT` unset, doc2vec launches a local Chrome: `PUPPETEER_EXECUTABLE_PATH` if set, else Puppeteer's downloaded Chrome for Testing, else a system `chromium`.
+
 ## Controller Mode
 
 Besides the one-shot sync, doc2vec can run as a **long-lived controller** that schedules sync jobs, records run history and statistics in Postgres, and serves a web UI for monitoring and managing configs:
@@ -591,12 +627,17 @@ spec:
             - name: OPENAI_API_KEY
               valueFrom:
                 secretKeyRef: { name: doc2vec-secrets, key: openai-api-key }
+            # The browser runs in its own Deployment (e.g. ghcr.io/browserless/chromium
+            # with TIMEOUT=0); see "Docker Images and the Browser". Drop this variable
+            # and use the `-chrome` image to bundle the browser instead.
+            - name: BROWSER_WS_ENDPOINT
+              value: ws://browserless:3000?token=$(BROWSERLESS_TOKEN)
           volumeMounts:
             - name: configs
               mountPath: /etc/doc2vec
           resources:
             requests: { memory: 1Gi }
-            limits: { memory: 4Gi }   # website sources launch headless Chromium
+            limits: { memory: 2Gi }
       volumes:
         - name: configs
           configMap: { name: doc2vec-configs }

package/dist/content-processor.js

@@ -36,7 +36,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContentProcessor = exports.BrowserLaunchError = void 0;
+exports.ContentProcessor = exports.BROWSER_WS_ENDPOINT_ENV = exports.BrowserLaunchError = void 0;
+exports.remoteBrowserEndpoint = remoteBrowserEndpoint;
+exports.openBrowser = openBrowser;
 exports.resolveBrowserExecutablePath = resolveBrowserExecutablePath;
 const readability_1 = require("@mozilla/readability");
 const axios_1 = __importDefault(require("axios"));
@@ -62,6 +64,56 @@ class BrowserLaunchError extends Error {
     }
 }
 exports.BrowserLaunchError = BrowserLaunchError;
+/**
+ * Where the browser comes from.
+ *
+ * Unset (the default): this process launches its own Chromium, which the image
+ * has to carry. Set: this process CONNECTS to a browser running elsewhere over
+ * the Chrome DevTools Protocol, and the image needs no browser at all. The
+ * value is a full CDP WebSocket URL, e.g.
+ *   ws://browserless.qdrant.svc.cluster.local:3000?token=<token>
+ *
+ * A remote browser is the one that keeps the app image small and lets the
+ * browser be updated independently of the app (a browser image tracks Chrome
+ * stable; a distro chromium is frozen at whatever the distro rebuilt last).
+ */
+exports.BROWSER_WS_ENDPOINT_ENV = 'BROWSER_WS_ENDPOINT';
+function remoteBrowserEndpoint() {
+    const raw = process.env[exports.BROWSER_WS_ENDPOINT_ENV]?.trim();
+    return raw ? raw : undefined;
+}
+/**
+ * Chromium flags for a browser THIS process launches. They are not sent to a
+ * remote browser: launch flags belong to the process that starts the browser.
+ */
+const LOCAL_LAUNCH_ARGS = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-dev-shm-usage', // Use /tmp instead of /dev/shm (critical in Docker, default 64MB)
+    '--disable-gpu',
+    '--disable-extensions',
+];
+const BROWSER_PROTOCOL_TIMEOUT_MS = 60000;
+/**
+ * Open a browser: connect to the remote one named by BROWSER_WS_ENDPOINT, or
+ * launch a local one. Callers close it with browser.close() in both cases —
+ * a browserless-style service gives each connection its own browser, so
+ * closing ends only this caller's session.
+ */
+async function openBrowser() {
+    const endpoint = remoteBrowserEndpoint();
+    if (endpoint) {
+        return puppeteer_1.default.connect({
+            browserWSEndpoint: endpoint,
+            protocolTimeout: BROWSER_PROTOCOL_TIMEOUT_MS,
+        });
+    }
+    return puppeteer_1.default.launch({
+        executablePath: await resolveBrowserExecutablePath(),
+        args: LOCAL_LAUNCH_ARGS,
+        protocolTimeout: BROWSER_PROTOCOL_TIMEOUT_MS,
+    });
+}
 /**
  * Resolve which browser binary to launch, in order of preference:
  *  1. PUPPETEER_EXECUTABLE_PATH (explicit override)
@@ -421,21 +473,9 @@ class ContentProcessor {
         let browser = null;
         let page = null;
         const launchBrowser = async () => {
-            const executablePath = await resolveBrowserExecutablePath();
-            const launchOptions = {
-                executablePath,
-                args: [
-                    '--no-sandbox',
-                    '--disable-setuid-sandbox',
-                    '--disable-dev-shm-usage', // Use /tmp instead of /dev/shm (critical in Docker, default 64MB)
-                    '--disable-gpu',
-                    '--disable-extensions',
-                ],
-                protocolTimeout: 60000,
-            };
             let b;
             try {
-                b = await puppeteer_1.default.launch(launchOptions);
+                b = await openBrowser();
             }
             catch (firstError) {
                 // One retry after a pause — the failure may be transient (e.g.
@@ -445,7 +485,7 @@ class ContentProcessor {
                 logger.warn(`Browser launch failed (${memoryDiagnostics()}), retrying once in 5s...`, firstError);
                 await new Promise(resolve => setTimeout(resolve, 5000));
                 try {
-                    b = await puppeteer_1.default.launch(launchOptions);
+                    b = await openBrowser();
                 }
                 catch (secondError) {
                     throw new BrowserLaunchError(`browser failed to launch twice (${memoryDiagnostics()}): ${secondError instanceof Error ? secondError.message : String(secondError)}`);
@@ -962,18 +1002,8 @@ class ContentProcessor {
                 page = existingPage;
             }
             else {
-                // Standalone mode: launch a browser for this single page
-                browser = await puppeteer_1.default.launch({
-                    executablePath: await resolveBrowserExecutablePath(),
-                    args: [
-                        '--no-sandbox',
-                        '--disable-setuid-sandbox',
-                        '--disable-dev-shm-usage',
-                        '--disable-gpu',
-                        '--disable-extensions',
-                    ],
-                    protocolTimeout: 60000,
-                });
+                // Standalone mode: open a browser for this single page
+                browser = await openBrowser();
                 page = await browser.newPage();
             }
             logger.debug(`Navigating to ${url}`);
@@ -1062,7 +1092,7 @@ class ContentProcessor {
                 classesToPreserve: ['article-content', 'original-h1'],
             });
             const article = reader.parse();
-            if (!article) {
+            if (!article || article.content == null) {
                 logger.warn(`Failed to parse article content with Readability for ${url}`);
                 return { content: null, links, finalUrl, etag };
             }