@nitpicker/crawler 0.9.0 → 0.12.0

package/lib/crawler/close-browser-safely.js

@@ -0,0 +1,73 @@
+import { crawlerLog } from '../debug.js';
+import { killProcessTree } from './kill-process-tree.js';
+/**
+ * Default time to wait for a graceful `browser.close()` before force-killing
+ * the underlying Chromium process, in milliseconds.
+ */
+const DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000;
+/**
+ * Closes a Puppeteer browser, falling back to a hard tree-kill if the graceful
+ * close hangs.
+ *
+ * WHY: When a page's Chromium session dies mid-scrape (e.g. a viewport change
+ * detaches the frame, surfacing `Attempted to use detached Frame` or
+ * `Session closed`), the CDP connection can be left wedged. A bare
+ * `await browser.close()` then never settles, stalling the `deal()` worker and
+ * hanging the whole crawl. Racing the close against a timeout and SIGKILLing
+ * the Chromium process tree (parent + renderer/network/zygote children) on
+ * expiry guarantees the worker always completes and no orphan subprocesses are
+ * left behind.
+ *
+ * The losing timer is cleared explicitly in `.finally()` so it never keeps the
+ * event loop alive after the race settles (a plain `delay()` in `Promise.race`
+ * would leak the timer until it fires).
+ *
+ * The tree-kill happens via {@link killProcessTree}, which enumerates
+ * descendants through `ps` (POSIX) or delegates to `taskkill /T /F` (Windows).
+ * `childProcess.kill('SIGKILL')` is still invoked on the parent up-front
+ * because Node's `ChildProcess.killed` flag governs how Node treats the
+ * spawn handle (reaping etc.); without it the parent would linger in Node's
+ * process table even after the OS-level kill.
+ * @param browser - The browser to close.
+ * @param timeoutMs - Milliseconds to wait for a graceful close before force-killing.
+ *   Defaults to {@link DEFAULT_CLOSE_TIMEOUT_MS}.
+ * @param deps - Test-time overrides (default-free for production callers).
+ * @returns `true` if the graceful close timed out (and a tree-kill was
+ *   attempted), `false` if `close()` settled in time.
+ */
+export async function closeBrowserSafely(browser, timeoutMs = DEFAULT_CLOSE_TIMEOUT_MS, deps = {}) {
+    // Capture the process up-front: after a successful close() puppeteer
+    // releases its internal reference and process() returns null, so we would
+    // have no handle to tree-kill on timeout.
+    const childProcess = browser.process();
+    let timeoutHandle;
+    const timedOut = await Promise.race([
+        browser
+            .close()
+            .then(() => false)
+            .catch(() => false),
+        new Promise((resolve) => {
+            timeoutHandle = setTimeout(() => resolve(true), timeoutMs);
+        }),
+    ]).finally(() => {
+        if (timeoutHandle) {
+            clearTimeout(timeoutHandle);
+        }
+    });
+    if (timedOut && childProcess && !childProcess.killed) {
+        // Mark the Node ChildProcess as killed so Node's reaping logic treats
+        // it correctly; then walk the OS process tree.
+        childProcess.kill('SIGKILL');
+        // Capture pid once: ChildProcess.pid is technically `number | undefined`
+        // (undefined before spawn settles), and reading it twice across the
+        // `await` below would force the second read to re-widen back to
+        // `number | undefined` regardless of the typeof guard. Snapshotting
+        // makes the type and the runtime value match.
+        const pid = childProcess.pid;
+        if (typeof pid === 'number') {
+            const killTree = deps.killTree ?? ((p, sig) => killProcessTree(p, sig, { log: crawlerLog }));
+            await killTree(pid, 'SIGKILL');
+        }
+    }
+    return timedOut;
+}

package/lib/crawler/crawler.d.ts

@@ -53,8 +53,11 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * @param pending - URLs that were pending (not yet scraped) in the previous session.
      * @param scraped - URLs that were already scraped in the previous session.
      * @param resources - Resource URLs that were already captured in the previous session.
+     * @param pagesScrapedOffset - Number of HTML pages already rendered in previous
+     *   sessions, used to seed the session-spanning progress counter. Defaults to 0
+     *   for callers that don't need cross-session accuracy in the progress display.
      */
-    resume(pending: string[], scraped: string[], resources: string[]): void;
+    resume(pending: string[], scraped: string[], resources: string[], pagesScrapedOffset?: number): void;
     /**
      * Start crawling from one or more root URLs.
      *
@@ -68,7 +71,8 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * before reaching the dealer so a URL that exists in both sources — which
      * is common in append-mode when a new root coincides with a repromoted
      * previously-external page — does not race on two parallel slots.
-     * @param urls - The list of root URLs to begin crawling from. Must be non-empty.
+     * @param urls - The list of root URLs to begin crawling from. May be empty
+     *   when resumed pending URLs already exist (for example `--retry-failed`).
      * @param opts - Optional overrides; currently only `recursive` is honoured.
      * @param opts.recursive - When `false`, disables recursive discovery and forces list-mode.
      *   Defaults to the constructor option's `recursive` value.
@@ -77,6 +81,7 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
     start(urls: ExURL[], opts?: {
         recursive?: boolean;
     }): void;
+    private _launchBrowserAndScrape;
     /**
      * The default maximum number of concurrent scraping processes.
      *