@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/crawler/build-js-redirect-edge.d.ts

@@ -0,0 +1,68 @@
+import type { RedirectEdgeResult } from './types.js';
+import type { PageData } from '@d-zero/beholder';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Parameters for {@link buildJsRedirectEdge}.
+ */
+export interface BuildJsRedirectEdgeParams {
+    /** Source URL puppeteer was asked to navigate to. */
+    url: ExURL;
+    /**
+     * Whether `url` is external to the crawl scope. Propagated onto the
+     * synthesised `PageData` so downstream `linkList.done` classifies the
+     * source the same way the original scrape would have.
+     */
+    isExternal: boolean;
+    /**
+     * Raw error message from `BrowserScrapeResult.error`. The rescue only
+     * fires when this matches the narrow `Page.goto returned null` sentinel
+     * (see {@link isJsRedirectErrorShape}) — anything else falls through
+     * so genuine browser failures surface unchanged.
+     */
+    errorMessage: string | null | undefined;
+    /**
+     * Value of `BrowserScrapeResult.postNavigationUrl` — the URL puppeteer's
+     * `page.url()` reported after the throw. Passed to
+     * {@link deriveJsRedirectTarget} for canonicalisation, credential strip,
+     * and scheme filtering.
+     */
+    postNavigationUrl: string | null | undefined;
+    /**
+     * When present, the rescue builds the redirect-edge `PageData` by
+     * spreading this HEAD-pre-flight result and overriding `redirectPaths`.
+     * Use this on the HEAD-success-then-puppeteer-fail path so the source row
+     * carries the real HTTP-level status / content-type from HEAD. When
+     * omitted, the rescue builds a synthetic placeholder via
+     * {@link linkToPageData} with `status = -1` / `statusText = errorMessage`
+     * — used on the HEAD-fail-then-puppeteer-fallback path where there is
+     * no HEAD response to draw from.
+     */
+    headCheckResult?: PageData;
+}
+/**
+ * Build a JS-redirect `RedirectEdgeResult` when the browser-scrape error
+ * looks like puppeteer's `page.goto() === null` shape AND `page.url()`
+ * resolved to a meaningfully-different URL.
+ *
+ * **Why this helper exists** — both rescue call sites in
+ * `Crawler.#scrapePage` (the HEAD-success-then-puppeteer-fail tail and the
+ * HEAD-fail-then-puppeteer-fallback branch inside `#sendHeadRequest`) run
+ * the same three-step recipe: classify the error shape → derive a JS target
+ * from `page.url()` → synthesise a redirect-edge result. Inlining the recipe
+ * twice means a future change to `deriveJsRedirectTarget`'s contract or the
+ * sentinel string has to be applied to both copies in sync; collapsing into
+ * a single helper keeps the rescue's invariants in one place. The
+ * intentional difference between the two call sites (whether to fold a
+ * HEAD result into the synthesised PageData or to start from a
+ * `linkToPageData` placeholder) is reduced to a single optional parameter
+ * (`headCheckResult`), preserving both shapes without behavioural drift.
+ *
+ * The helper logs at `crawlerLog` whenever it fires so operators tailing
+ * `DEBUG=Nitpicker:Crawler` see exactly which URL got rescued and to which
+ * destination — without this trail the rescue is invisible to anyone
+ * debugging "why did this 200-OK source row land in the archive as a 301?".
+ * @param params - Inputs gathered at the call site.
+ * @returns A `RedirectEdgeResult` when the rescue applies, otherwise
+ *   `null` (the caller should fall through to the existing error path).
+ */
+export declare function buildJsRedirectEdge(params: BuildJsRedirectEdgeParams): RedirectEdgeResult | null;

package/lib/crawler/build-js-redirect-edge.js

@@ -0,0 +1,57 @@
+import { crawlerLog } from '../debug.js';
+import { deriveJsRedirectTarget } from './derive-js-redirect-target.js';
+import { isJsRedirectErrorShape } from './is-js-redirect-error-shape.js';
+import { linkToPageData } from './link-to-page-data.js';
+/**
+ * Build a JS-redirect `RedirectEdgeResult` when the browser-scrape error
+ * looks like puppeteer's `page.goto() === null` shape AND `page.url()`
+ * resolved to a meaningfully-different URL.
+ *
+ * **Why this helper exists** — both rescue call sites in
+ * `Crawler.#scrapePage` (the HEAD-success-then-puppeteer-fail tail and the
+ * HEAD-fail-then-puppeteer-fallback branch inside `#sendHeadRequest`) run
+ * the same three-step recipe: classify the error shape → derive a JS target
+ * from `page.url()` → synthesise a redirect-edge result. Inlining the recipe
+ * twice means a future change to `deriveJsRedirectTarget`'s contract or the
+ * sentinel string has to be applied to both copies in sync; collapsing into
+ * a single helper keeps the rescue's invariants in one place. The
+ * intentional difference between the two call sites (whether to fold a
+ * HEAD result into the synthesised PageData or to start from a
+ * `linkToPageData` placeholder) is reduced to a single optional parameter
+ * (`headCheckResult`), preserving both shapes without behavioural drift.
+ *
+ * The helper logs at `crawlerLog` whenever it fires so operators tailing
+ * `DEBUG=Nitpicker:Crawler` see exactly which URL got rescued and to which
+ * destination — without this trail the rescue is invisible to anyone
+ * debugging "why did this 200-OK source row land in the archive as a 301?".
+ * @param params - Inputs gathered at the call site.
+ * @returns A `RedirectEdgeResult` when the rescue applies, otherwise
+ *   `null` (the caller should fall through to the existing error path).
+ */
+export function buildJsRedirectEdge(params) {
+    if (!isJsRedirectErrorShape(params.errorMessage)) {
+        return null;
+    }
+    const jsRedirectTarget = deriveJsRedirectTarget(params.url.withoutHashAndAuth, params.postNavigationUrl);
+    if (jsRedirectTarget === null) {
+        return null;
+    }
+    const pageData = params.headCheckResult
+        ? { ...params.headCheckResult, redirectPaths: [jsRedirectTarget] }
+        : linkToPageData({
+            url: params.url,
+            isExternal: params.isExternal,
+            isLowerLayer: false,
+            dest: {
+                redirectPaths: [jsRedirectTarget],
+                status: -1,
+                statusText: typeof params.errorMessage === 'string' ? params.errorMessage : '',
+                contentType: null,
+                contentLength: null,
+                responseHeaders: null,
+                title: '',
+            },
+        });
+    crawlerLog('JS-redirect rescue fired for %s → %s (HEAD %s)', params.url.href, jsRedirectTarget, params.headCheckResult ? 'available' : 'absent');
+    return { type: 'redirect-edge', source: 'js-redirect', pageData };
+}

package/lib/crawler/build-redirect-event.d.ts

@@ -0,0 +1,24 @@
+import type { CrawlerEventTypes, InventoryMode } from './types.js';
+import type { PageData } from '../utils/types/types.js';
+/**
+ * Build the payload for the {@link CrawlerEventTypes.redirect} event so the
+ * `{ result, source }` shape is constructed in one place. Pure function —
+ * no I/O, no side effects, just the wiring that connects
+ * {@link derivePageSource} to the emit call.
+ *
+ * Exists so a unit test can pin the wiring directly. Before extraction
+ * the assembly lived inline in `Crawler.#scrape`, which meant the only
+ * way to catch "someone deletes the `source` field" or "someone passes
+ * `inventoryMode` where it should pass `derivePageSource(...)`" was the
+ * inventory E2E. Now a single-purpose spec exercises the construction.
+ *
+ * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
+ * URL the redirect chain STARTS from (the page being scraped), not the
+ * destination — see `Database.recordRedirect` JSDoc for why
+ * lineage propagates from origin, not from the destination.
+ * @param pageData - HEAD-resolved page data carrying the redirect chain.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The originating page URL keyed by `withoutHashAndAuth`.
+ * @returns The exact event payload to feed `emit('redirect', ...)`.
+ */
+export declare function buildRedirectEvent(pageData: PageData, inventoryMode: InventoryMode | null, pageUrlWithoutHashAndAuth: string): CrawlerEventTypes['redirect'];

package/lib/crawler/build-redirect-event.js

@@ -0,0 +1,28 @@
+import { derivePageSource } from './derive-page-source.js';
+/**
+ * Build the payload for the {@link CrawlerEventTypes.redirect} event so the
+ * `{ result, source }` shape is constructed in one place. Pure function —
+ * no I/O, no side effects, just the wiring that connects
+ * {@link derivePageSource} to the emit call.
+ *
+ * Exists so a unit test can pin the wiring directly. Before extraction
+ * the assembly lived inline in `Crawler.#scrape`, which meant the only
+ * way to catch "someone deletes the `source` field" or "someone passes
+ * `inventoryMode` where it should pass `derivePageSource(...)`" was the
+ * inventory E2E. Now a single-purpose spec exercises the construction.
+ *
+ * The originating URL passed in `pageUrlWithoutHashAndAuth` must be the
+ * URL the redirect chain STARTS from (the page being scraped), not the
+ * destination — see `Database.recordRedirect` JSDoc for why
+ * lineage propagates from origin, not from the destination.
+ * @param pageData - HEAD-resolved page data carrying the redirect chain.
+ * @param inventoryMode - Inventory-mode config from `CrawlerOptions.inventoryMode`, or `null` outside `--inventory`.
+ * @param pageUrlWithoutHashAndAuth - The originating page URL keyed by `withoutHashAndAuth`.
+ * @returns The exact event payload to feed `emit('redirect', ...)`.
+ */
+export function buildRedirectEvent(pageData, inventoryMode, pageUrlWithoutHashAndAuth) {
+    return {
+        result: pageData,
+        source: derivePageSource(inventoryMode, pageUrlWithoutHashAndAuth),
+    };
+}

package/lib/crawler/capture-image-dom-paths.d.ts

@@ -0,0 +1,33 @@
+import type { DomPathCandidate } from '../archive/populate-entity-tables/types.js';
+import type { Page as PuppeteerPage } from 'puppeteer';
+/**
+ * Captures every `<img>` in the rendered document — its `outerHTML` and
+ * its `dom_path` string — in document order, for `image_items.dom_path_text_id`
+ * resolution at write time.
+ *
+ * `@d-zero/beholder`'s image metadata carries the element's `outerHTML`
+ * (`sourceCode`) but no positional information, so the crawler runs this
+ * one extra `page.evaluate` after `scrapeStart` returns, while the page
+ * is still alive. The in-browser walk is
+ * {@link ./collect-image-dom-paths.ts} — a self-contained function whose
+ * source puppeteer serialises into the page, and whose spec pins its
+ * output against the Node-side
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} so a
+ * live-crawled archive and one whose dom paths were reconstructed from
+ * HTML snapshots by the migration script produce identical strings for
+ * identical DOM shapes.
+ *
+ * Returns `undefined` on any evaluation failure (page context died,
+ * navigation raced the call) — dom paths are best-effort enrichment and
+ * a capture failure must not fail the scrape. Callers fall back to the
+ * `unknown/<id>` synthetic marker per image via
+ * {@link ../archive/populate-entity-tables/match-images-to-dom-paths.ts}.
+ * @param page - The live puppeteer page, after the scrape completed and
+ *   before the browser closes.
+ * @returns Candidates in document order, or `undefined` when the page
+ *   could not be evaluated.
+ * @example
+ * const result = await scraper.scrapeStart(page, url, options);
+ * const imageDomPaths = await captureImageDomPaths(page);
+ */
+export declare function captureImageDomPaths(page: PuppeteerPage): Promise<DomPathCandidate[] | undefined>;

package/lib/crawler/capture-image-dom-paths.js

@@ -0,0 +1,39 @@
+import { collectImageDomPaths } from './collect-image-dom-paths.js';
+/**
+ * Captures every `<img>` in the rendered document — its `outerHTML` and
+ * its `dom_path` string — in document order, for `image_items.dom_path_text_id`
+ * resolution at write time.
+ *
+ * `@d-zero/beholder`'s image metadata carries the element's `outerHTML`
+ * (`sourceCode`) but no positional information, so the crawler runs this
+ * one extra `page.evaluate` after `scrapeStart` returns, while the page
+ * is still alive. The in-browser walk is
+ * {@link ./collect-image-dom-paths.ts} — a self-contained function whose
+ * source puppeteer serialises into the page, and whose spec pins its
+ * output against the Node-side
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} so a
+ * live-crawled archive and one whose dom paths were reconstructed from
+ * HTML snapshots by the migration script produce identical strings for
+ * identical DOM shapes.
+ *
+ * Returns `undefined` on any evaluation failure (page context died,
+ * navigation raced the call) — dom paths are best-effort enrichment and
+ * a capture failure must not fail the scrape. Callers fall back to the
+ * `unknown/<id>` synthetic marker per image via
+ * {@link ../archive/populate-entity-tables/match-images-to-dom-paths.ts}.
+ * @param page - The live puppeteer page, after the scrape completed and
+ *   before the browser closes.
+ * @returns Candidates in document order, or `undefined` when the page
+ *   could not be evaluated.
+ * @example
+ * const result = await scraper.scrapeStart(page, url, options);
+ * const imageDomPaths = await captureImageDomPaths(page);
+ */
+export async function captureImageDomPaths(page) {
+    try {
+        return await page.evaluate(collectImageDomPaths);
+    }
+    catch {
+        return undefined;
+    }
+}

package/lib/crawler/clear-dns-burned-host-cache.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Clears the DNS-burned host cache and resets the short-circuit counter.
+ * Called between crawl sessions, alongside `clearDestinationCache`, to
+ * avoid leaking state from a previous session into a fresh one.
+ */
+export declare function clearDnsBurnedHostCache(): void;

package/lib/crawler/clear-dns-burned-host-cache.js

@@ -0,0 +1,11 @@
+import { dnsBurnedHostCache } from './dns-burned-host-cache.js';
+import { dnsBurnedHostShortCircuitCounter } from './dns-burned-host-short-circuit-counter.js';
+/**
+ * Clears the DNS-burned host cache and resets the short-circuit counter.
+ * Called between crawl sessions, alongside `clearDestinationCache`, to
+ * avoid leaking state from a previous session into a fresh one.
+ */
+export function clearDnsBurnedHostCache() {
+    dnsBurnedHostCache.clear();
+    dnsBurnedHostShortCircuitCounter.count = 0;
+}

package/lib/crawler/collect-image-dom-paths.d.ts

@@ -0,0 +1,23 @@
+import type { DomPathCandidate } from '../archive/populate-entity-tables/types.js';
+/**
+ * Collects every `<img>` in a document — its `outerHTML` and its
+ * `dom_path` string — in document order.
+ *
+ * **Self-contained by contract.** This function is passed verbatim to
+ * puppeteer's `page.evaluate`, which serialises the function source and
+ * executes it inside the browser: it must not reference imports,
+ * module-scope bindings, or any closure state. That constraint is also
+ * what makes single-sourcing possible — the exact same function body
+ * runs in the browser during a live crawl AND in Node (against a jsdom
+ * document) in its spec, so the dom-path derivation cannot drift between
+ * the two runtimes. The spec additionally pins its output against
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} (the
+ * Node-side derivation the migration script uses) element-for-element.
+ * @param doc - The document to walk. Defaults to the global `document`,
+ *   which is how the in-browser `page.evaluate(collectImageDomPaths)`
+ *   call resolves it; Node callers (specs) pass a jsdom document.
+ * @returns Candidates in document order.
+ * @example
+ * const candidates = await page.evaluate(collectImageDomPaths);
+ */
+export declare function collectImageDomPaths(doc?: Document): DomPathCandidate[];

package/lib/crawler/collect-image-dom-paths.js

@@ -0,0 +1,64 @@
+/**
+ * Collects every `<img>` in a document — its `outerHTML` and its
+ * `dom_path` string — in document order.
+ *
+ * **Self-contained by contract.** This function is passed verbatim to
+ * puppeteer's `page.evaluate`, which serialises the function source and
+ * executes it inside the browser: it must not reference imports,
+ * module-scope bindings, or any closure state. That constraint is also
+ * what makes single-sourcing possible — the exact same function body
+ * runs in the browser during a live crawl AND in Node (against a jsdom
+ * document) in its spec, so the dom-path derivation cannot drift between
+ * the two runtimes. The spec additionally pins its output against
+ * {@link ../archive/populate-entity-tables/derive-dom-path.ts} (the
+ * Node-side derivation the migration script uses) element-for-element.
+ * @param doc - The document to walk. Defaults to the global `document`,
+ *   which is how the in-browser `page.evaluate(collectImageDomPaths)`
+ *   call resolves it; Node callers (specs) pass a jsdom document.
+ * @returns Candidates in document order.
+ * @example
+ * const candidates = await page.evaluate(collectImageDomPaths);
+ */
+export function collectImageDomPaths(doc) {
+    const target = doc ?? document;
+    /**
+     * Counts earlier same-tag siblings, returning the 1-based ordinal.
+     * @param element - The element whose ordinal is being computed.
+     * @param tag - The element's lower-cased tag name.
+     * @returns 1-based ordinal among same-tag siblings.
+     */
+    function computeSiblingOrdinal(element, tag) {
+        let count = 1;
+        let sibling = element.previousElementSibling;
+        while (sibling !== null) {
+            if (sibling.tagName.toLowerCase() === tag) {
+                count += 1;
+            }
+            sibling = sibling.previousElementSibling;
+        }
+        return count;
+    }
+    /**
+     * Derives the slash-joined dom_path for one element.
+     * @param element - The element to derive the path for.
+     * @returns The dom_path string.
+     */
+    function deriveDomPath(element) {
+        const segments = [];
+        let current = element;
+        while (current !== null) {
+            const tag = current.tagName.toLowerCase();
+            if (tag === 'html') {
+                segments.unshift('html');
+                break;
+            }
+            segments.unshift(`${tag}[${computeSiblingOrdinal(current, tag)}]`);
+            current = current.parentElement;
+        }
+        return segments.join('/');
+    }
+    return Array.from(target.querySelectorAll('img'), (img) => ({
+        outerHTML: img.outerHTML,
+        path: deriveDomPath(img),
+    }));
+}

package/lib/crawler/crawler.d.ts

@@ -71,7 +71,8 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * before reaching the dealer so a URL that exists in both sources — which
      * is common in append-mode when a new root coincides with a repromoted
      * previously-external page — does not race on two parallel slots.
-     * @param urls - The list of root URLs to begin crawling from. Must be non-empty.
+     * @param urls - The list of root URLs to begin crawling from. May be empty
+     *   when resumed pending URLs already exist (for example `--retry-failed`).
      * @param opts - Optional overrides; currently only `recursive` is honoured.
      * @param opts.recursive - When `false`, disables recursive discovery and forces list-mode.
      *   Defaults to the constructor option's `recursive` value.
@@ -80,6 +81,26 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
     start(urls: ExURL[], opts?: {
         recursive?: boolean;
     }): void;
+    /**
+     * Launches a fresh Puppeteer browser, runs the beholder scraper, and cleans up.
+     *
+     * WHY per-URL browser: Each URL gets its own browser instance to ensure
+     * complete isolation (cookies, cache, service workers). The browser is always
+     * closed in the `finally` block, even on error.
+     *
+     * The cascade-guard contract for the puppeteer-fallback success / skipped
+     * branches can be exercised via `vi.spyOn(Crawler.prototype,
+     * '_launchBrowserAndScrape')` in unit tests. There is no production
+     * consumer outside this class.
+     * @internal
+     * @param url - Target URL to scrape
+     * @param update - Callback for progress messages
+     * @param isExternal - Whether the URL is external to the crawl scope
+     * @param metadataOnly - When true, only extract title metadata
+     * @param headCheckResult - Optional HEAD result to pass to the scraper, avoiding a redundant request
+     * @returns The scrape result from beholder
+     */
+    private _launchBrowserAndScrape;
     /**
      * The default maximum number of concurrent scraping processes.
      *