@nitpicker/crawler 0.19.0 → 0.20.0

package/lib/crawler/crawler.js

@@ -55,6 +55,7 @@ import { PreloadShortCircuitError } from './preload-short-circuit-error.js';
 import { probeNetwork } from './probe-network.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 import { redirectDestKey } from './redirect-dest-key.js';
+import { resolveResultWentOffHost } from './resolve-result-went-off-host.js';
 import { resourceToPageData } from './resource-to-page-data.js';
 import { RobotsChecker } from './robots-checker.js';
 import { shouldBurnHost } from './should-burn-host.js';
@@ -631,13 +632,16 @@ export default class Crawler extends EventEmitter {
                     // below. Deliberately NOT also excluding `opts?.metadataOnly`
                     // (unlike the tracker's observation side, which does skip
                     // metadata-only pages — they carry no reliable signature): with
-                    // `--recursive=false`, `handle-scrape-end.ts` marks EVERY anchor
-                    // metadata-only, internal or not, so excluding them here would
-                    // silently disable `--dedupe-cap` for anchor discovery whenever
-                    // `--recursive=false` is set — while gate 2 (the JS-redirect
-                    // direct enqueue below) has no such exclusion and would still
-                    // cap the very same shape, an inconsistency between the two
-                    // discovery paths.
+                    // `--recursive=false`, every INTERNAL anchor that reaches this
+                    // closure is metadata-only (`handle-scrape-end.ts` never issues
+                    // a full-scrape `addUrl` call outside recursive mode; external
+                    // anchors reach it too when `--fetch-external` is on, but those
+                    // are already filtered by the scope check below), so excluding
+                    // `opts?.metadataOnly` here would silently disable
+                    // `--dedupe-cap` for anchor discovery whenever `--recursive=false`
+                    // is set — while gate 2 (the JS-redirect direct enqueue below)
+                    // has no such exclusion and would still cap the very same
+                    // shape, an inconsistency between the two discovery paths.
                     if (this.#options.dedupeCap !== null &&
                         findScopeEntry(newUrl, this.#scope, this.#options) !== null) {
                         const gateShapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
@@ -1171,9 +1175,34 @@ export default class Crawler extends EventEmitter {
                     }
                     log('Saving results%dots%');
                     this.#handleResult(result, url, enqueue, concurrency, precomputedBodyHash);
-                    const parentSource = await this.#resolveParentSource(url);
-                    this.#handleResources(result.resources, parentSource);
-                    this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
+                    // Skip sub-resources / console logs for a result that turned out
+                    // external — NOT the same as this worker's own `isExternal`
+                    // (computed from `url` before navigation). Beholder decides
+                    // `isExternal: false` before navigating and only flips it to
+                    // `true` after seeing the destination's hostname, so a
+                    // same-host source that redirects cross-host still has its
+                    // request/response/console listeners attached under the
+                    // pre-navigation `isExternal: false` for the whole trip. Those
+                    // listeners keep capturing the destination's sub-resources and
+                    // console output even after the flip, so without this guard a
+                    // cross-host redirect leaks the OFF-SCOPE destination's data
+                    // into this archive: its console output would be recorded as
+                    // this page's quality signal, and its resources would leave a
+                    // `resource_ref_edges` row on the (now content-less) redirect
+                    // SOURCE — `linkRedirectSources` deletes that source's
+                    // `anchor_edges` / `image_items` but not `resource_ref_edges`.
+                    // A genuinely external URL never reaches this branch with
+                    // non-empty `resources` / `consoleLogs` in the first place —
+                    // beholder never attaches these listeners for one, per the
+                    // `isExternal` gate in `#fetchData` — so this guard is a no-op
+                    // outside the cross-host-redirect case. See
+                    // `resolveResultWentOffHost`'s JSDoc for how it answers this for
+                    // a `type: 'error'` result, which has no `pageData` to read.
+                    if (!resolveResultWentOffHost(result, url)) {
+                        const parentSource = await this.#resolveParentSource(url);
+                        this.#handleResources(result.resources, parentSource);
+                        this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
+                    }
                     log(formatResultSummary(result));
                     // Phase errors must be emitted AFTER 'page' / 'externalPage'
                     // so the orchestrator's WriteQueue sees `setPage` before

package/lib/crawler/handle-scrape-end.js

@@ -40,9 +40,12 @@ export function handleScrapeEnd(result, linkList, scope, options, addUrl) {
  *    deepest matching scope.
  * 2. For internal anchors without credentials, inherits auth from the matched
  *    scope and rebuilds `withoutHash` with the injected auth.
- * 3. In recursive mode: enqueues internal anchors for full scraping, and
- *    external anchors for metadata-only scraping (when `fetchExternal` is on).
- * 4. In non-recursive mode: enqueues every anchor for metadata-only scraping.
+ * 3. External anchors are skipped entirely when `fetchExternal` is off — this
+ *    gate applies in both recursive and non-recursive mode.
+ * 4. In recursive mode: enqueues internal anchors for full scraping, and
+ *    external anchors (when not skipped by 3) for metadata-only scraping.
+ * 5. In non-recursive mode: enqueues every anchor that survives gate 3 for
+ *    metadata-only scraping, internal or not.
  * @param anchors - The list of anchor data extracted from the page.
  * @param scope - Map of hostnames to their scope URLs.
  * @param options - Crawler configuration options.
@@ -69,13 +72,19 @@ function processAnchors(anchors, scope, options, addUrl) {
             const withoutHash = `${anchor.href.protocol}//${auth}${host}${body ? `/${body}` : ''}`;
             anchor.href.withoutHash = withoutHash;
         }
-        if (options.recursive) {
-            if (matchedScope) {
-                addUrl(anchor.href);
-            }
-            else if (options.fetchExternal) {
-                addUrl(anchor.href, { metadataOnly: true });
-            }
+        // `fetchExternal` gates external anchors the same way in both modes —
+        // checked once, ahead of the recursive/non-recursive split below,
+        // rather than only inside the recursive branch. Non-recursive
+        // discovery has no other point that enforces `fetchExternal`: every
+        // anchor that reaches the split falls straight into the unconditional
+        // metadata-only `addUrl` call, so scoping the check to the recursive
+        // branch alone would make `--list` / `--single` / `--no-recursive`
+        // unable to honour `--no-fetch-external` at all.
+        if (!matchedScope && !options.fetchExternal) {
+            continue;
+        }
+        if (options.recursive && matchedScope) {
+            addUrl(anchor.href);
             continue;
         }
         addUrl(anchor.href, { metadataOnly: true });

package/lib/crawler/resolve-result-went-off-host.d.ts

@@ -0,0 +1,34 @@
+import type { BrowserScrapeResult } from './types.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Determines whether a scrape result's actual destination ended up on a
+ * different host than the URL that was requested.
+ *
+ * Exists for the cross-host-redirect leak guard in `Crawler`'s worker body:
+ * beholder decides `isExternal: false` before navigating and only flips it
+ * to `true` after seeing the destination's hostname, so a same-host source
+ * that redirects cross-host still has its sub-resource / console listeners
+ * attached under the pre-navigation `isExternal: false` for the whole trip
+ * — the caller uses this function's answer to decide whether to discard
+ * what those listeners captured.
+ *
+ * `pageData` (present on `type: 'success'`) carries the authoritative
+ * post-navigation `isExternal` and is used directly. A `type: 'error'`
+ * result has no `pageData`, so `postNavigationUrl` — the puppeteer-side
+ * `page.url()` captured by the JS-redirect rescue (see `BrowserScrapeResult`
+ * JSDoc) — is used as the fallback signal instead. When neither is
+ * available (e.g. the browser context died before `page.url()` could be
+ * read, or the result is `type: 'skipped'`, which carries neither field),
+ * this returns `false` — the alternative would discard legitimate
+ * console/resource data for the overwhelmingly common same-host case on
+ * the mere possibility of a leak this function has no evidence for.
+ * @param result - The non-redirect-edge scrape outcome.
+ * @param url - The originally-requested URL.
+ * @returns `true` when the browser is known to have ended up off-host.
+ * @example
+ * const wentOffHost = resolveResultWentOffHost(result, url);
+ * if (!wentOffHost) {
+ *   handleResources(result.resources, parentSource);
+ * }
+ */
+export declare function resolveResultWentOffHost(result: BrowserScrapeResult, url: ExURL): boolean;

package/lib/crawler/resolve-result-went-off-host.js

@@ -0,0 +1,39 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+/**
+ * Determines whether a scrape result's actual destination ended up on a
+ * different host than the URL that was requested.
+ *
+ * Exists for the cross-host-redirect leak guard in `Crawler`'s worker body:
+ * beholder decides `isExternal: false` before navigating and only flips it
+ * to `true` after seeing the destination's hostname, so a same-host source
+ * that redirects cross-host still has its sub-resource / console listeners
+ * attached under the pre-navigation `isExternal: false` for the whole trip
+ * — the caller uses this function's answer to decide whether to discard
+ * what those listeners captured.
+ *
+ * `pageData` (present on `type: 'success'`) carries the authoritative
+ * post-navigation `isExternal` and is used directly. A `type: 'error'`
+ * result has no `pageData`, so `postNavigationUrl` — the puppeteer-side
+ * `page.url()` captured by the JS-redirect rescue (see `BrowserScrapeResult`
+ * JSDoc) — is used as the fallback signal instead. When neither is
+ * available (e.g. the browser context died before `page.url()` could be
+ * read, or the result is `type: 'skipped'`, which carries neither field),
+ * this returns `false` — the alternative would discard legitimate
+ * console/resource data for the overwhelmingly common same-host case on
+ * the mere possibility of a leak this function has no evidence for.
+ * @param result - The non-redirect-edge scrape outcome.
+ * @param url - The originally-requested URL.
+ * @returns `true` when the browser is known to have ended up off-host.
+ * @example
+ * const wentOffHost = resolveResultWentOffHost(result, url);
+ * if (!wentOffHost) {
+ *   handleResources(result.resources, parentSource);
+ * }
+ */
+export function resolveResultWentOffHost(result, url) {
+    if (result.pageData !== undefined) {
+        return result.pageData.isExternal;
+    }
+    return (result.postNavigationUrl !== undefined &&
+        parseUrl(result.postNavigationUrl)?.hostname !== url.hostname);
+}

package/lib/crawler/types.d.ts

@@ -42,8 +42,18 @@ export interface RedirectEdgeResult {
  * The outcome of {@link Crawler.#scrapePage}: either a normal scrape result from
  * the browser/HEAD pipeline, or a {@link RedirectEdgeResult} when the URL's
  * redirect destination was already rendered and only the edge needs recording.
+ *
+ * The non-redirect-edge member is {@link BrowserScrapeResult}, not bare
+ * beholder `ScrapeResult` — every `#scrapePage` return site is either a
+ * `_launchBrowserAndScrape` call (which returns `BrowserScrapeResult`) or a
+ * plain `ScrapeResult`-shaped literal (structurally compatible, since
+ * `postNavigationUrl` is optional). Callers that reach a `type: 'error'`
+ * result and need to know whether the browser ended up off-host before
+ * failing (`pageData` is absent on error, so `pageData.isExternal` is not
+ * available) read `postNavigationUrl` for that signal — see
+ * `resolveResultWentOffHost`.
  */
-export type ScrapeOutcome = ScrapeResult | RedirectEdgeResult;
+export type ScrapeOutcome = BrowserScrapeResult | RedirectEdgeResult;
 /**
  * Internal envelope returned by {@link Crawler.#launchBrowserAndScrape} that
  * augments beholder's {@link ScrapeResult} with the puppeteer-side

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.19.0",
+	"version": "0.20.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -31,7 +31,7 @@
 	},
 	"dependencies": {
 		"@d-zero/beholder": "4.2.3",
-		"@d-zero/dealer": "1.12.0",
+		"@d-zero/dealer": "1.13.1",
 		"@d-zero/fs": "0.2.7",
 		"@d-zero/shared": "0.23.0",
 		"ansi-colors": "4.1.3",
@@ -51,5 +51,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "b79b5c304820d924168a139e7148e0593c29fe1c"
+	"gitHead": "cbe8a4e74322386a6e07b2d4f1417dfba222f0b1"
 }