@nitpicker/crawler 0.18.1 → 0.20.0

package/lib/crawler/crawler.js

@@ -1,4 +1,3 @@
-var _a;
 import { existsSync } from 'node:fs';
 import path from 'node:path';
 import Scraper from '@d-zero/beholder';
@@ -14,6 +13,7 @@ import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
 import { buildRedirectEvent } from './build-redirect-event.js';
+import { captureCustomElements } from './capture-custom-elements.js';
 import { captureImageDomPaths } from './capture-image-dom-paths.js';
 import { chooseProbeHost } from './choose-probe-host.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
@@ -55,6 +55,7 @@ import { PreloadShortCircuitError } from './preload-short-circuit-error.js';
 import { probeNetwork } from './probe-network.js';
 import { protocolAgnosticKey } from './protocol-agnostic-key.js';
 import { redirectDestKey } from './redirect-dest-key.js';
+import { resolveResultWentOffHost } from './resolve-result-went-off-host.js';
 import { resourceToPageData } from './resource-to-page-data.js';
 import { RobotsChecker } from './robots-checker.js';
 import { shouldBurnHost } from './should-burn-host.js';
@@ -92,7 +93,7 @@ const DEFAULT_DEDUPE_MAP_CAP = 100_000;
  * Crawling is performed concurrently using the dealer pattern, with
  * configurable parallelism up to {@link Crawler.MAX_PROCESS_LENGTH}.
  */
-class Crawler extends EventEmitter {
+export default class Crawler extends EventEmitter {
     /** Controller used to cancel the deal-based crawl via its AbortSignal. */
     #abortController = new AbortController();
     /**
@@ -556,10 +557,11 @@ class Crawler extends EventEmitter {
      *   queue, prioritising likely-HTML URLs to the front (see {@link partitionUrlsByHtml}).
      *   Accepts a batch so a group of URLs (e.g. predicted pagination) keeps its order.
      * @param concurrency - Current concurrency level, used to determine predicted URL count
-     * @param precomputedBodyHash - This page's body hash, if the caller already
-     *   computed it (the predicted-content-duplicate check, A-3, computes it for
-     *   every predicted page regardless of `--dedupe-cap`) — reused for the
-     *   dedupe-cap observation below instead of hashing the same html twice.
+     * @param precomputedBodyHash - This page's body hash, computed once by the
+     *   caller for every internal page with a rendered HTML body (see the
+     *   computation site's comment) — reused here for the dedupe-cap
+     *   observation, and forwarded via the `page` event so `update-page.ts`
+     *   does not hash the same html again.
      */
     #handleResult(result, url, enqueue, concurrency, precomputedBodyHash) {
         switch (result.type) {
@@ -595,6 +597,14 @@ class Crawler extends EventEmitter {
                     const shapeKey = computeShapeKey(result.pageData.url.withoutHashAndAuth);
                     const metaSig = computeMetaSignature(result.pageData.meta);
                     if (shapeKey && metaSig) {
+                        // `precomputedBodyHash` is non-null here in every reachable
+                        // case: this branch's guard (`!isExternal && html.length > 0`,
+                        // modulo the `isMetadataOnly` exclusion which only narrows it)
+                        // is a subset of the unconditional computation site's condition
+                        // above. The `??` fallback is not expected to ever fire — it is
+                        // kept only as a defensive backstop against a future edit to
+                        // either condition silently breaking that invariant, favouring
+                        // a slow-but-correct recomputation over a crash.
                         const bodyHash = precomputedBodyHash ?? computeBodyHash(result.pageData.html);
                         const ogUrlMismatch = resolveOgUrlMismatch(result.pageData.meta, result.pageData.url.href);
                         const event = this.#dedupeCapTracker.observe({
@@ -622,13 +632,16 @@ class Crawler extends EventEmitter {
                     // below. Deliberately NOT also excluding `opts?.metadataOnly`
                     // (unlike the tracker's observation side, which does skip
                     // metadata-only pages — they carry no reliable signature): with
-                    // `--recursive=false`, `handle-scrape-end.ts` marks EVERY anchor
-                    // metadata-only, internal or not, so excluding them here would
-                    // silently disable `--dedupe-cap` for anchor discovery whenever
-                    // `--recursive=false` is set — while gate 2 (the JS-redirect
-                    // direct enqueue below) has no such exclusion and would still
-                    // cap the very same shape, an inconsistency between the two
-                    // discovery paths.
+                    // `--recursive=false`, every INTERNAL anchor that reaches this
+                    // closure is metadata-only (`handle-scrape-end.ts` never issues
+                    // a full-scrape `addUrl` call outside recursive mode; external
+                    // anchors reach it too when `--fetch-external` is on, but those
+                    // are already filtered by the scope check below), so excluding
+                    // `opts?.metadataOnly` here would silently disable
+                    // `--dedupe-cap` for anchor discovery whenever `--recursive=false`
+                    // is set — while gate 2 (the JS-redirect direct enqueue below)
+                    // has no such exclusion and would still cap the very same
+                    // shape, an inconsistency between the two discovery paths.
                     if (this.#options.dedupeCap !== null &&
                         findScopeEntry(newUrl, this.#scope, this.#options) !== null) {
                         const gateShapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
@@ -702,6 +715,7 @@ class Crawler extends EventEmitter {
                         void this.emit('page', {
                             result: result.pageData,
                             source: pageSource,
+                            bodyHash: precomputedBodyHash,
                         });
                     }
                 }
@@ -740,7 +754,16 @@ class Crawler extends EventEmitter {
                         });
                     }
                     else {
-                        void this.emit('page', { result: pageResult, source: pageSource });
+                        // `pageResult` here always carries `html: ''` (`linkToPageData`'s
+                        // error-fallback shape), so it never reaches `update-page.ts`'s
+                        // `html.length > 0` write gate — `bodyHash: null` is explicit
+                        // rather than relying on that gate to make an omitted field
+                        // harmless, so this stays correct if that ever changes.
+                        void this.emit('page', {
+                            result: pageResult,
+                            source: pageSource,
+                            bodyHash: null,
+                        });
                     }
                 }
                 void this.emit('error', {
@@ -906,7 +929,7 @@ class Crawler extends EventEmitter {
         }
         const concurrency = this.#options.parallels
             ? Math.max(this.#options.parallels, 1)
-            : _a.MAX_PROCESS_LENGTH;
+            : Crawler.MAX_PROCESS_LENGTH;
         await deal(initialUrls, (url, update, _index, setLineHeader, push, unshift) => {
             const matchedScope = findScopeEntry(url, this.#scope, this.#options);
             const isExternal = matchedScope === null;
@@ -961,10 +984,12 @@ class Crawler extends EventEmitter {
                 const markBrowserScrape = () => {
                     renderedInBrowser = true;
                 };
-                // Set by the predicted-content-duplicate check below (A-3) when it
-                // computes this page's body hash, so `#handleResult`'s dedupe-cap
-                // observation (also gated on this page's html) can reuse it instead
-                // of hashing the same html a second time.
+                // Set below for every internal page with a rendered HTML body
+                // (not just predicted ones — see the computation site's comment),
+                // so both `#handleResult`'s dedupe-cap observation and the `page`
+                // event's `bodyHash` payload (ultimately consumed by
+                // `update-page.ts`'s `page_meta.body_hash` write) reuse this one
+                // value instead of each hashing the same html again.
                 let precomputedBodyHash = null;
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
@@ -1104,30 +1129,41 @@ class Crawler extends EventEmitter {
                         log(c.dim('Predicted (discarded)'));
                         return;
                     }
-                    // Discard a predicted URL whose rendered body is a
-                    // byte-for-byte duplicate of the previous predicted page of the
-                    // same shape, and stop generating further predictions for that
-                    // shape (checked above, in the pagination-pattern branch). This
-                    // is the always-on backstop against a site that returns 2xx for
-                    // any extrapolated token but ignores it entirely (e.g. always
-                    // serving the same "no results" template) — `shouldDiscardPredicted`
-                    // alone cannot see this, since it only inspects HTTP status.
-                    if (isPredicted &&
-                        result.type === 'success' &&
+                    // Compute this page's body hash once, up front, for every
+                    // internal page with a rendered HTML body — not just predicted
+                    // ones. This condition intentionally mirrors `update-page.ts`'s
+                    // `writeHtml && page.html.length > 0` write gate (internal pages
+                    // are exactly the ones `setPage` — as opposed to
+                    // `setExternalPage` — writes a body through), so the value
+                    // computed here can be forwarded through the `page` event all
+                    // the way to that write and reused there instead of hashing the
+                    // same html a second time.
+                    if (result.type === 'success' &&
                         result.pageData &&
+                        !result.pageData.isExternal &&
                         result.pageData.html.length > 0) {
-                        const shapeKey = computeShapeKey(url.withoutHashAndAuth);
-                        if (shapeKey) {
-                            const bodyHash = computeBodyHash(result.pageData.html);
-                            precomputedBodyHash = bodyHash;
-                            const lastBodyHash = this.#predictedShapeBodyHashes.get(shapeKey) ?? null;
-                            if (isPredictedContentDuplicate(bodyHash, lastBodyHash)) {
-                                this.#predictedShapeStopped.add(shapeKey);
-                                handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
-                                log(c.dim('Predicted (content duplicate, discarded)'));
-                                return;
+                        precomputedBodyHash = computeBodyHash(result.pageData.html);
+                        // Discard a predicted URL whose rendered body is a
+                        // byte-for-byte duplicate of the previous predicted page of
+                        // the same shape, and stop generating further predictions for
+                        // that shape (checked above, in the pagination-pattern
+                        // branch). This is the always-on backstop against a site
+                        // that returns 2xx for any extrapolated token but ignores it
+                        // entirely (e.g. always serving the same "no results"
+                        // template) — `shouldDiscardPredicted` alone cannot see
+                        // this, since it only inspects HTTP status.
+                        if (isPredicted) {
+                            const shapeKey = computeShapeKey(url.withoutHashAndAuth);
+                            if (shapeKey) {
+                                const lastBodyHash = this.#predictedShapeBodyHashes.get(shapeKey) ?? null;
+                                if (isPredictedContentDuplicate(precomputedBodyHash, lastBodyHash)) {
+                                    this.#predictedShapeStopped.add(shapeKey);
+                                    handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
+                                    log(c.dim('Predicted (content duplicate, discarded)'));
+                                    return;
+                                }
+                                this.#predictedShapeBodyHashes.set(shapeKey, precomputedBodyHash);
                             }
-                            this.#predictedShapeBodyHashes.set(shapeKey, bodyHash);
                         }
                     }
                     // Count only after discard check: rendered HTML pages that
@@ -1139,9 +1175,34 @@ class Crawler extends EventEmitter {
                     }
                     log('Saving results%dots%');
                     this.#handleResult(result, url, enqueue, concurrency, precomputedBodyHash);
-                    const parentSource = await this.#resolveParentSource(url);
-                    this.#handleResources(result.resources, parentSource);
-                    this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
+                    // Skip sub-resources / console logs for a result that turned out
+                    // external — NOT the same as this worker's own `isExternal`
+                    // (computed from `url` before navigation). Beholder decides
+                    // `isExternal: false` before navigating and only flips it to
+                    // `true` after seeing the destination's hostname, so a
+                    // same-host source that redirects cross-host still has its
+                    // request/response/console listeners attached under the
+                    // pre-navigation `isExternal: false` for the whole trip. Those
+                    // listeners keep capturing the destination's sub-resources and
+                    // console output even after the flip, so without this guard a
+                    // cross-host redirect leaks the OFF-SCOPE destination's data
+                    // into this archive: its console output would be recorded as
+                    // this page's quality signal, and its resources would leave a
+                    // `resource_ref_edges` row on the (now content-less) redirect
+                    // SOURCE — `linkRedirectSources` deletes that source's
+                    // `anchor_edges` / `image_items` but not `resource_ref_edges`.
+                    // A genuinely external URL never reaches this branch with
+                    // non-empty `resources` / `consoleLogs` in the first place —
+                    // beholder never attaches these listeners for one, per the
+                    // `isExternal` gate in `#fetchData` — so this guard is a no-op
+                    // outside the cross-host-redirect case. See
+                    // `resolveResultWentOffHost`'s JSDoc for how it answers this for
+                    // a `type: 'error'` result, which has no `pageData` to read.
+                    if (!resolveResultWentOffHost(result, url)) {
+                        const parentSource = await this.#resolveParentSource(url);
+                        this.#handleResources(result.resources, parentSource);
+                        this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
+                    }
                     log(formatResultSummary(result));
                     // Phase errors must be emitted AFTER 'page' / 'externalPage'
                     // so the orchestrator's WriteQueue sees `setPage` before
@@ -1875,17 +1936,42 @@ class Crawler extends EventEmitter {
             // `image_items.dom_path_text_id` resolution at write time; a
             // capture failure (or a page with no images) falls back to the
             // synthetic `unknown/<n>` markers, so this stays best-effort.
-            if (result.type === 'success' &&
-                result.pageData &&
-                result.pageData.imageList.length > 0) {
-                const imageDomPaths = await captureImageDomPaths(page);
-                if (imageDomPaths !== undefined) {
-                    const withDomPaths = {
-                        ...result.pageData,
-                        imageDomPaths,
-                    };
-                    result.pageData = withDomPaths;
-                }
+            // Image dom-path capture and custom-element (Web Component)
+            // capture are two independent `page.evaluate()` round-trips over
+            // the same still-alive `page` — neither reads the other's
+            // output, so they run concurrently instead of paying both
+            // round-trips' latency in sequence. Each keeps its own
+            // pre-existing gate (image dom-paths on a non-empty imageList,
+            // custom elements on a resolved main-content region — unrelated
+            // to each other, so neither gate is relaxed by running together).
+            // Custom-element capture exists because beholder's
+            // `MainContentsData` has no `customElements` category, so
+            // nitpicker captures it itself.
+            const shouldCaptureImageDomPaths = result.type === 'success' &&
+                result.pageData !== undefined &&
+                result.pageData.imageList.length > 0;
+            const shouldCaptureCustomElements = result.type === 'success' &&
+                result.pageData !== undefined &&
+                result.pageData.mainContents !== null;
+            const [imageDomPaths, mainContentCustomElements] = await Promise.all([
+                shouldCaptureImageDomPaths ? captureImageDomPaths(page) : undefined,
+                shouldCaptureCustomElements
+                    ? captureCustomElements(page, this.#options.mainContentSelector)
+                    : undefined,
+            ]);
+            if (imageDomPaths !== undefined && result.pageData) {
+                const withDomPaths = {
+                    ...result.pageData,
+                    imageDomPaths,
+                };
+                result.pageData = withDomPaths;
+            }
+            if (mainContentCustomElements !== undefined && result.pageData) {
+                const withCustomElements = {
+                    ...result.pageData,
+                    mainContentCustomElements,
+                };
+                result.pageData = withCustomElements;
             }
             update('Closing browser%dots%');
             // JS-redirect rescue capture: when `scrapeStart` catches a
@@ -1970,8 +2056,6 @@ class Crawler extends EventEmitter {
      */
     static MAX_PROCESS_LENGTH = 10;
 }
-_a = Crawler;
-export default Crawler;
 /**
  * Colorize an HTTP status code string for terminal display.
  *

package/lib/crawler/fetch-destination.js

@@ -1,4 +1,5 @@
 import { delay } from '@d-zero/shared/delay';
+import { raceWithTimeout } from '@d-zero/shared/race-with-timeout';
 import redirects from 'follow-redirects';
 import { decodeAuthCredential } from './decode-auth-credential.js';
 import { destinationCache } from './destination-cache.js';
@@ -34,19 +35,11 @@ export async function fetchDestination(params) {
     }
     const effectiveMethod = titleBytesLimit == null ? method : 'GET';
     const raceTimeoutMs = timeout ?? DEFAULT_HEAD_TIMEOUT_MS;
-    // Race the fetch against the requested timeout. The losing timer is cleared
-    // explicitly so it never keeps the event loop alive after the race settles
-    // (a plain `delay()` in `Promise.race` would leak the timer until it fires).
-    let timeoutHandle;
-    const result = await Promise.race([
-        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent, timeout).catch((error) => error instanceof Error ? error : new Error(String(error))),
-        new Promise((resolve) => {
-            timeoutHandle = setTimeout(() => resolve(new NetTimeoutError(url.href)), raceTimeoutMs);
-        }),
-    ]).finally(() => {
-        if (timeoutHandle)
-            clearTimeout(timeoutHandle);
-    });
+    // Race the fetch against the requested timeout via `raceWithTimeout`, which
+    // clears the losing timer internally so it never keeps the event loop
+    // alive after the race settles.
+    const { result: challengeResult, timeout: timedOut } = await raceWithTimeout(() => _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent, timeout).catch((error) => error instanceof Error ? error : new Error(String(error))), raceTimeoutMs);
+    const result = timedOut ? new NetTimeoutError(url.href) : challengeResult;
     // HEAD failure fallback: a WAF / middlebox that silently drops HEAD will
     // surface as NetTimeoutError / parse-error / connection-reset here even
     // though the same URL serves a normal GET response. Try GET once (using

package/lib/crawler/find-package-dir.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Locates `packageName`'s directory the way Node's module resolution would —
+ * walking `node_modules` upward from `fromDir` — without going through the
+ * package's own `exports` map.
+ *
+ * A plain `require.resolve()` on an ESM-only package (`exports` declaring
+ * only the `"import"` condition) fails from a CJS `createRequire` context,
+ * and there is no stable public API to run Node's ESM resolution algorithm
+ * from an arbitrary base directory. Walking `node_modules` directly
+ * sidesteps the `exports` map entirely — it only needs the package's
+ * directory, not its resolved entry file.
+ * @param fromDir - Directory to start walking upward from.
+ * @param packageName - Package name to locate (e.g. `puppeteer`,
+ *   `@d-zero/beholder`).
+ * @throws {Error} When no `node_modules/<packageName>` directory is found
+ *   between `fromDir` and the filesystem root.
+ * @example
+ * ```ts
+ * import { findPackageDir } from './find-package-dir.js';
+ *
+ * const puppeteerDir = findPackageDir(import.meta.dirname, 'puppeteer');
+ * ```
+ */
+export declare function findPackageDir(fromDir: string, packageName: string): string;

package/lib/crawler/find-package-dir.js

@@ -0,0 +1,39 @@
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+/**
+ * Locates `packageName`'s directory the way Node's module resolution would —
+ * walking `node_modules` upward from `fromDir` — without going through the
+ * package's own `exports` map.
+ *
+ * A plain `require.resolve()` on an ESM-only package (`exports` declaring
+ * only the `"import"` condition) fails from a CJS `createRequire` context,
+ * and there is no stable public API to run Node's ESM resolution algorithm
+ * from an arbitrary base directory. Walking `node_modules` directly
+ * sidesteps the `exports` map entirely — it only needs the package's
+ * directory, not its resolved entry file.
+ * @param fromDir - Directory to start walking upward from.
+ * @param packageName - Package name to locate (e.g. `puppeteer`,
+ *   `@d-zero/beholder`).
+ * @throws {Error} When no `node_modules/<packageName>` directory is found
+ *   between `fromDir` and the filesystem root.
+ * @example
+ * ```ts
+ * import { findPackageDir } from './find-package-dir.js';
+ *
+ * const puppeteerDir = findPackageDir(import.meta.dirname, 'puppeteer');
+ * ```
+ */
+export function findPackageDir(fromDir, packageName) {
+    let dir = fromDir;
+    for (;;) {
+        const candidate = path.join(dir, 'node_modules', packageName);
+        if (existsSync(path.join(candidate, 'package.json'))) {
+            return candidate;
+        }
+        const parent = path.dirname(dir);
+        if (parent === dir) {
+            throw new Error(`Could not locate "${packageName}" from ${fromDir}`);
+        }
+        dir = parent;
+    }
+}

package/lib/crawler/handle-scrape-end.js

@@ -40,9 +40,12 @@ export function handleScrapeEnd(result, linkList, scope, options, addUrl) {
  *    deepest matching scope.
  * 2. For internal anchors without credentials, inherits auth from the matched
  *    scope and rebuilds `withoutHash` with the injected auth.
- * 3. In recursive mode: enqueues internal anchors for full scraping, and
- *    external anchors for metadata-only scraping (when `fetchExternal` is on).
- * 4. In non-recursive mode: enqueues every anchor for metadata-only scraping.
+ * 3. External anchors are skipped entirely when `fetchExternal` is off — this
+ *    gate applies in both recursive and non-recursive mode.
+ * 4. In recursive mode: enqueues internal anchors for full scraping, and
+ *    external anchors (when not skipped by 3) for metadata-only scraping.
+ * 5. In non-recursive mode: enqueues every anchor that survives gate 3 for
+ *    metadata-only scraping, internal or not.
  * @param anchors - The list of anchor data extracted from the page.
  * @param scope - Map of hostnames to their scope URLs.
  * @param options - Crawler configuration options.
@@ -69,13 +72,19 @@ function processAnchors(anchors, scope, options, addUrl) {
             const withoutHash = `${anchor.href.protocol}//${auth}${host}${body ? `/${body}` : ''}`;
             anchor.href.withoutHash = withoutHash;
         }
-        if (options.recursive) {
-            if (matchedScope) {
-                addUrl(anchor.href);
-            }
-            else if (options.fetchExternal) {
-                addUrl(anchor.href, { metadataOnly: true });
-            }
+        // `fetchExternal` gates external anchors the same way in both modes —
+        // checked once, ahead of the recursive/non-recursive split below,
+        // rather than only inside the recursive branch. Non-recursive
+        // discovery has no other point that enforces `fetchExternal`: every
+        // anchor that reaches the split falls straight into the unconditional
+        // metadata-only `addUrl` call, so scoping the check to the recursive
+        // branch alone would make `--list` / `--single` / `--no-recursive`
+        // unable to honour `--no-fetch-external` at all.
+        if (!matchedScope && !options.fetchExternal) {
+            continue;
+        }
+        if (options.recursive && matchedScope) {
+            addUrl(anchor.href);
             continue;
         }
         addUrl(anchor.href, { metadataOnly: true });

package/lib/crawler/resolve-result-went-off-host.d.ts

@@ -0,0 +1,34 @@
+import type { BrowserScrapeResult } from './types.js';
+import type { ExURL } from '@d-zero/shared/parse-url';
+/**
+ * Determines whether a scrape result's actual destination ended up on a
+ * different host than the URL that was requested.
+ *
+ * Exists for the cross-host-redirect leak guard in `Crawler`'s worker body:
+ * beholder decides `isExternal: false` before navigating and only flips it
+ * to `true` after seeing the destination's hostname, so a same-host source
+ * that redirects cross-host still has its sub-resource / console listeners
+ * attached under the pre-navigation `isExternal: false` for the whole trip
+ * — the caller uses this function's answer to decide whether to discard
+ * what those listeners captured.
+ *
+ * `pageData` (present on `type: 'success'`) carries the authoritative
+ * post-navigation `isExternal` and is used directly. A `type: 'error'`
+ * result has no `pageData`, so `postNavigationUrl` — the puppeteer-side
+ * `page.url()` captured by the JS-redirect rescue (see `BrowserScrapeResult`
+ * JSDoc) — is used as the fallback signal instead. When neither is
+ * available (e.g. the browser context died before `page.url()` could be
+ * read, or the result is `type: 'skipped'`, which carries neither field),
+ * this returns `false` — the alternative would discard legitimate
+ * console/resource data for the overwhelmingly common same-host case on
+ * the mere possibility of a leak this function has no evidence for.
+ * @param result - The non-redirect-edge scrape outcome.
+ * @param url - The originally-requested URL.
+ * @returns `true` when the browser is known to have ended up off-host.
+ * @example
+ * const wentOffHost = resolveResultWentOffHost(result, url);
+ * if (!wentOffHost) {
+ *   handleResources(result.resources, parentSource);
+ * }
+ */
+export declare function resolveResultWentOffHost(result: BrowserScrapeResult, url: ExURL): boolean;

package/lib/crawler/resolve-result-went-off-host.js

@@ -0,0 +1,39 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+/**
+ * Determines whether a scrape result's actual destination ended up on a
+ * different host than the URL that was requested.
+ *
+ * Exists for the cross-host-redirect leak guard in `Crawler`'s worker body:
+ * beholder decides `isExternal: false` before navigating and only flips it
+ * to `true` after seeing the destination's hostname, so a same-host source
+ * that redirects cross-host still has its sub-resource / console listeners
+ * attached under the pre-navigation `isExternal: false` for the whole trip
+ * — the caller uses this function's answer to decide whether to discard
+ * what those listeners captured.
+ *
+ * `pageData` (present on `type: 'success'`) carries the authoritative
+ * post-navigation `isExternal` and is used directly. A `type: 'error'`
+ * result has no `pageData`, so `postNavigationUrl` — the puppeteer-side
+ * `page.url()` captured by the JS-redirect rescue (see `BrowserScrapeResult`
+ * JSDoc) — is used as the fallback signal instead. When neither is
+ * available (e.g. the browser context died before `page.url()` could be
+ * read, or the result is `type: 'skipped'`, which carries neither field),
+ * this returns `false` — the alternative would discard legitimate
+ * console/resource data for the overwhelmingly common same-host case on
+ * the mere possibility of a leak this function has no evidence for.
+ * @param result - The non-redirect-edge scrape outcome.
+ * @param url - The originally-requested URL.
+ * @returns `true` when the browser is known to have ended up off-host.
+ * @example
+ * const wentOffHost = resolveResultWentOffHost(result, url);
+ * if (!wentOffHost) {
+ *   handleResources(result.resources, parentSource);
+ * }
+ */
+export function resolveResultWentOffHost(result, url) {
+    if (result.pageData !== undefined) {
+        return result.pageData.isExternal;
+    }
+    return (result.postNavigationUrl !== undefined &&
+        parseUrl(result.postNavigationUrl)?.hostname !== url.hostname);
+}

package/lib/crawler/scan-js-resource-for-license-comment.d.ts

@@ -0,0 +1,32 @@
+import type { TechnologySignalPartial } from '../archive/meta/technologies/types.js';
+/** Options for {@link scanJsResourceForLicenseComment}. */
+export interface ScanJsResourceForLicenseCommentOptions {
+    /** Byte cap on the response body read. Defaults to {@link DEFAULT_BYTE_LIMIT}. */
+    byteLimit?: number;
+    /** Network timeout in milliseconds. Defaults to {@link DEFAULT_TIMEOUT_MS}. */
+    timeout?: number;
+    /** User-Agent header to send. */
+    userAgent?: string;
+}
+/**
+ * Reads up to `byteLimit` bytes from a JS resource's leading bytes and tests
+ * them against {@link TECHNOLOGY_SIGNAL_DEFINITIONS}' `js-license-comment`
+ * patterns (e.g. Vue's leading license-banner comment). Best-effort: any network
+ * failure, timeout, or non-2xx response resolves `null` rather than
+ * throwing — a single unreachable JS resource must not abort the enrichment
+ * pass over the rest of the archive's resources (see
+ * `scanJsResourcesForTechnologySignals`, this function's only caller).
+ *
+ * Not cached and not routed through `destinationCache` (unlike
+ * `fetchDestination`): callers are expected to persist the outcome in
+ * `technology_js_scan_cache`, keyed by `resourceId`, so a resource is never
+ * scanned twice across the archive's lifetime.
+ * @param url - The JS resource's absolute URL.
+ * @param options - Byte cap, timeout, and User-Agent overrides.
+ * @returns The first matching signal, or `null` when nothing matched (or
+ *   the fetch failed).
+ * @example
+ * const signal = await scanJsResourceForLicenseComment('https://example.com/_astro/app.js');
+ * // { technology: 'Vue', signalType: 'js-license-comment', evidence: 'Vue.js license banner text', weight: 55, category: 'JavaScript frameworks' }
+ */
+export declare function scanJsResourceForLicenseComment(url: string, options?: ScanJsResourceForLicenseCommentOptions): Promise<TechnologySignalPartial | null>;