@nitpicker/crawler 0.18.1 → 0.19.0

package/lib/crawler/crawler.js

@@ -1,4 +1,3 @@
-var _a;
 import { existsSync } from 'node:fs';
 import path from 'node:path';
 import Scraper from '@d-zero/beholder';
@@ -14,6 +13,7 @@ import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
 import { buildRedirectEvent } from './build-redirect-event.js';
+import { captureCustomElements } from './capture-custom-elements.js';
 import { captureImageDomPaths } from './capture-image-dom-paths.js';
 import { chooseProbeHost } from './choose-probe-host.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
@@ -92,7 +92,7 @@ const DEFAULT_DEDUPE_MAP_CAP = 100_000;
  * Crawling is performed concurrently using the dealer pattern, with
  * configurable parallelism up to {@link Crawler.MAX_PROCESS_LENGTH}.
  */
-class Crawler extends EventEmitter {
+export default class Crawler extends EventEmitter {
     /** Controller used to cancel the deal-based crawl via its AbortSignal. */
     #abortController = new AbortController();
     /**
@@ -556,10 +556,11 @@ class Crawler extends EventEmitter {
      *   queue, prioritising likely-HTML URLs to the front (see {@link partitionUrlsByHtml}).
      *   Accepts a batch so a group of URLs (e.g. predicted pagination) keeps its order.
      * @param concurrency - Current concurrency level, used to determine predicted URL count
-     * @param precomputedBodyHash - This page's body hash, if the caller already
-     *   computed it (the predicted-content-duplicate check, A-3, computes it for
-     *   every predicted page regardless of `--dedupe-cap`) — reused for the
-     *   dedupe-cap observation below instead of hashing the same html twice.
+     * @param precomputedBodyHash - This page's body hash, computed once by the
+     *   caller for every internal page with a rendered HTML body (see the
+     *   computation site's comment) — reused here for the dedupe-cap
+     *   observation, and forwarded via the `page` event so `update-page.ts`
+     *   does not hash the same html again.
      */
     #handleResult(result, url, enqueue, concurrency, precomputedBodyHash) {
         switch (result.type) {
@@ -595,6 +596,14 @@ class Crawler extends EventEmitter {
                     const shapeKey = computeShapeKey(result.pageData.url.withoutHashAndAuth);
                     const metaSig = computeMetaSignature(result.pageData.meta);
                     if (shapeKey && metaSig) {
+                        // `precomputedBodyHash` is non-null here in every reachable
+                        // case: this branch's guard (`!isExternal && html.length > 0`,
+                        // modulo the `isMetadataOnly` exclusion which only narrows it)
+                        // is a subset of the unconditional computation site's condition
+                        // above. The `??` fallback is not expected to ever fire — it is
+                        // kept only as a defensive backstop against a future edit to
+                        // either condition silently breaking that invariant, favouring
+                        // a slow-but-correct recomputation over a crash.
                         const bodyHash = precomputedBodyHash ?? computeBodyHash(result.pageData.html);
                         const ogUrlMismatch = resolveOgUrlMismatch(result.pageData.meta, result.pageData.url.href);
                         const event = this.#dedupeCapTracker.observe({
@@ -702,6 +711,7 @@ class Crawler extends EventEmitter {
                         void this.emit('page', {
                             result: result.pageData,
                             source: pageSource,
+                            bodyHash: precomputedBodyHash,
                         });
                     }
                 }
@@ -740,7 +750,16 @@ class Crawler extends EventEmitter {
                         });
                     }
                     else {
-                        void this.emit('page', { result: pageResult, source: pageSource });
+                        // `pageResult` here always carries `html: ''` (`linkToPageData`'s
+                        // error-fallback shape), so it never reaches `update-page.ts`'s
+                        // `html.length > 0` write gate — `bodyHash: null` is explicit
+                        // rather than relying on that gate to make an omitted field
+                        // harmless, so this stays correct if that ever changes.
+                        void this.emit('page', {
+                            result: pageResult,
+                            source: pageSource,
+                            bodyHash: null,
+                        });
                     }
                 }
                 void this.emit('error', {
@@ -906,7 +925,7 @@ class Crawler extends EventEmitter {
         }
         const concurrency = this.#options.parallels
             ? Math.max(this.#options.parallels, 1)
-            : _a.MAX_PROCESS_LENGTH;
+            : Crawler.MAX_PROCESS_LENGTH;
         await deal(initialUrls, (url, update, _index, setLineHeader, push, unshift) => {
             const matchedScope = findScopeEntry(url, this.#scope, this.#options);
             const isExternal = matchedScope === null;
@@ -961,10 +980,12 @@ class Crawler extends EventEmitter {
                 const markBrowserScrape = () => {
                     renderedInBrowser = true;
                 };
-                // Set by the predicted-content-duplicate check below (A-3) when it
-                // computes this page's body hash, so `#handleResult`'s dedupe-cap
-                // observation (also gated on this page's html) can reuse it instead
-                // of hashing the same html a second time.
+                // Set below for every internal page with a rendered HTML body
+                // (not just predicted ones — see the computation site's comment),
+                // so both `#handleResult`'s dedupe-cap observation and the `page`
+                // event's `bodyHash` payload (ultimately consumed by
+                // `update-page.ts`'s `page_meta.body_hash` write) reuse this one
+                // value instead of each hashing the same html again.
                 let precomputedBodyHash = null;
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
@@ -1104,30 +1125,41 @@ class Crawler extends EventEmitter {
                         log(c.dim('Predicted (discarded)'));
                         return;
                     }
-                    // Discard a predicted URL whose rendered body is a
-                    // byte-for-byte duplicate of the previous predicted page of the
-                    // same shape, and stop generating further predictions for that
-                    // shape (checked above, in the pagination-pattern branch). This
-                    // is the always-on backstop against a site that returns 2xx for
-                    // any extrapolated token but ignores it entirely (e.g. always
-                    // serving the same "no results" template) — `shouldDiscardPredicted`
-                    // alone cannot see this, since it only inspects HTTP status.
-                    if (isPredicted &&
-                        result.type === 'success' &&
+                    // Compute this page's body hash once, up front, for every
+                    // internal page with a rendered HTML body — not just predicted
+                    // ones. This condition intentionally mirrors `update-page.ts`'s
+                    // `writeHtml && page.html.length > 0` write gate (internal pages
+                    // are exactly the ones `setPage` — as opposed to
+                    // `setExternalPage` — writes a body through), so the value
+                    // computed here can be forwarded through the `page` event all
+                    // the way to that write and reused there instead of hashing the
+                    // same html a second time.
+                    if (result.type === 'success' &&
                         result.pageData &&
+                        !result.pageData.isExternal &&
                         result.pageData.html.length > 0) {
-                        const shapeKey = computeShapeKey(url.withoutHashAndAuth);
-                        if (shapeKey) {
-                            const bodyHash = computeBodyHash(result.pageData.html);
-                            precomputedBodyHash = bodyHash;
-                            const lastBodyHash = this.#predictedShapeBodyHashes.get(shapeKey) ?? null;
-                            if (isPredictedContentDuplicate(bodyHash, lastBodyHash)) {
-                                this.#predictedShapeStopped.add(shapeKey);
-                                handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
-                                log(c.dim('Predicted (content duplicate, discarded)'));
-                                return;
+                        precomputedBodyHash = computeBodyHash(result.pageData.html);
+                        // Discard a predicted URL whose rendered body is a
+                        // byte-for-byte duplicate of the previous predicted page of
+                        // the same shape, and stop generating further predictions for
+                        // that shape (checked above, in the pagination-pattern
+                        // branch). This is the always-on backstop against a site
+                        // that returns 2xx for any extrapolated token but ignores it
+                        // entirely (e.g. always serving the same "no results"
+                        // template) — `shouldDiscardPredicted` alone cannot see
+                        // this, since it only inspects HTTP status.
+                        if (isPredicted) {
+                            const shapeKey = computeShapeKey(url.withoutHashAndAuth);
+                            if (shapeKey) {
+                                const lastBodyHash = this.#predictedShapeBodyHashes.get(shapeKey) ?? null;
+                                if (isPredictedContentDuplicate(precomputedBodyHash, lastBodyHash)) {
+                                    this.#predictedShapeStopped.add(shapeKey);
+                                    handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
+                                    log(c.dim('Predicted (content duplicate, discarded)'));
+                                    return;
+                                }
+                                this.#predictedShapeBodyHashes.set(shapeKey, precomputedBodyHash);
                             }
-                            this.#predictedShapeBodyHashes.set(shapeKey, bodyHash);
                         }
                     }
                     // Count only after discard check: rendered HTML pages that
@@ -1875,17 +1907,42 @@ class Crawler extends EventEmitter {
             // `image_items.dom_path_text_id` resolution at write time; a
             // capture failure (or a page with no images) falls back to the
             // synthetic `unknown/<n>` markers, so this stays best-effort.
-            if (result.type === 'success' &&
-                result.pageData &&
-                result.pageData.imageList.length > 0) {
-                const imageDomPaths = await captureImageDomPaths(page);
-                if (imageDomPaths !== undefined) {
-                    const withDomPaths = {
-                        ...result.pageData,
-                        imageDomPaths,
-                    };
-                    result.pageData = withDomPaths;
-                }
+            // Image dom-path capture and custom-element (Web Component)
+            // capture are two independent `page.evaluate()` round-trips over
+            // the same still-alive `page` — neither reads the other's
+            // output, so they run concurrently instead of paying both
+            // round-trips' latency in sequence. Each keeps its own
+            // pre-existing gate (image dom-paths on a non-empty imageList,
+            // custom elements on a resolved main-content region — unrelated
+            // to each other, so neither gate is relaxed by running together).
+            // Custom-element capture exists because beholder's
+            // `MainContentsData` has no `customElements` category, so
+            // nitpicker captures it itself.
+            const shouldCaptureImageDomPaths = result.type === 'success' &&
+                result.pageData !== undefined &&
+                result.pageData.imageList.length > 0;
+            const shouldCaptureCustomElements = result.type === 'success' &&
+                result.pageData !== undefined &&
+                result.pageData.mainContents !== null;
+            const [imageDomPaths, mainContentCustomElements] = await Promise.all([
+                shouldCaptureImageDomPaths ? captureImageDomPaths(page) : undefined,
+                shouldCaptureCustomElements
+                    ? captureCustomElements(page, this.#options.mainContentSelector)
+                    : undefined,
+            ]);
+            if (imageDomPaths !== undefined && result.pageData) {
+                const withDomPaths = {
+                    ...result.pageData,
+                    imageDomPaths,
+                };
+                result.pageData = withDomPaths;
+            }
+            if (mainContentCustomElements !== undefined && result.pageData) {
+                const withCustomElements = {
+                    ...result.pageData,
+                    mainContentCustomElements,
+                };
+                result.pageData = withCustomElements;
             }
             update('Closing browser%dots%');
             // JS-redirect rescue capture: when `scrapeStart` catches a
@@ -1970,8 +2027,6 @@ class Crawler extends EventEmitter {
      */
     static MAX_PROCESS_LENGTH = 10;
 }
-_a = Crawler;
-export default Crawler;
 /**
  * Colorize an HTTP status code string for terminal display.
  *

package/lib/crawler/fetch-destination.js

@@ -1,4 +1,5 @@
 import { delay } from '@d-zero/shared/delay';
+import { raceWithTimeout } from '@d-zero/shared/race-with-timeout';
 import redirects from 'follow-redirects';
 import { decodeAuthCredential } from './decode-auth-credential.js';
 import { destinationCache } from './destination-cache.js';
@@ -34,19 +35,11 @@ export async function fetchDestination(params) {
     }
     const effectiveMethod = titleBytesLimit == null ? method : 'GET';
     const raceTimeoutMs = timeout ?? DEFAULT_HEAD_TIMEOUT_MS;
-    // Race the fetch against the requested timeout. The losing timer is cleared
-    // explicitly so it never keeps the event loop alive after the race settles
-    // (a plain `delay()` in `Promise.race` would leak the timer until it fires).
-    let timeoutHandle;
-    const result = await Promise.race([
-        _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent, timeout).catch((error) => error instanceof Error ? error : new Error(String(error))),
-        new Promise((resolve) => {
-            timeoutHandle = setTimeout(() => resolve(new NetTimeoutError(url.href)), raceTimeoutMs);
-        }),
-    ]).finally(() => {
-        if (timeoutHandle)
-            clearTimeout(timeoutHandle);
-    });
+    // Race the fetch against the requested timeout via `raceWithTimeout`, which
+    // clears the losing timer internally so it never keeps the event loop
+    // alive after the race settles.
+    const { result: challengeResult, timeout: timedOut } = await raceWithTimeout(() => _fetchHead(url, isExternal, effectiveMethod, titleBytesLimit, userAgent, timeout).catch((error) => error instanceof Error ? error : new Error(String(error))), raceTimeoutMs);
+    const result = timedOut ? new NetTimeoutError(url.href) : challengeResult;
     // HEAD failure fallback: a WAF / middlebox that silently drops HEAD will
     // surface as NetTimeoutError / parse-error / connection-reset here even
     // though the same URL serves a normal GET response. Try GET once (using

package/lib/crawler/find-package-dir.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Locates `packageName`'s directory the way Node's module resolution would —
+ * walking `node_modules` upward from `fromDir` — without going through the
+ * package's own `exports` map.
+ *
+ * A plain `require.resolve()` on an ESM-only package (`exports` declaring
+ * only the `"import"` condition) fails from a CJS `createRequire` context,
+ * and there is no stable public API to run Node's ESM resolution algorithm
+ * from an arbitrary base directory. Walking `node_modules` directly
+ * sidesteps the `exports` map entirely — it only needs the package's
+ * directory, not its resolved entry file.
+ * @param fromDir - Directory to start walking upward from.
+ * @param packageName - Package name to locate (e.g. `puppeteer`,
+ *   `@d-zero/beholder`).
+ * @throws {Error} When no `node_modules/<packageName>` directory is found
+ *   between `fromDir` and the filesystem root.
+ * @example
+ * ```ts
+ * import { findPackageDir } from './find-package-dir.js';
+ *
+ * const puppeteerDir = findPackageDir(import.meta.dirname, 'puppeteer');
+ * ```
+ */
+export declare function findPackageDir(fromDir: string, packageName: string): string;

package/lib/crawler/find-package-dir.js

@@ -0,0 +1,39 @@
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+/**
+ * Locates `packageName`'s directory the way Node's module resolution would —
+ * walking `node_modules` upward from `fromDir` — without going through the
+ * package's own `exports` map.
+ *
+ * A plain `require.resolve()` on an ESM-only package (`exports` declaring
+ * only the `"import"` condition) fails from a CJS `createRequire` context,
+ * and there is no stable public API to run Node's ESM resolution algorithm
+ * from an arbitrary base directory. Walking `node_modules` directly
+ * sidesteps the `exports` map entirely — it only needs the package's
+ * directory, not its resolved entry file.
+ * @param fromDir - Directory to start walking upward from.
+ * @param packageName - Package name to locate (e.g. `puppeteer`,
+ *   `@d-zero/beholder`).
+ * @throws {Error} When no `node_modules/<packageName>` directory is found
+ *   between `fromDir` and the filesystem root.
+ * @example
+ * ```ts
+ * import { findPackageDir } from './find-package-dir.js';
+ *
+ * const puppeteerDir = findPackageDir(import.meta.dirname, 'puppeteer');
+ * ```
+ */
+export function findPackageDir(fromDir, packageName) {
+    let dir = fromDir;
+    for (;;) {
+        const candidate = path.join(dir, 'node_modules', packageName);
+        if (existsSync(path.join(candidate, 'package.json'))) {
+            return candidate;
+        }
+        const parent = path.dirname(dir);
+        if (parent === dir) {
+            throw new Error(`Could not locate "${packageName}" from ${fromDir}`);
+        }
+        dir = parent;
+    }
+}

package/lib/crawler/scan-js-resource-for-license-comment.d.ts

@@ -0,0 +1,32 @@
+import type { TechnologySignalPartial } from '../archive/meta/technologies/types.js';
+/** Options for {@link scanJsResourceForLicenseComment}. */
+export interface ScanJsResourceForLicenseCommentOptions {
+    /** Byte cap on the response body read. Defaults to {@link DEFAULT_BYTE_LIMIT}. */
+    byteLimit?: number;
+    /** Network timeout in milliseconds. Defaults to {@link DEFAULT_TIMEOUT_MS}. */
+    timeout?: number;
+    /** User-Agent header to send. */
+    userAgent?: string;
+}
+/**
+ * Reads up to `byteLimit` bytes from a JS resource's leading bytes and tests
+ * them against {@link TECHNOLOGY_SIGNAL_DEFINITIONS}' `js-license-comment`
+ * patterns (e.g. Vue's leading license-banner comment). Best-effort: any network
+ * failure, timeout, or non-2xx response resolves `null` rather than
+ * throwing — a single unreachable JS resource must not abort the enrichment
+ * pass over the rest of the archive's resources (see
+ * `scanJsResourcesForTechnologySignals`, this function's only caller).
+ *
+ * Not cached and not routed through `destinationCache` (unlike
+ * `fetchDestination`): callers are expected to persist the outcome in
+ * `technology_js_scan_cache`, keyed by `resourceId`, so a resource is never
+ * scanned twice across the archive's lifetime.
+ * @param url - The JS resource's absolute URL.
+ * @param options - Byte cap, timeout, and User-Agent overrides.
+ * @returns The first matching signal, or `null` when nothing matched (or
+ *   the fetch failed).
+ * @example
+ * const signal = await scanJsResourceForLicenseComment('https://example.com/_astro/app.js');
+ * // { technology: 'Vue', signalType: 'js-license-comment', evidence: 'Vue.js license banner text', weight: 55, category: 'JavaScript frameworks' }
+ */
+export declare function scanJsResourceForLicenseComment(url: string, options?: ScanJsResourceForLicenseCommentOptions): Promise<TechnologySignalPartial | null>;

package/lib/crawler/scan-js-resource-for-license-comment.js

@@ -0,0 +1,140 @@
+import redirects from 'follow-redirects';
+import { TECHNOLOGY_SIGNAL_DEFINITIONS } from '../archive/meta/technologies/technology-signal-definitions.js';
+/** Default cap on bytes read from a JS resource before giving up on a match. */
+const DEFAULT_BYTE_LIMIT = 8192;
+/** Default network timeout, shorter than `fetchDestination`'s HEAD budget — this is a best-effort enrichment pass, not the crawl's critical path. */
+const DEFAULT_TIMEOUT_MS = 8000;
+const EVIDENCE_MAX_LENGTH = 200;
+/** Definitions this scan tests against — only the `js-license-comment` signal type applies to JS resource bodies (the rest match HTML). */
+const JS_LICENSE_COMMENT_DEFINITIONS = TECHNOLOGY_SIGNAL_DEFINITIONS.filter((def) => def.signalType === 'js-license-comment');
+/**
+ * Reads up to `byteLimit` bytes from a JS resource's leading bytes and tests
+ * them against {@link TECHNOLOGY_SIGNAL_DEFINITIONS}' `js-license-comment`
+ * patterns (e.g. Vue's leading license-banner comment). Best-effort: any network
+ * failure, timeout, or non-2xx response resolves `null` rather than
+ * throwing — a single unreachable JS resource must not abort the enrichment
+ * pass over the rest of the archive's resources (see
+ * `scanJsResourcesForTechnologySignals`, this function's only caller).
+ *
+ * Not cached and not routed through `destinationCache` (unlike
+ * `fetchDestination`): callers are expected to persist the outcome in
+ * `technology_js_scan_cache`, keyed by `resourceId`, so a resource is never
+ * scanned twice across the archive's lifetime.
+ * @param url - The JS resource's absolute URL.
+ * @param options - Byte cap, timeout, and User-Agent overrides.
+ * @returns The first matching signal, or `null` when nothing matched (or
+ *   the fetch failed).
+ * @example
+ * const signal = await scanJsResourceForLicenseComment('https://example.com/_astro/app.js');
+ * // { technology: 'Vue', signalType: 'js-license-comment', evidence: 'Vue.js license banner text', weight: 55, category: 'JavaScript frameworks' }
+ */
+export async function scanJsResourceForLicenseComment(url, options = {}) {
+    const byteLimit = options.byteLimit ?? DEFAULT_BYTE_LIMIT;
+    const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
+    const body = await readLeadingBytes(url, byteLimit, timeout, options.userAgent).catch(() => null);
+    if (body == null)
+        return null;
+    for (const def of JS_LICENSE_COMMENT_DEFINITIONS) {
+        const match = def.pattern.exec(body);
+        if (match) {
+            return {
+                technology: def.technology,
+                signalType: def.signalType,
+                evidence: match[0].slice(0, EVIDENCE_MAX_LENGTH),
+                weight: def.weight,
+                category: def.category ?? null,
+            };
+        }
+    }
+    return null;
+}
+/**
+ * Streams up to `byteLimit` bytes of a GET response body, destroying the
+ * connection once the cap is reached (or the stream ends first). Mirrors
+ * `fetchDestination`'s `titleBytesLimit` byte-cap technique (same
+ * `follow-redirects` + manual chunk accumulation + `req.destroy()` shape),
+ * but generic over "leading bytes as text" rather than title extraction —
+ * this module has no `PageData` to build.
+ * @param url - The absolute URL to fetch.
+ * @param byteLimit - Byte cap on the accumulated body.
+ * @param timeout - Milliseconds before the request is aborted.
+ * @param userAgent - Optional User-Agent header.
+ * @returns The accumulated body text, or `null` on a non-2xx response.
+ */
+async function readLeadingBytes(url, byteLimit, timeout, userAgent) {
+    return new Promise((resolve, reject) => {
+        const parsed = new URL(url);
+        const request = {
+            protocol: parsed.protocol,
+            hostname: parsed.hostname,
+            port: parsed.port || undefined,
+            path: `${parsed.pathname}${parsed.search}`,
+            method: 'GET',
+            trackRedirects: true,
+            headers: {
+                host: parsed.host,
+                ...(userAgent ? { 'User-Agent': userAgent } : {}),
+                Accept: '*/*',
+                'Accept-Encoding': 'identity',
+            },
+        };
+        let req;
+        let destroyed = false;
+        let settled = false;
+        const timeoutHandle = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            destroyed = true;
+            req.destroy();
+            resolve(null);
+        }, timeout);
+        const response = (res) => {
+            const status = res.statusCode ?? 0;
+            if (status < 200 || status >= 300) {
+                settled = true;
+                clearTimeout(timeoutHandle);
+                destroyed = true;
+                req.destroy();
+                resolve(null);
+                return;
+            }
+            const chunks = [];
+            let totalBytes = 0;
+            res.on('data', (chunk) => {
+                if (settled)
+                    return;
+                chunks.push(chunk);
+                totalBytes += chunk.length;
+                if (totalBytes >= byteLimit) {
+                    settled = true;
+                    clearTimeout(timeoutHandle);
+                    destroyed = true;
+                    req.destroy();
+                    resolve(Buffer.concat(chunks).toString('utf8'));
+                }
+            });
+            res.on('end', () => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timeoutHandle);
+                resolve(Buffer.concat(chunks).toString('utf8'));
+            });
+        };
+        if (parsed.protocol === 'https:') {
+            req = redirects.https.request(request, response);
+        }
+        else {
+            req = redirects.http.request(request, response);
+        }
+        req.on('error', (error) => {
+            clearTimeout(timeoutHandle);
+            // Ignore errors caused by our own intentional req.destroy() above.
+            if (destroyed)
+                return;
+            reject(error);
+        });
+        req.end();
+    });
+}

package/lib/crawler/scan-js-resources-for-technology-signals.d.ts

@@ -0,0 +1,58 @@
+import type { ArchiveAccessor } from '../archive/archive-accessor.js';
+/** Options for {@link scanJsResourcesForTechnologySignals}. */
+export interface ScanJsResourcesForTechnologySignalsOptions {
+    /** Maximum concurrent network fetches. Defaults to {@link DEFAULT_CONCURRENCY}. */
+    concurrency?: number;
+    /** Forwarded to `scanJsResourceForLicenseComment`. */
+    byteLimit?: number;
+    /** Forwarded to `scanJsResourceForLicenseComment`. */
+    timeout?: number;
+    /** Forwarded to `scanJsResourceForLicenseComment`. */
+    userAgent?: string;
+    /** Called after each resource finishes scanning (matched or not). */
+    onProgress?: (done: number, total: number) => void;
+}
+/** Outcome counters returned by {@link scanJsResourcesForTechnologySignals}. */
+export interface ScanJsResourcesForTechnologySignalsResult {
+    /** JS resources eligible for scanning (internal, not yet cached). */
+    candidateCount: number;
+    /** Resources actually scanned this run (equals `candidateCount` barring a mid-run crash). */
+    scannedCount: number;
+    /** Resources whose leading bytes matched a known license comment. */
+    matchedCount: number;
+    /** Distinct pages whose `technology_signals` / `page_technologies` were updated as a result. */
+    pagesUpdatedCount: number;
+}
+/**
+ * Post-crawl network enrichment (distinct from crawl-time extraction and
+ * from read-model-time backfill — see ARCHITECTURE.md): re-fetches the
+ * leading bytes of every not-yet-scanned internal JS resource, tests them
+ * for a known technology's license comment, and folds any match into the
+ * referencing pages' `technology_signals` / `page_technologies`.
+ *
+ * Each resource is scanned at most once ever, across the archive's whole
+ * lifetime — outcomes (including non-matches) are recorded in
+ * `technology_js_scan_cache` keyed by `resourceId`, so a later
+ * `--append`/`--retry-failed` run only pays the network cost for resources
+ * discovered since the last run.
+ *
+ * A single resource can be referenced by many pages (a shared bundle); a
+ * match is applied to every one of them independently. Network scanning
+ * runs at bounded concurrency; the per-page DB recombination that follows
+ * runs after every scan has settled, never concurrently for the same page,
+ * so two resources that both resolve to the same page cannot race each
+ * other's read-modify-write.
+ *
+ * Best-effort like `scanJsResourceForLicenseComment`: an unreachable
+ * resource is recorded as a non-match (cached as scanned, `technology:
+ * null`) rather than retried or surfaced as an error — a flaky CDN must not
+ * block the rest of the archive's enrichment, and the resource will not be
+ * retried until `technology_js_scan_cache` itself is cleared.
+ * @param accessor - The archive to enrich.
+ * @param options - Concurrency, byte-cap, timeout, and progress overrides.
+ * @returns Counters describing what was scanned, matched, and updated.
+ * @example
+ * const result = await scanJsResourcesForTechnologySignals(archive);
+ * // { candidateCount: 42, scannedCount: 42, matchedCount: 3, pagesUpdatedCount: 57 }
+ */
+export declare function scanJsResourcesForTechnologySignals(accessor: ArchiveAccessor, options?: ScanJsResourcesForTechnologySignalsOptions): Promise<ScanJsResourcesForTechnologySignalsResult>;