@nitpicker/crawler 0.15.0 → 0.16.0

package/lib/crawler/assert-chrome-installed.js

@@ -0,0 +1,43 @@
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import pkg from '../../package.json' with { type: 'json' };
+/**
+ * Verifies that Puppeteer can resolve an installed Chrome/Chromium executable
+ * before a crawl starts.
+ *
+ * A crawl otherwise only discovers a missing browser deep inside the
+ * per-URL scrape loop (`Crawler#_launchBrowserAndScrape`), where it surfaces
+ * as one more scrape error among many — the CLI still prints "Crawl
+ * completed" and writes an archive, so a missing Chrome (a fatal
+ * precondition, not a per-page failure) is easy to miss. Calling this once,
+ * before any archive I/O begins, turns it into an immediate, actionable
+ * failure instead.
+ * @param executablePath - Explicit override, matching
+ *   {@link CrawlerOptions.executablePath}. Pass `null` (or omit) to check
+ *   Puppeteer's own pinned Chrome resolution instead.
+ * @throws {Error} When the resolved executable path does not exist on disk.
+ * @example
+ * ```ts
+ * import { assertChromeIsInstalled } from '@nitpicker/crawler';
+ *
+ * // Throws with install instructions before any crawl work starts.
+ * await assertChromeIsInstalled();
+ * ```
+ */
+export async function assertChromeIsInstalled(executablePath) {
+    if (executablePath) {
+        const execPath = path.resolve(executablePath);
+        if (existsSync(execPath)) {
+            return;
+        }
+        throw new Error(`Executable path does not exist: ${execPath}`);
+    }
+    const puppeteer = await import('puppeteer');
+    const resolvedPath = await puppeteer.executablePath();
+    if (existsSync(resolvedPath)) {
+        return;
+    }
+    const puppeteerVersion = pkg.dependencies.puppeteer;
+    throw new Error(`Chrome executable not found at: ${resolvedPath}\n` +
+        `Run \`npx puppeteer@${puppeteerVersion} browsers install chrome\` to install the Chrome build Puppeteer expects, then retry.`);
+}

package/lib/crawler/crawler.d.ts

@@ -37,6 +37,18 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      * {@link #runDeal}.
      */
     abort(): void;
+    /**
+     * Per-shape count of anchors the dedupe-cap enqueue gates rejected after
+     * that shape capped (opt-in `--dedupe-cap`). Read by
+     * `CrawlerOrchestrator` at `crawlEnd` to finalize each
+     * `dedupe_cap_events.rejected_count` exactly once — rejections are
+     * accumulated in memory rather than written to the archive per-rejection
+     * to avoid write amplification (a capped trap can generate an unbounded
+     * number of rejected anchors).
+     * @returns A snapshot of the per-shape rejection counts. Empty when
+     *   `--dedupe-cap` was not enabled or no shape has capped yet.
+     */
+    getDedupeCapRejections(): ReadonlyMap<string, number>;
     /**
      * Retrieve the list of Chromium process IDs that are still running.
      *

package/lib/crawler/crawler.js

@@ -9,6 +9,7 @@ import { retryCall } from '@d-zero/shared/retry';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import c from 'ansi-colors';
 import pkg from '../../package.json' with { type: 'json' };
+import { computeBodyHash } from '../archive/body-hash/compute-body-hash.js';
 import { classifyErrorKind } from '../classify-error-kind.js';
 import { crawlerLog } from '../debug.js';
 import { buildJsRedirectEdge } from './build-js-redirect-edge.js';
@@ -16,6 +17,12 @@ import { buildRedirectEvent } from './build-redirect-event.js';
 import { captureImageDomPaths } from './capture-image-dom-paths.js';
 import { chooseProbeHost } from './choose-probe-host.js';
 import { createChangePhaseHandler } from './create-change-phase-handler.js';
+import { decodeAuthCredential } from './decode-auth-credential.js';
+import { computeMetaSignature } from './dedupe/compute-meta-signature.js';
+import { computeShapeKey } from './dedupe/compute-shape-key.js';
+import DedupeCapTracker from './dedupe/dedupe-cap-tracker.js';
+import { isPredictedContentDuplicate } from './dedupe/is-predicted-content-duplicate.js';
+import { resolveOgUrlMismatch } from './dedupe/resolve-og-url-mismatch.js';
 import { derivePageSource } from './derive-page-source.js';
 import { destinationCache } from './destination-cache.js';
 import { detectPaginationPattern } from './detect-pagination-pattern.js';
@@ -72,6 +79,8 @@ const DEFAULT_NETWORK_OUTAGE_ERROR_THRESHOLD = 5;
 const DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD = 2;
 /** Default {@link CrawlerOptions.networkOutageProbeIntervalMs}. */
 const DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS = 10_000;
+/** Default {@link CrawlerOptions.dedupeMapCap}. */
+const DEFAULT_DEDUPE_MAP_CAP = 100_000;
 /**
  * The core crawler engine that discovers and scrapes web pages.
  *
@@ -86,6 +95,24 @@ const DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS = 10_000;
 class Crawler extends EventEmitter {
     /** Controller used to cancel the deal-based crawl via its AbortSignal. */
     #abortController = new AbortController();
+    /**
+     * Per-shape count of anchors rejected by the dedupe-cap enqueue gates
+     * after that shape capped. Read by {@link getDedupeCapRejections} at
+     * `crawlEnd` so the orchestrator can finalize each
+     * `dedupe_cap_events.rejected_count` exactly once (see
+     * `Crawler#getDedupeCapRejections`'s JSDoc for why this is not written
+     * to the archive incrementally).
+     */
+    #dedupeCapRejectionCounts = new Map();
+    /**
+     * Opt-in (`--dedupe-cap`) same-cluster soft cap. Always constructed
+     * (Misra-Gries state stays empty when {@link CrawlerOptions.dedupeCap} is
+     * `null`), gated on by `#options.dedupeCap !== null` at each call site
+     * rather than being conditionally `undefined`, so the two enqueue gates
+     * and the observation call in {@link #handleResult} do not need to
+     * null-check a class field.
+     */
+    #dedupeCapTracker;
     /** Tracks discovered URLs, their scrape status, and deduplication. */
     #linkList = new LinkList();
     /**
@@ -129,6 +156,23 @@ class Crawler extends EventEmitter {
      * `insertPageError`, so the FK resolution via URL always finds the row.
      */
     #pendingPhaseErrors = new Map();
+    /**
+     * Predicted-pagination body-hash tracking (always-on — independent of
+     * the opt-in `--dedupe-cap` tracker). Maps a URL shape key
+     * ({@link computeShapeKey}) to the {@link computeBodyHash} of the most
+     * recently scraped *predicted* page of that shape. Never reset mid-crawl
+     * (persists for the whole session, like {@link #scrapedDestinations}).
+     */
+    #predictedShapeBodyHashes = new Map();
+    /**
+     * Shapes for which {@link #predictedShapeBodyHashes} detected a
+     * content-duplicate predicted page (see {@link isPredictedContentDuplicate}).
+     * Once a shape lands here, no further predicted URLs are generated for it
+     * (checked in {@link #handleResult}'s pagination-pattern branch) — the
+     * cheapest possible way to stop a self-generating trap without needing
+     * the opt-in dedupe-cap machinery.
+     */
+    #predictedShapeStopped = new Set();
     /** Set of resource URLs (without hash) already captured, for deduplication. */
     #resources = new Set();
     /** Number of HTML pages (isTarget=1) scraped in previous sessions, used to seed the progress counter on resume. */
@@ -205,12 +249,16 @@ class Crawler extends EventEmitter {
             networkOutageHostThreshold: options?.networkOutageHostThreshold ?? DEFAULT_NETWORK_OUTAGE_HOST_THRESHOLD,
             networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs ?? DEFAULT_NETWORK_OUTAGE_PROBE_INTERVAL_MS,
             networkProbe: options?.networkProbe ?? null,
+            dedupeCap: options?.dedupeCap ?? null,
+            dedupeMapCap: options?.dedupeMapCap ?? DEFAULT_DEDUPE_MAP_CAP,
+            preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
         };
         this.#networkOutageDetector = new NetworkOutageDetector({
             windowMs: this.#options.networkOutageWindowMs,
             errorThreshold: this.#options.networkOutageErrorThreshold,
             hostThreshold: this.#options.networkOutageHostThreshold,
         });
+        this.#dedupeCapTracker = new DedupeCapTracker({ cap: this.#options.dedupeCap ?? 0, mapCap: this.#options.dedupeMapCap }, this.#options.preloadedStickyShapeKeys);
         this.#robotsChecker = new RobotsChecker(this.#options.userAgent, !this.#options.ignoreRobots);
         for (const urlStr of this.#options.roots) {
             const url = parseUrl(urlStr, this.#options);
@@ -231,6 +279,20 @@ class Crawler extends EventEmitter {
     abort() {
         this.#abortController.abort();
     }
+    /**
+     * Per-shape count of anchors the dedupe-cap enqueue gates rejected after
+     * that shape capped (opt-in `--dedupe-cap`). Read by
+     * `CrawlerOrchestrator` at `crawlEnd` to finalize each
+     * `dedupe_cap_events.rejected_count` exactly once — rejections are
+     * accumulated in memory rather than written to the archive per-rejection
+     * to avoid write amplification (a capped trap can generate an unbounded
+     * number of rejected anchors).
+     * @returns A snapshot of the per-shape rejection counts. Empty when
+     *   `--dedupe-cap` was not enabled or no shape has capped yet.
+     */
+    getDedupeCapRejections() {
+        return this.#dedupeCapRejectionCounts;
+    }
     /**
      * Retrieve the list of Chromium process IDs that are still running.
      *
@@ -493,21 +555,92 @@ class Crawler extends EventEmitter {
      * @param enqueue - Callback to enqueue newly discovered URLs into the dealer
      *   queue, prioritising likely-HTML URLs to the front (see {@link partitionUrlsByHtml}).
      *   Accepts a batch so a group of URLs (e.g. predicted pagination) keeps its order.
-     * @param paginationState - Mutable state for predicted pagination cascade prevention
-     * @param paginationState.lastPushedUrl
-     * @param paginationState.lastPushedWasPredicted
      * @param concurrency - Current concurrency level, used to determine predicted URL count
+     * @param precomputedBodyHash - This page's body hash, if the caller already
+     *   computed it (the predicted-content-duplicate check, A-3, computes it for
+     *   every predicted page regardless of `--dedupe-cap`) — reused for the
+     *   dedupe-cap observation below instead of hashing the same html twice.
      */
-    #handleResult(result, url, enqueue, paginationState, concurrency) {
+    #handleResult(result, url, enqueue, concurrency, precomputedBodyHash) {
         switch (result.type) {
             case 'success': {
                 if (!result.pageData)
                     break;
+                // Scoped to this one page's anchor list (fresh per `#handleResult`
+                // call, not shared across pages): pagination-pattern detection
+                // compares consecutive anchors as they are discovered by
+                // `processAnchors`'s single synchronous loop below, so "consecutive"
+                // must mean "adjacent in this document", not "adjacent in whatever
+                // order the crawl's workers happened to finish". Sharing this state
+                // across pages/workers let `step` be computed from two unrelated
+                // URLs, compounding across rounds until a `/news/date/{year}/`
+                // pager's predicted token overflowed into scientific notation
+                // (`1.7715854126052197e+120`, observed in production).
+                const paginationState = {
+                    lastPushedUrl: null,
+                    lastPushedWasPredicted: false,
+                };
+                // Feed this page's own signature into the same-cluster tracker
+                // (opt-in via `--dedupe-cap`). This is deliberately separate
+                // from the enqueue gates below: gating decides whether to
+                // admit a not-yet-scraped anchor based on shape alone; this
+                // observes the page that was JUST scraped, using its actual
+                // meta/body content. External and metadata-only pages carry no
+                // useful signal for this feature and are skipped, matching the
+                // signature-scope exclusions in `computeMetaSignature`'s design.
+                if (this.#options.dedupeCap !== null &&
+                    !result.pageData.isExternal &&
+                    !this.#linkList.isMetadataOnly(result.pageData.url.withoutHash) &&
+                    result.pageData.html.length > 0) {
+                    const shapeKey = computeShapeKey(result.pageData.url.withoutHashAndAuth);
+                    const metaSig = computeMetaSignature(result.pageData.meta);
+                    if (shapeKey && metaSig) {
+                        const bodyHash = precomputedBodyHash ?? computeBodyHash(result.pageData.html);
+                        const ogUrlMismatch = resolveOgUrlMismatch(result.pageData.meta, result.pageData.url.href);
+                        const event = this.#dedupeCapTracker.observe({
+                            shapeKey,
+                            metaSig,
+                            bodyHash,
+                            ogUrlMismatch,
+                            url: result.pageData.url.href,
+                        });
+                        if (event) {
+                            void this.emit('dedupeCap', event);
+                        }
+                    }
+                }
                 handleScrapeEnd(result.pageData, this.#linkList, this.#scope, this.#options, (newUrl, opts) => {
+                    // Gate 1: blocks real anchors discovered on this page whose
+                    // shape is already confirmed as a trap. This does NOT cover
+                    // predicted URLs — `generatePredictedUrls`'s output is
+                    // pushed directly below (`this.#linkList.add(specUrl, ...)`),
+                    // bypassing this closure entirely — so the predicted-URL
+                    // generation site below has its own equivalent check
+                    // (`shapeIsStopped`, combined with `#predictedShapeStopped`).
+                    // External anchors are out of scope for the cap (issue #208:
+                    // "cap 適用は internal only"), enforced by the scope check
+                    // below. Deliberately NOT also excluding `opts?.metadataOnly`
+                    // (unlike the tracker's observation side, which does skip
+                    // metadata-only pages — they carry no reliable signature): with
+                    // `--recursive=false`, `handle-scrape-end.ts` marks EVERY anchor
+                    // metadata-only, internal or not, so excluding them here would
+                    // silently disable `--dedupe-cap` for anchor discovery whenever
+                    // `--recursive=false` is set — while gate 2 (the JS-redirect
+                    // direct enqueue below) has no such exclusion and would still
+                    // cap the very same shape, an inconsistency between the two
+                    // discovery paths.
+                    if (this.#options.dedupeCap !== null &&
+                        findScopeEntry(newUrl, this.#scope, this.#options) !== null) {
+                        const gateShapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
+                        if (gateShapeKey && this.#dedupeCapTracker.isCapped(gateShapeKey)) {
+                            this.#recordDedupeCapRejection(gateShapeKey);
+                            return;
+                        }
+                    }
                     this.#linkList.add(newUrl, opts);
                     void enqueue(newUrl);
                     // Predicted pagination detection
-                    if (!paginationState || !concurrency)
+                    if (!concurrency)
                         return;
                     // metadataOnly / external: update tracking but skip pattern detection
                     if (opts?.metadataOnly ||
@@ -521,22 +654,37 @@ class Crawler extends EventEmitter {
                         !paginationState.lastPushedWasPredicted) {
                         const pattern = detectPaginationPattern(paginationState.lastPushedUrl, newUrl.withoutHashAndAuth);
                         if (pattern) {
-                            const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
-                            const specUrls = [];
-                            for (const specUrlStr of urls) {
-                                const specUrl = parseUrl(specUrlStr, this.#options);
-                                if (specUrl) {
-                                    this.#linkList.add(specUrl, { predicted: true });
-                                    specUrls.push(specUrl);
+                            // Stop generating further predicted URLs for this shape
+                            // once EITHER confirmation mechanism has fired — the
+                            // always-on content-duplication check
+                            // (`#predictedShapeStopped`), or the opt-in
+                            // `--dedupe-cap` tracker (`#dedupeCapTracker.isCapped`,
+                            // only consulted when the flag is set). Falls through to
+                            // the plain (non-predicted) bookkeeping below instead of
+                            // returning, since the anchor itself is still real.
+                            const shapeKey = computeShapeKey(newUrl.withoutHashAndAuth);
+                            const shapeIsStopped = shapeKey !== null &&
+                                (this.#predictedShapeStopped.has(shapeKey) ||
+                                    (this.#options.dedupeCap !== null &&
+                                        this.#dedupeCapTracker.isCapped(shapeKey)));
+                            if (!shapeIsStopped) {
+                                const urls = generatePredictedUrls(pattern, newUrl.withoutHashAndAuth, concurrency);
+                                const specUrls = [];
+                                for (const specUrlStr of urls) {
+                                    const specUrl = parseUrl(specUrlStr, this.#options);
+                                    if (specUrl) {
+                                        this.#linkList.add(specUrl, { predicted: true });
+                                        specUrls.push(specUrl);
+                                    }
                                 }
+                                // Enqueue as one batch so ascending page order is kept
+                                // at the front of the queue (see enqueue in #runDeal).
+                                if (specUrls.length > 0)
+                                    void enqueue(...specUrls);
+                                paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
+                                paginationState.lastPushedWasPredicted = true;
+                                return;
                             }
-                            // Enqueue as one batch so ascending page order is kept
-                            // at the front of the queue (see enqueue in #runDeal).
-                            if (specUrls.length > 0)
-                                void enqueue(...specUrls);
-                            paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
-                            paginationState.lastPushedWasPredicted = true;
-                            return;
                         }
                     }
                     paginationState.lastPushedUrl = newUrl.withoutHashAndAuth;
@@ -630,6 +778,19 @@ class Crawler extends EventEmitter {
             window: { startedAt, endedAt },
         });
     }
+    /**
+     * Increments {@link #dedupeCapRejectionCounts} for one shape. Scoped to
+     * the two concrete enqueue-time rejections (a real anchor or a
+     * JS-redirect destination that was discovered but blocked) — it does
+     * NOT count predicted URLs that were never generated at all because
+     * their shape was already stopped (see the `shapeIsStopped` check in
+     * {@link #handleResult}), since nothing concrete existed there to
+     * reject.
+     * @param shapeKey - The capped shape a rejection is being recorded for.
+     */
+    #recordDedupeCapRejection(shapeKey) {
+        this.#dedupeCapRejectionCounts.set(shapeKey, (this.#dedupeCapRejectionCounts.get(shapeKey) ?? 0) + 1);
+    }
     /**
      * Feed one observed network-layer error into
      * {@link #networkOutageDetector} and hand off to
@@ -746,11 +907,6 @@ class Crawler extends EventEmitter {
         const concurrency = this.#options.parallels
             ? Math.max(this.#options.parallels, 1)
             : _a.MAX_PROCESS_LENGTH;
-        // Predicted pagination state
-        const paginationState = {
-            lastPushedUrl: null,
-            lastPushedWasPredicted: false,
-        };
         await deal(initialUrls, (url, update, _index, setLineHeader, push, unshift) => {
             const matchedScope = findScopeEntry(url, this.#scope, this.#options);
             const isExternal = matchedScope === null;
@@ -805,6 +961,11 @@ class Crawler extends EventEmitter {
                 const markBrowserScrape = () => {
                     renderedInBrowser = true;
                 };
+                // Set by the predicted-content-duplicate check below (A-3) when it
+                // computes this page's body hash, so `#handleResult`'s dedupe-cap
+                // observation (also gated on this page's html) can reuse it instead
+                // of hashing the same html a second time.
+                let precomputedBodyHash = null;
                 try {
                     const robotsAllowed = await this.#robotsChecker.isAllowed(url);
                     if (!robotsAllowed) {
@@ -882,8 +1043,25 @@ class Crawler extends EventEmitter {
                             if (destination) {
                                 const destinationUrl = parseUrl(destination, this.#options);
                                 if (destinationUrl) {
-                                    this.#linkList.add(destinationUrl);
-                                    void enqueue(destinationUrl);
+                                    // Gate 2: this direct enqueue does not go through
+                                    // `#handleResult`'s addUrl closure (gate 1), so it needs
+                                    // its own same-cluster-cap check — a JS-redirect trap
+                                    // that advances a parameter via `location.replace()`
+                                    // would otherwise keep re-entering the queue here.
+                                    const gateShapeKey = computeShapeKey(destinationUrl.withoutHashAndAuth);
+                                    const isCapped = this.#options.dedupeCap !== null &&
+                                        gateShapeKey !== null &&
+                                        findScopeEntry(destinationUrl, this.#scope, this.#options) !==
+                                            null &&
+                                        this.#dedupeCapTracker.isCapped(gateShapeKey);
+                                    if (isCapped) {
+                                        if (gateShapeKey)
+                                            this.#recordDedupeCapRejection(gateShapeKey);
+                                    }
+                                    else {
+                                        this.#linkList.add(destinationUrl);
+                                        void enqueue(destinationUrl);
+                                    }
                                 }
                                 else {
                                     // `deriveJsRedirectTarget` already canonicalises
@@ -926,6 +1104,32 @@ class Crawler extends EventEmitter {
                         log(c.dim('Predicted (discarded)'));
                         return;
                     }
+                    // Discard a predicted URL whose rendered body is a
+                    // byte-for-byte duplicate of the previous predicted page of the
+                    // same shape, and stop generating further predictions for that
+                    // shape (checked above, in the pagination-pattern branch). This
+                    // is the always-on backstop against a site that returns 2xx for
+                    // any extrapolated token but ignores it entirely (e.g. always
+                    // serving the same "no results" template) — `shouldDiscardPredicted`
+                    // alone cannot see this, since it only inspects HTTP status.
+                    if (isPredicted &&
+                        result.type === 'success' &&
+                        result.pageData &&
+                        result.pageData.html.length > 0) {
+                        const shapeKey = computeShapeKey(url.withoutHashAndAuth);
+                        if (shapeKey) {
+                            const bodyHash = computeBodyHash(result.pageData.html);
+                            precomputedBodyHash = bodyHash;
+                            const lastBodyHash = this.#predictedShapeBodyHashes.get(shapeKey) ?? null;
+                            if (isPredictedContentDuplicate(bodyHash, lastBodyHash)) {
+                                this.#predictedShapeStopped.add(shapeKey);
+                                handleIgnoreAndSkip(url, this.#linkList, this.#scope, this.#options);
+                                log(c.dim('Predicted (content duplicate, discarded)'));
+                                return;
+                            }
+                            this.#predictedShapeBodyHashes.set(shapeKey, bodyHash);
+                        }
+                    }
                     // Count only after discard check: rendered HTML pages that
                     // will be persisted to the archive. Launch failures bypass
                     // this point via the catch block; discarded predicted URLs
@@ -934,7 +1138,7 @@ class Crawler extends EventEmitter {
                         pagesScraped++;
                     }
                     log('Saving results%dots%');
-                    this.#handleResult(result, url, enqueue, paginationState, concurrency);
+                    this.#handleResult(result, url, enqueue, concurrency, precomputedBodyHash);
                     const parentSource = await this.#resolveParentSource(url);
                     this.#handleResources(result.resources, parentSource);
                     this.#handleConsoleLogs(result.consoleLogs, url, result.pageData?.redirectPaths ?? []);
@@ -1625,9 +1829,15 @@ class Crawler extends EventEmitter {
             // Verified by `scope-auth-leak.e2e.ts`: removing either piece
             // causes that test to fail (without auth → main 401 hangs;
             // without strip → scope cred leaks to off-scope sub-resource).
+            //
+            // The ExURL fields keep the WHATWG percent-encoded form, but
+            // `page.authenticate` sends its arguments verbatim — decode
+            // first or a password containing `[`/`]`/`{`/`}`/`=` etc.
+            // authenticates with the wrong literal (see
+            // `decode-auth-credential.ts`).
             await page.authenticate({
-                username: url.username ?? '',
-                password: url.password ?? '',
+                username: decodeAuthCredential(url.username),
+                password: decodeAuthCredential(url.password),
             });
             // Re-parse from `withoutHashAndAuth` rather than mutating the
             // re-parsed `url.href` object: ExURL pre-computes `href`,

package/lib/crawler/decode-auth-credential.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Decode a percent-encoded userinfo credential (username or password)
+ * parsed out of a URL into the literal string the origin server expects.
+ *
+ * The WHATWG URL parser percent-encodes characters outside the userinfo
+ * set (`[`, `]`, `{`, `}`, `=`, `:`, `@`, non-ASCII, …) and keeps the
+ * `username` / `password` fields in that encoded form. Consumers that
+ * forward credentials out-of-band — `page.authenticate()` for the
+ * browser session, the `auth` request option for the HEAD pre-flight —
+ * must send the decoded literal, or any credential containing such a
+ * character silently authenticates with the wrong string and the server
+ * answers 401. Node's own `urlToOptions` applies the same
+ * `decodeURIComponent` step for `http.request(url)`.
+ *
+ * A malformed sequence (a literal `%` the parser left untouched, e.g. a
+ * user typing `pa%ssword` without encoding it) would make
+ * `decodeURIComponent` throw, so the raw value is returned as a
+ * fallback — identical to the pre-decode behavior for that input.
+ * @param value - The raw (possibly percent-encoded) credential field, or
+ *   `null` when the URL carries no userinfo.
+ * @returns The decoded credential, or an empty string for `null` input.
+ * @example
+ * ```ts
+ * const url = parseUrl('https://user:pa%5Dss%7Bword%3D@example.com/')!;
+ * decodeAuthCredential(url.password); // => 'pa]ss{word='
+ * decodeAuthCredential(null); // => ''
+ * ```
+ */
+export declare function decodeAuthCredential(value: string | null): string;

package/lib/crawler/decode-auth-credential.js

@@ -0,0 +1,39 @@
+/**
+ * Decode a percent-encoded userinfo credential (username or password)
+ * parsed out of a URL into the literal string the origin server expects.
+ *
+ * The WHATWG URL parser percent-encodes characters outside the userinfo
+ * set (`[`, `]`, `{`, `}`, `=`, `:`, `@`, non-ASCII, …) and keeps the
+ * `username` / `password` fields in that encoded form. Consumers that
+ * forward credentials out-of-band — `page.authenticate()` for the
+ * browser session, the `auth` request option for the HEAD pre-flight —
+ * must send the decoded literal, or any credential containing such a
+ * character silently authenticates with the wrong string and the server
+ * answers 401. Node's own `urlToOptions` applies the same
+ * `decodeURIComponent` step for `http.request(url)`.
+ *
+ * A malformed sequence (a literal `%` the parser left untouched, e.g. a
+ * user typing `pa%ssword` without encoding it) would make
+ * `decodeURIComponent` throw, so the raw value is returned as a
+ * fallback — identical to the pre-decode behavior for that input.
+ * @param value - The raw (possibly percent-encoded) credential field, or
+ *   `null` when the URL carries no userinfo.
+ * @returns The decoded credential, or an empty string for `null` input.
+ * @example
+ * ```ts
+ * const url = parseUrl('https://user:pa%5Dss%7Bword%3D@example.com/')!;
+ * decodeAuthCredential(url.password); // => 'pa]ss{word='
+ * decodeAuthCredential(null); // => ''
+ * ```
+ */
+export function decodeAuthCredential(value) {
+    if (!value) {
+        return '';
+    }
+    try {
+        return decodeURIComponent(value);
+    }
+    catch {
+        return value;
+    }
+}

package/lib/crawler/dedupe/compute-meta-signature.d.ts

@@ -0,0 +1,30 @@
+import type { Meta } from '@d-zero/beholder';
+/**
+ * Computes a signature of the four meta fields most likely to be identical
+ * across a self-generating pager/cluster trap: `<title>`, `<meta
+ * name="description">`, `og:title`, `og:url`.
+ *
+ * Returns `null` when the page carries no useful signal — an empty `<title>`
+ * with `og.title` and `og.url` both absent (the two Open Graph fields this
+ * signature actually hashes) is typical of a non-content page (e.g. a bare
+ * external redirect stub), and counting it would let a `null`-ish signature
+ * accidentally look like a majority match against unrelated pages.
+ *
+ * Uses the four fields as-written (no URL absolutisation for `og.url`): all
+ * pages on one site are rendered by the same template, so `og:url` is either
+ * consistently relative or consistently absolute across a trap's pages, and
+ * hashing the raw value keeps this computation a pure string op with no need
+ * for a page-URL argument. Absolutisation only matters when *comparing*
+ * `og:url` against the page's own URL (a different signal, computed
+ * separately at cap-scoring time).
+ * @param meta - Beholder-derived metadata for the page.
+ * @returns A hex-encoded SHA-1 signature, or `null` if the page has no title
+ *   and no Open Graph tags.
+ * @example
+ * ```ts
+ * computeMetaSignature({ title: 'お知らせ', og: { url: '/news' } } as Meta);
+ * // => a stable hex digest
+ * computeMetaSignature({ title: '', og: {} } as Meta); // => null
+ * ```
+ */
+export declare function computeMetaSignature(meta: Meta): string | null;

package/lib/crawler/dedupe/compute-shape-key.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Computes a URL "shape" key: the host plus path/query with every path
+ * segment that contains a digit collapsed to a fixed placeholder, and every
+ * query value (regardless of content) collapsed to a fixed placeholder.
+ *
+ * This absorbs both the "numeric pager" trap shape (`/news/date/2024/` and
+ * `/news/date/1.5e+32/` collapse to the same key) and the "query trap" shape
+ * (`?page=1` / `?page=2` / `?session=ab12cd` all collapse to the same key),
+ * without needing two separate `parentPath` definitions the way issue
+ * #208's original proposal did.
+ *
+ * Uses `../decompose-url.ts` (the pagination-detection one) — NOT
+ * `../../archive/populate-ref-tables/decompose-url.ts`, an unrelated same-named
+ * module with a different `DecomposedUrl` shape used for ref-table population.
+ *
+ * The masking rule here is the deliberate inverse of
+ * `../../archive/body-hash/mask-dynamic-ids.ts`: that module leaves
+ * pure-digit tokens untouched (they are more likely stable content than a
+ * dynamic id) and only masks mixed alphanumeric runs. A shape key needs the
+ * opposite: ANY digit inside a path segment marks it as "probably a
+ * pagination/date/id token", so the whole segment is collapsed. Do not share
+ * masking logic between the two — they classify the same kind of text for
+ * opposite purposes.
+ * @param url - A URL string (protocol-agnostic `//host/...` or full
+ *   `https://host/...`), typically `ExURL.withoutHashAndAuth`.
+ * @returns The shape key, or `null` if `url` cannot be decomposed.
+ * @example
+ * ```ts
+ * computeShapeKey('//example.com/news/date/2024/');
+ * // => 'example.com/news/date/{n}/'
+ * computeShapeKey('//example.com/news/date/1.5e+32/');
+ * // => 'example.com/news/date/{n}/' — same shape
+ * computeShapeKey('//example.com/list?page=1');
+ * // => 'example.com/list?page={v}'
+ * ```
+ */
+export declare function computeShapeKey(url: string): string | null;