@nitpicker/crawler 0.14.0 → 0.16.0

package/lib/crawler/decode-auth-credential.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Decode a percent-encoded userinfo credential (username or password)
+ * parsed out of a URL into the literal string the origin server expects.
+ *
+ * The WHATWG URL parser percent-encodes characters outside the userinfo
+ * set (`[`, `]`, `{`, `}`, `=`, `:`, `@`, non-ASCII, …) and keeps the
+ * `username` / `password` fields in that encoded form. Consumers that
+ * forward credentials out-of-band — `page.authenticate()` for the
+ * browser session, the `auth` request option for the HEAD pre-flight —
+ * must send the decoded literal, or any credential containing such a
+ * character silently authenticates with the wrong string and the server
+ * answers 401. Node's own `urlToOptions` applies the same
+ * `decodeURIComponent` step for `http.request(url)`.
+ *
+ * A malformed sequence (a literal `%` the parser left untouched, e.g. a
+ * user typing `pa%ssword` without encoding it) would make
+ * `decodeURIComponent` throw, so the raw value is returned as a
+ * fallback — identical to the pre-decode behavior for that input.
+ * @param value - The raw (possibly percent-encoded) credential field, or
+ *   `null` when the URL carries no userinfo.
+ * @returns The decoded credential, or an empty string for `null` input.
+ * @example
+ * ```ts
+ * const url = parseUrl('https://user:pa%5Dss%7Bword%3D@example.com/')!;
+ * decodeAuthCredential(url.password); // => 'pa]ss{word='
+ * decodeAuthCredential(null); // => ''
+ * ```
+ */
+export declare function decodeAuthCredential(value: string | null): string;

package/lib/crawler/decode-auth-credential.js

@@ -0,0 +1,39 @@
+/**
+ * Decode a percent-encoded userinfo credential (username or password)
+ * parsed out of a URL into the literal string the origin server expects.
+ *
+ * The WHATWG URL parser percent-encodes characters outside the userinfo
+ * set (`[`, `]`, `{`, `}`, `=`, `:`, `@`, non-ASCII, …) and keeps the
+ * `username` / `password` fields in that encoded form. Consumers that
+ * forward credentials out-of-band — `page.authenticate()` for the
+ * browser session, the `auth` request option for the HEAD pre-flight —
+ * must send the decoded literal, or any credential containing such a
+ * character silently authenticates with the wrong string and the server
+ * answers 401. Node's own `urlToOptions` applies the same
+ * `decodeURIComponent` step for `http.request(url)`.
+ *
+ * A malformed sequence (a literal `%` the parser left untouched, e.g. a
+ * user typing `pa%ssword` without encoding it) would make
+ * `decodeURIComponent` throw, so the raw value is returned as a
+ * fallback — identical to the pre-decode behavior for that input.
+ * @param value - The raw (possibly percent-encoded) credential field, or
+ *   `null` when the URL carries no userinfo.
+ * @returns The decoded credential, or an empty string for `null` input.
+ * @example
+ * ```ts
+ * const url = parseUrl('https://user:pa%5Dss%7Bword%3D@example.com/')!;
+ * decodeAuthCredential(url.password); // => 'pa]ss{word='
+ * decodeAuthCredential(null); // => ''
+ * ```
+ */
+export function decodeAuthCredential(value) {
+    if (!value) {
+        return '';
+    }
+    try {
+        return decodeURIComponent(value);
+    }
+    catch {
+        return value;
+    }
+}

package/lib/crawler/dedupe/compute-meta-signature.d.ts

@@ -0,0 +1,30 @@
+import type { Meta } from '@d-zero/beholder';
+/**
+ * Computes a signature of the four meta fields most likely to be identical
+ * across a self-generating pager/cluster trap: `<title>`, `<meta
+ * name="description">`, `og:title`, `og:url`.
+ *
+ * Returns `null` when the page carries no useful signal — an empty `<title>`
+ * with `og.title` and `og.url` both absent (the two Open Graph fields this
+ * signature actually hashes) is typical of a non-content page (e.g. a bare
+ * external redirect stub), and counting it would let a `null`-ish signature
+ * accidentally look like a majority match against unrelated pages.
+ *
+ * Uses the four fields as-written (no URL absolutisation for `og.url`): all
+ * pages on one site are rendered by the same template, so `og:url` is either
+ * consistently relative or consistently absolute across a trap's pages, and
+ * hashing the raw value keeps this computation a pure string op with no need
+ * for a page-URL argument. Absolutisation only matters when *comparing*
+ * `og:url` against the page's own URL (a different signal, computed
+ * separately at cap-scoring time).
+ * @param meta - Beholder-derived metadata for the page.
+ * @returns A hex-encoded SHA-1 signature, or `null` if the page has no title
+ *   and no Open Graph tags.
+ * @example
+ * ```ts
+ * computeMetaSignature({ title: 'お知らせ', og: { url: '/news' } } as Meta);
+ * // => a stable hex digest
+ * computeMetaSignature({ title: '', og: {} } as Meta); // => null
+ * ```
+ */
+export declare function computeMetaSignature(meta: Meta): string | null;

package/lib/crawler/dedupe/compute-shape-key.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Computes a URL "shape" key: the host plus path/query with every path
+ * segment that contains a digit collapsed to a fixed placeholder, and every
+ * query value (regardless of content) collapsed to a fixed placeholder.
+ *
+ * This absorbs both the "numeric pager" trap shape (`/news/date/2024/` and
+ * `/news/date/1.5e+32/` collapse to the same key) and the "query trap" shape
+ * (`?page=1` / `?page=2` / `?session=ab12cd` all collapse to the same key),
+ * without needing two separate `parentPath` definitions the way issue
+ * #208's original proposal did.
+ *
+ * Uses `../decompose-url.ts` (the pagination-detection one) — NOT
+ * `../../archive/populate-ref-tables/decompose-url.ts`, an unrelated same-named
+ * module with a different `DecomposedUrl` shape used for ref-table population.
+ *
+ * The masking rule here is the deliberate inverse of
+ * `../../archive/body-hash/mask-dynamic-ids.ts`: that module leaves
+ * pure-digit tokens untouched (they are more likely stable content than a
+ * dynamic id) and only masks mixed alphanumeric runs. A shape key needs the
+ * opposite: ANY digit inside a path segment marks it as "probably a
+ * pagination/date/id token", so the whole segment is collapsed. Do not share
+ * masking logic between the two — they classify the same kind of text for
+ * opposite purposes.
+ * @param url - A URL string (protocol-agnostic `//host/...` or full
+ *   `https://host/...`), typically `ExURL.withoutHashAndAuth`.
+ * @returns The shape key, or `null` if `url` cannot be decomposed.
+ * @example
+ * ```ts
+ * computeShapeKey('//example.com/news/date/2024/');
+ * // => 'example.com/news/date/{n}/'
+ * computeShapeKey('//example.com/news/date/1.5e+32/');
+ * // => 'example.com/news/date/{n}/' — same shape
+ * computeShapeKey('//example.com/list?page=1');
+ * // => 'example.com/list?page={v}'
+ * ```
+ */
+export declare function computeShapeKey(url: string): string | null;

package/lib/crawler/dedupe/compute-shape-key.js

@@ -0,0 +1,56 @@
+import { decomposeUrl } from '../decompose-url.js';
+const DIGIT_CONTAINING_SEGMENT_PATTERN = /\d/;
+const SEGMENT_PLACEHOLDER = '{n}';
+const VALUE_PLACEHOLDER = '{v}';
+/**
+ * Computes a URL "shape" key: the host plus path/query with every path
+ * segment that contains a digit collapsed to a fixed placeholder, and every
+ * query value (regardless of content) collapsed to a fixed placeholder.
+ *
+ * This absorbs both the "numeric pager" trap shape (`/news/date/2024/` and
+ * `/news/date/1.5e+32/` collapse to the same key) and the "query trap" shape
+ * (`?page=1` / `?page=2` / `?session=ab12cd` all collapse to the same key),
+ * without needing two separate `parentPath` definitions the way issue
+ * #208's original proposal did.
+ *
+ * Uses `../decompose-url.ts` (the pagination-detection one) — NOT
+ * `../../archive/populate-ref-tables/decompose-url.ts`, an unrelated same-named
+ * module with a different `DecomposedUrl` shape used for ref-table population.
+ *
+ * The masking rule here is the deliberate inverse of
+ * `../../archive/body-hash/mask-dynamic-ids.ts`: that module leaves
+ * pure-digit tokens untouched (they are more likely stable content than a
+ * dynamic id) and only masks mixed alphanumeric runs. A shape key needs the
+ * opposite: ANY digit inside a path segment marks it as "probably a
+ * pagination/date/id token", so the whole segment is collapsed. Do not share
+ * masking logic between the two — they classify the same kind of text for
+ * opposite purposes.
+ * @param url - A URL string (protocol-agnostic `//host/...` or full
+ *   `https://host/...`), typically `ExURL.withoutHashAndAuth`.
+ * @returns The shape key, or `null` if `url` cannot be decomposed.
+ * @example
+ * ```ts
+ * computeShapeKey('//example.com/news/date/2024/');
+ * // => 'example.com/news/date/{n}/'
+ * computeShapeKey('//example.com/news/date/1.5e+32/');
+ * // => 'example.com/news/date/{n}/' — same shape
+ * computeShapeKey('//example.com/list?page=1');
+ * // => 'example.com/list?page={v}'
+ * ```
+ */
+export function computeShapeKey(url) {
+    const decomposed = decomposeUrl(url);
+    if (!decomposed)
+        return null;
+    const { host, pathSegments, queryKeys } = decomposed;
+    const shapedSegments = pathSegments.map((segment) => DIGIT_CONTAINING_SEGMENT_PATTERN.test(segment) ? SEGMENT_PLACEHOLDER : segment);
+    let key = host;
+    if (shapedSegments.length > 0) {
+        key += `/${shapedSegments.join('/')}`;
+    }
+    if (queryKeys.length > 0) {
+        const pairs = queryKeys.map((k) => `${k}=${VALUE_PLACEHOLDER}`);
+        key += `?${pairs.join('&')}`;
+    }
+    return key;
+}

package/lib/crawler/dedupe/dedupe-cap-tracker.d.ts

@@ -0,0 +1,84 @@
+import type { DedupeCapEvent, DedupeCapObservation, DedupeCapOptions } from './types.js';
+/**
+ * Tracks, per URL shape, whether the crawl has run into a same-metadata
+ * cluster trap (a pager/query-parameter trap the site keeps serving 2xx
+ * for), and confirms it via a Misra-Gries majority-vote counter rather than
+ * a plain observation count.
+ *
+ * **Why Misra-Gries (one slot per shape) instead of a multi-layer memory
+ * design (issue #208's original proposal)**: a single `{ metaSig, count }`
+ * slot per shape can never overcount — `count` is a lower bound on the true
+ * number of matching observations, so false-positive cap firing is
+ * structurally impossible regardless of how many unrelated legit pages
+ * share a shape (e.g. `/product/{id}` with thousands of genuinely distinct
+ * pages). This makes most of that original proposal's eviction machinery
+ * unnecessary: age-based eviction and "parent-path bucket completion"
+ * eviction both exist there only to bound memory for a naive
+ * `Map<sig, count>`, which this design never needs since it holds at most
+ * one slot per *shape* (not per signature-per-shape). Only "cap-reached
+ * sticky migration" (`#sticky`) and "hard map cap with LRU eviction"
+ * (`mapCap`) remain relevant here.
+ *
+ * **Known limitation (accepted, not fixed)**: Misra-Gries (k=1) can only
+ * detect a *strict majority* signature. If a trap alternates between two
+ * near-equally-frequent `metaSig` values for the same shape (e.g. an
+ * even/odd-year template split), `count` oscillates near zero and the cap
+ * never fires. A total-observation-count backstop was considered and
+ * rejected: it would misfire on legitimate large sections (e.g. a
+ * `/product/{id}` catalogue with thousands of distinct, correctly-unique
+ * pages sharing one shape). Because arrival order under concurrent
+ * crawling is non-deterministic, whether this alternating-signature case
+ * fires is itself non-deterministic — test fixtures for this tracker use a
+ * single dominant `metaSig` per shape to keep results deterministic.
+ *
+ * **Rejected alternative — top-K "space-saving" per shape**: keeping the
+ * top K=4 `{metaSig, count}` candidates per shape (instead of one) was
+ * considered so a shape could distinguish more than one competing
+ * signature. Rejected because eviction semantics have no safe default: if
+ * an evicted candidate's count is inherited by its replacement (the
+ * textbook space-saving guarantee), churn through a large legitimate
+ * section (e.g. thousands of distinct `/product/{id}` pages sharing one
+ * shape) inflates an unrelated candidate's inherited count and can
+ * false-positive cap it; if not inherited, a genuine trap can be evicted
+ * before it accumulates enough count, producing a false negative. A single
+ * majority-vote slot per shape has neither failure mode.
+ *
+ * **Rejected alternative — streak counting** (increment on a match with the
+ * immediately preceding observation, reset to zero otherwise): fails the
+ * same way under concurrent crawling as the alternating-signature case
+ * above — a trap and an unrelated same-shape legit page interleaving resets
+ * the streak before it can reach the cap.
+ * @see {@link https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm} for the underlying algorithm (Misra-Gries generalises it to top-K; this uses K=1).
+ */
+export default class DedupeCapTracker {
+    #private;
+    /** Number of distinct shapes currently held in the (non-sticky) state map. Exposed for the `mapCap` bound assertion in tests. */
+    get size(): number;
+    /** Number of shapes confirmed capped (sticky) so far. */
+    get stickyCount(): number;
+    /**
+     * @param options - `--dedupe-cap` / `--dedupe-map-cap` thresholds.
+     * @param preloadedSticky - Shape keys already confirmed capped in a prior
+     *   session (from `dedupe_cap_events.shape_key`), seeded so `--resume` /
+     *   `--append` / `--retry-failed` / `--inventory` do not re-admit a trap
+     *   this crawl already paid the cost of discovering once.
+     */
+    constructor(options: DedupeCapOptions, preloadedSticky?: Iterable<string>);
+    /**
+     * Whether a shape has already been confirmed capped. Callers gate
+     * enqueue decisions on this before ever calling {@link observe}.
+     * @param shapeKey
+     */
+    isCapped(shapeKey: string): boolean;
+    /**
+     * Registers one page's observation and applies the Misra-Gries
+     * majority-vote update for its shape.
+     * @param observation - See {@link DedupeCapObservation}. Callers must not
+     *   call this for a shape that is already capped (check {@link isCapped}
+     *   first) — doing so is a no-op returning `null`, since the shape's slot
+     *   was already dropped from `#state` when it capped.
+     * @returns A {@link DedupeCapEvent} the instant this observation causes
+     *   the shape to newly cross its effective threshold, otherwise `null`.
+     */
+    observe(observation: DedupeCapObservation): DedupeCapEvent | null;
+}

package/lib/crawler/dedupe/dedupe-cap-tracker.js

@@ -0,0 +1,185 @@
+import { isShapeCapped } from './is-shape-capped.js';
+/**
+ * Tracks, per URL shape, whether the crawl has run into a same-metadata
+ * cluster trap (a pager/query-parameter trap the site keeps serving 2xx
+ * for), and confirms it via a Misra-Gries majority-vote counter rather than
+ * a plain observation count.
+ *
+ * **Why Misra-Gries (one slot per shape) instead of a multi-layer memory
+ * design (issue #208's original proposal)**: a single `{ metaSig, count }`
+ * slot per shape can never overcount — `count` is a lower bound on the true
+ * number of matching observations, so false-positive cap firing is
+ * structurally impossible regardless of how many unrelated legit pages
+ * share a shape (e.g. `/product/{id}` with thousands of genuinely distinct
+ * pages). This makes most of that original proposal's eviction machinery
+ * unnecessary: age-based eviction and "parent-path bucket completion"
+ * eviction both exist there only to bound memory for a naive
+ * `Map<sig, count>`, which this design never needs since it holds at most
+ * one slot per *shape* (not per signature-per-shape). Only "cap-reached
+ * sticky migration" (`#sticky`) and "hard map cap with LRU eviction"
+ * (`mapCap`) remain relevant here.
+ *
+ * **Known limitation (accepted, not fixed)**: Misra-Gries (k=1) can only
+ * detect a *strict majority* signature. If a trap alternates between two
+ * near-equally-frequent `metaSig` values for the same shape (e.g. an
+ * even/odd-year template split), `count` oscillates near zero and the cap
+ * never fires. A total-observation-count backstop was considered and
+ * rejected: it would misfire on legitimate large sections (e.g. a
+ * `/product/{id}` catalogue with thousands of distinct, correctly-unique
+ * pages sharing one shape). Because arrival order under concurrent
+ * crawling is non-deterministic, whether this alternating-signature case
+ * fires is itself non-deterministic — test fixtures for this tracker use a
+ * single dominant `metaSig` per shape to keep results deterministic.
+ *
+ * **Rejected alternative — top-K "space-saving" per shape**: keeping the
+ * top K=4 `{metaSig, count}` candidates per shape (instead of one) was
+ * considered so a shape could distinguish more than one competing
+ * signature. Rejected because eviction semantics have no safe default: if
+ * an evicted candidate's count is inherited by its replacement (the
+ * textbook space-saving guarantee), churn through a large legitimate
+ * section (e.g. thousands of distinct `/product/{id}` pages sharing one
+ * shape) inflates an unrelated candidate's inherited count and can
+ * false-positive cap it; if not inherited, a genuine trap can be evicted
+ * before it accumulates enough count, producing a false negative. A single
+ * majority-vote slot per shape has neither failure mode.
+ *
+ * **Rejected alternative — streak counting** (increment on a match with the
+ * immediately preceding observation, reset to zero otherwise): fails the
+ * same way under concurrent crawling as the alternating-signature case
+ * above — a trap and an unrelated same-shape legit page interleaving resets
+ * the streak before it can reach the cap.
+ * @see {@link https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_majority_vote_algorithm} for the underlying algorithm (Misra-Gries generalises it to top-K; this uses K=1).
+ */
+export default class DedupeCapTracker {
+    #options;
+    #state = new Map();
+    #sticky;
+    /** Number of distinct shapes currently held in the (non-sticky) state map. Exposed for the `mapCap` bound assertion in tests. */
+    get size() {
+        return this.#state.size;
+    }
+    /** Number of shapes confirmed capped (sticky) so far. */
+    get stickyCount() {
+        return this.#sticky.size;
+    }
+    /**
+     * @param options - `--dedupe-cap` / `--dedupe-map-cap` thresholds.
+     * @param preloadedSticky - Shape keys already confirmed capped in a prior
+     *   session (from `dedupe_cap_events.shape_key`), seeded so `--resume` /
+     *   `--append` / `--retry-failed` / `--inventory` do not re-admit a trap
+     *   this crawl already paid the cost of discovering once.
+     */
+    constructor(options, preloadedSticky = []) {
+        this.#options = options;
+        this.#sticky = new Set(preloadedSticky);
+    }
+    /**
+     * Whether a shape has already been confirmed capped. Callers gate
+     * enqueue decisions on this before ever calling {@link observe}.
+     * @param shapeKey
+     */
+    isCapped(shapeKey) {
+        return isShapeCapped(this.#sticky, shapeKey);
+    }
+    /**
+     * Registers one page's observation and applies the Misra-Gries
+     * majority-vote update for its shape.
+     * @param observation - See {@link DedupeCapObservation}. Callers must not
+     *   call this for a shape that is already capped (check {@link isCapped}
+     *   first) — doing so is a no-op returning `null`, since the shape's slot
+     *   was already dropped from `#state` when it capped.
+     * @returns A {@link DedupeCapEvent} the instant this observation causes
+     *   the shape to newly cross its effective threshold, otherwise `null`.
+     */
+    observe(observation) {
+        const { shapeKey, metaSig, bodyHash, ogUrlMismatch, url } = observation;
+        if (this.#sticky.has(shapeKey))
+            return null;
+        const existing = this.#state.get(shapeKey);
+        let slot;
+        // The body-hash confidence signal only means something when compared
+        // against a hash a PRIOR observation already recorded for this shape —
+        // comparing a freshly-created (or just-reset) slot's `bodyHash`
+        // against itself would trivially "match" every single time (it is the
+        // same value), collapsing the threshold on the very first
+        // observation of any shape. So this stays `false` whenever the slot
+        // has no observation history to compare against yet.
+        let bodyHashMatches;
+        if (!existing) {
+            slot = { metaSig, count: 1, bodyHash };
+            bodyHashMatches = false;
+        }
+        else if (existing.metaSig === metaSig) {
+            existing.count++;
+            bodyHashMatches = existing.bodyHash.equals(bodyHash);
+            // Track the most recently observed body for this shape, not the
+            // one recorded when the slot was first created — otherwise a
+            // shape whose first page differs from an otherwise-identical run
+            // of later pages (e.g. a one-off warmup response) would compare
+            // every later page against that stale first hash forever and
+            // never see a match.
+            existing.bodyHash = bodyHash;
+            slot = existing;
+        }
+        else {
+            existing.count--;
+            if (existing.count <= 0) {
+                slot = { metaSig, count: 1, bodyHash };
+            }
+            else {
+                slot = existing;
+            }
+            bodyHashMatches = false;
+        }
+        // Re-insert to move this shape to the "most recently touched" end of
+        // the Map's iteration order, which `#enforceHardCap` relies on to
+        // evict the least-recently-touched shape first.
+        this.#state.delete(shapeKey);
+        this.#state.set(shapeKey, slot);
+        const effectiveThreshold = computeEffectiveThreshold(this.#options.cap, bodyHashMatches, ogUrlMismatch);
+        if (slot.count >= effectiveThreshold) {
+            this.#state.delete(shapeKey);
+            this.#sticky.add(shapeKey);
+            return {
+                shapeKey,
+                sampleUrl: url,
+                bodyHash,
+                effectiveThreshold,
+                observedCount: slot.count,
+            };
+        }
+        this.#enforceHardCap();
+        return null;
+    }
+    /**
+     * Evicts the least-recently-touched shape(s) until `#state` is back
+     * within `mapCap`. This is the pathological-case backstop — under normal
+     * operation the Misra-Gries design keeps `#state` bounded by the number
+     * of distinct shapes actually seen, which rarely approaches `mapCap`.
+     */
+    #enforceHardCap() {
+        while (this.#state.size > this.#options.mapCap) {
+            const oldestKey = this.#state.keys().next().value;
+            if (oldestKey === undefined)
+                break;
+            this.#state.delete(oldestKey);
+        }
+    }
+}
+/**
+ * Computes the effective same-cluster cap threshold: the base `--dedupe-cap`
+ * value, halved independently for each confidence signal present (a
+ * matching `body_hash` and an `og:url` that does not point at the page
+ * itself), rounded up so the threshold never reaches zero.
+ * @param baseCap
+ * @param bodyHashMatches
+ * @param ogUrlMismatch
+ */
+function computeEffectiveThreshold(baseCap, bodyHashMatches, ogUrlMismatch) {
+    let threshold = baseCap;
+    if (bodyHashMatches)
+        threshold = Math.ceil(threshold / 2);
+    if (ogUrlMismatch)
+        threshold = Math.ceil(threshold / 2);
+    return Math.max(threshold, 1);
+}

package/lib/crawler/dedupe/is-predicted-content-duplicate.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Determines whether a predicted URL's rendered body is a byte-for-byte
+ * duplicate of the previous predicted page generated for the same URL shape.
+ *
+ * The comparison is deliberately against the previous *predicted* page, not
+ * the origin (real) page the pattern was detected from: a real listing
+ * page's body legitimately differs from an empty/placeholder predicted
+ * page's body regardless of whether the site is a trap, so comparing
+ * against the origin would never fire. Two consecutive predicted pages of
+ * the same shape rendering identical bodies is direct evidence the site
+ * ignores the extrapolated token entirely (e.g. an out-of-range
+ * `/news/date/{n}/` always serves the same "no results" template).
+ * @param bodyHash - The `computeBodyHash` result for the predicted page just scraped.
+ * @param lastBodyHash - The previous predicted page's body hash for the same
+ *   shape, or `null` if this is the first predicted page seen for the shape.
+ * @returns `true` when both hashes exist and are byte-identical.
+ * @example
+ * ```ts
+ * isPredictedContentDuplicate(Buffer.from('a'), null); // false — no prior hash yet
+ * isPredictedContentDuplicate(Buffer.from('a'), Buffer.from('a')); // true
+ * isPredictedContentDuplicate(Buffer.from('a'), Buffer.from('b')); // false
+ * ```
+ */
+export declare function isPredictedContentDuplicate(bodyHash: Buffer, lastBodyHash: Buffer | null): boolean;

package/lib/crawler/dedupe/is-predicted-content-duplicate.js

@@ -0,0 +1,26 @@
+/**
+ * Determines whether a predicted URL's rendered body is a byte-for-byte
+ * duplicate of the previous predicted page generated for the same URL shape.
+ *
+ * The comparison is deliberately against the previous *predicted* page, not
+ * the origin (real) page the pattern was detected from: a real listing
+ * page's body legitimately differs from an empty/placeholder predicted
+ * page's body regardless of whether the site is a trap, so comparing
+ * against the origin would never fire. Two consecutive predicted pages of
+ * the same shape rendering identical bodies is direct evidence the site
+ * ignores the extrapolated token entirely (e.g. an out-of-range
+ * `/news/date/{n}/` always serves the same "no results" template).
+ * @param bodyHash - The `computeBodyHash` result for the predicted page just scraped.
+ * @param lastBodyHash - The previous predicted page's body hash for the same
+ *   shape, or `null` if this is the first predicted page seen for the shape.
+ * @returns `true` when both hashes exist and are byte-identical.
+ * @example
+ * ```ts
+ * isPredictedContentDuplicate(Buffer.from('a'), null); // false — no prior hash yet
+ * isPredictedContentDuplicate(Buffer.from('a'), Buffer.from('a')); // true
+ * isPredictedContentDuplicate(Buffer.from('a'), Buffer.from('b')); // false
+ * ```
+ */
+export function isPredictedContentDuplicate(bodyHash, lastBodyHash) {
+    return lastBodyHash !== null && bodyHash.equals(lastBodyHash);
+}

package/lib/crawler/dedupe/is-shape-capped.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Checks whether a shape has already been confirmed as a same-cluster trap
+ * and immigrated to the sticky set (see `DedupeCapTracker`). A capped shape
+ * needs only this O(1) Set lookup — its Misra-Gries slot has already been
+ * dropped from the tracker's main state map.
+ * @param sticky - The tracker's sticky-shape set.
+ * @param shapeKey - The shape key to check.
+ * @returns `true` if the shape is capped.
+ */
+export declare function isShapeCapped(sticky: ReadonlySet<string>, shapeKey: string): boolean;

package/lib/crawler/dedupe/is-shape-capped.js

@@ -0,0 +1,12 @@
+/**
+ * Checks whether a shape has already been confirmed as a same-cluster trap
+ * and immigrated to the sticky set (see `DedupeCapTracker`). A capped shape
+ * needs only this O(1) Set lookup — its Misra-Gries slot has already been
+ * dropped from the tracker's main state map.
+ * @param sticky - The tracker's sticky-shape set.
+ * @param shapeKey - The shape key to check.
+ * @returns `true` if the shape is capped.
+ */
+export function isShapeCapped(sticky, shapeKey) {
+    return sticky.has(shapeKey);
+}

package/lib/crawler/dedupe/resolve-og-url-mismatch.d.ts

@@ -0,0 +1,31 @@
+import type { Meta } from '@d-zero/beholder';
+/**
+ * Determines whether a page's (absolutised) `og:url` points somewhere other
+ * than the page itself — one of the two confidence signals that lower the
+ * effective same-cluster cap threshold (see `DedupeCapTracker`). A pager
+ * trap's `og:url` typically still points at the parent listing page rather
+ * than the (fake) paginated URL, which this signal is built to catch.
+ *
+ * Duplicates the tiny URL-absolutisation logic from
+ * `../../archive/meta/derive-flat-from-meta.ts` rather than importing it:
+ * that file exports only `deriveFlatFromMeta` (one export per file is a
+ * project convention), so its internal `absolutizeUrl` helper is not
+ * reachable from here. `og:url` arrives un-absolutised (beholder extracts it
+ * via `getAttribute`, preserving relative URLs as-written) — comparing it to
+ * the page's own absolute URL without resolving it first would treat every
+ * relative self-reference (e.g. `content="./"`) as a mismatch, inflating
+ * this signal on ordinary pages.
+ * @param meta - Beholder-derived metadata for the page.
+ * @param pageUrl - The page's own absolute URL.
+ * @returns `true` if `og:url` is present and resolves to a URL different
+ *   from `pageUrl`; `false` if absent (no signal) or if it resolves to the
+ *   same URL.
+ * @example
+ * ```ts
+ * resolveOgUrlMismatch({ title: '', og: { url: '/news' } } as Meta, 'https://example.com/news/date/2024/');
+ * // => true — og:url points at the parent listing, not this page
+ * resolveOgUrlMismatch({ title: '', og: { url: './' } } as Meta, 'https://example.com/');
+ * // => false — relative self-reference resolves to the same URL
+ * ```
+ */
+export declare function resolveOgUrlMismatch(meta: Meta, pageUrl: string): boolean;

package/lib/crawler/dedupe/resolve-og-url-mismatch.js

@@ -0,0 +1,40 @@
+/**
+ * Determines whether a page's (absolutised) `og:url` points somewhere other
+ * than the page itself — one of the two confidence signals that lower the
+ * effective same-cluster cap threshold (see `DedupeCapTracker`). A pager
+ * trap's `og:url` typically still points at the parent listing page rather
+ * than the (fake) paginated URL, which this signal is built to catch.
+ *
+ * Duplicates the tiny URL-absolutisation logic from
+ * `../../archive/meta/derive-flat-from-meta.ts` rather than importing it:
+ * that file exports only `deriveFlatFromMeta` (one export per file is a
+ * project convention), so its internal `absolutizeUrl` helper is not
+ * reachable from here. `og:url` arrives un-absolutised (beholder extracts it
+ * via `getAttribute`, preserving relative URLs as-written) — comparing it to
+ * the page's own absolute URL without resolving it first would treat every
+ * relative self-reference (e.g. `content="./"`) as a mismatch, inflating
+ * this signal on ordinary pages.
+ * @param meta - Beholder-derived metadata for the page.
+ * @param pageUrl - The page's own absolute URL.
+ * @returns `true` if `og:url` is present and resolves to a URL different
+ *   from `pageUrl`; `false` if absent (no signal) or if it resolves to the
+ *   same URL.
+ * @example
+ * ```ts
+ * resolveOgUrlMismatch({ title: '', og: { url: '/news' } } as Meta, 'https://example.com/news/date/2024/');
+ * // => true — og:url points at the parent listing, not this page
+ * resolveOgUrlMismatch({ title: '', og: { url: './' } } as Meta, 'https://example.com/');
+ * // => false — relative self-reference resolves to the same URL
+ * ```
+ */
+export function resolveOgUrlMismatch(meta, pageUrl) {
+    const raw = meta.og?.url;
+    if (!raw)
+        return false;
+    try {
+        return new URL(raw, pageUrl).href !== pageUrl;
+    }
+    catch {
+        return false;
+    }
+}