@nitpicker/crawler 0.20.0 → 0.22.0

package/lib/crawler/dedupe/build-dedupe-cap-observation.d.ts

@@ -0,0 +1,74 @@
+import type { DedupeCapObservation } from './types.js';
+import type { DedupeCapObservationRow } from '../../archive/types.js';
+/**
+ * Reconstructs the `DedupeCapObservation` a previously-scraped page would
+ * have produced had it been observed live, from the archived
+ * `DedupeCapObservationRow` `listDedupeCapObservations` reads back out of
+ * `content_items` / `page_meta`. Used to replay a prior session's
+ * observations into a fresh `DedupeCapTracker` on `--resume` / `--append` /
+ * `--retry-failed` / `--inventory` / `--recrawl`, so the Misra-Gries
+ * counters those sessions accumulate are not silently discarded — only the
+ * confirmed-capped shapes (`DedupeCapTracker`'s sticky set) survive a
+ * process restart otherwise, and a shape that was close to (but short of)
+ * its threshold when the previous session ended would restart at count 0.
+ *
+ * Mirrors the exact exclusions the live crawl-time observation site applies
+ * (`Crawler`'s `'page'` handler, gated on `!isExternal && !isMetadataOnly &&
+ * html.length > 0`): `listDedupeCapObservations` already narrows to that
+ * same population at the SQL layer, so this function only needs to redo
+ * the two per-row signal computations that can independently yield "no
+ * signal" — `computeShapeKey` (a URL that fails to decompose) and
+ * `computeMetaSignature` (no title and no Open Graph tags) — and return
+ * `null` exactly where the live site would have skipped the page.
+ *
+ * **Known limitation — `og:url` absolutisation drift**: `computeMetaSignature`
+ * deliberately hashes `og:url` *as written* (no absolutisation — see its own
+ * JSDoc), but `page_meta.og_url_id` only ever stores the *absolutised* form
+ * (`derive-flat-from-meta.ts`'s `og_url: absolutizeUrl(og?.url, base)`) — the
+ * archive never keeps the raw, possibly-relative string. For a page whose
+ * template writes `og:url` as an absolute URL (the OGP-recommended, and most
+ * common, form) this is a no-op and the reconstructed `metaSig` matches the
+ * live one exactly. For a template that writes it relative, the replayed
+ * `metaSig` for that shape's already-archived members will not bit-for-bit
+ * match the `metaSig` a *newly*-scraped same-shape page computes live this
+ * session (e.g. a previously-failed page succeeding under `--retry-failed`),
+ * so the Misra-Gries counter may fail to recognise it as a repeat of the
+ * dominant signature. This can only cause under-counting (a missed majority
+ * match, decrementing instead of incrementing), never over-counting — a
+ * `metaSig` mismatch can never falsely trigger a cap — so it degrades this
+ * feature's benefit for relative-`og:url` sites without ever making the cap
+ * fire incorrectly.
+ *
+ * **Known limitation — hash-fragment drift in `ogUrlMismatch`**: the live
+ * call site (`Crawler`'s `'page'` handler) resolves `og:url` against
+ * `result.pageData.url.href` — the page's full URL, hash fragment included
+ * — while `listDedupeCapObservations` reads back `url_refs.url`, which is
+ * always stored as `withoutHashAndAuth` (see `insert-page.ts`); the archive
+ * never keeps the hash. For a page whose own URL carries a `#fragment` and
+ * whose `og:url` resolves to that same URL *without* the fragment (a
+ * correct, common self-reference), the live computation sees a mismatch
+ * (the fragment differs) while the replayed one — comparing two
+ * fragment-less strings — does not. Same failure direction as the
+ * `metaSig` limitation above: this can only suppress a mismatch replay
+ * would otherwise have detected, never fabricate one, so it degrades this
+ * signal's confidence-halving benefit for hash-carrying pages without ever
+ * making the cap fire incorrectly. `bodyHash` and the reconstructed `url`
+ * field are unaffected by this — only the `pageUrl` argument to
+ * `resolveOgUrlMismatch` is hash-stripped.
+ * @param row - One archived page's fields, from `listDedupeCapObservations`.
+ * @returns The reconstructed observation, or `null` if the row carries no
+ *   usable shape or meta signal (mirroring the live site's skip condition).
+ * @example
+ * ```ts
+ * buildDedupeCapObservation({
+ * 	url: 'https://example.com/news/date/2024/',
+ * 	title: 'お知らせ',
+ * 	description: null,
+ * 	ogTitle: null,
+ * 	ogUrl: null,
+ * 	bodyHash: Buffer.from('...'),
+ * });
+ * // => { shapeKey: 'example.com/news/date/{n}/', metaSig: '...', bodyHash, ogUrlMismatch: false, url: 'https://example.com/news/date/2024/' }
+ * ```
+ */
+export declare function buildDedupeCapObservation(row: DedupeCapObservationRow): DedupeCapObservation | null;

package/lib/crawler/dedupe/build-dedupe-cap-observation.js

@@ -0,0 +1,97 @@
+import { computeMetaSignature } from './compute-meta-signature.js';
+import { computeShapeKey } from './compute-shape-key.js';
+import { resolveOgUrlMismatch } from './resolve-og-url-mismatch.js';
+/**
+ * Reconstructs the `DedupeCapObservation` a previously-scraped page would
+ * have produced had it been observed live, from the archived
+ * `DedupeCapObservationRow` `listDedupeCapObservations` reads back out of
+ * `content_items` / `page_meta`. Used to replay a prior session's
+ * observations into a fresh `DedupeCapTracker` on `--resume` / `--append` /
+ * `--retry-failed` / `--inventory` / `--recrawl`, so the Misra-Gries
+ * counters those sessions accumulate are not silently discarded — only the
+ * confirmed-capped shapes (`DedupeCapTracker`'s sticky set) survive a
+ * process restart otherwise, and a shape that was close to (but short of)
+ * its threshold when the previous session ended would restart at count 0.
+ *
+ * Mirrors the exact exclusions the live crawl-time observation site applies
+ * (`Crawler`'s `'page'` handler, gated on `!isExternal && !isMetadataOnly &&
+ * html.length > 0`): `listDedupeCapObservations` already narrows to that
+ * same population at the SQL layer, so this function only needs to redo
+ * the two per-row signal computations that can independently yield "no
+ * signal" — `computeShapeKey` (a URL that fails to decompose) and
+ * `computeMetaSignature` (no title and no Open Graph tags) — and return
+ * `null` exactly where the live site would have skipped the page.
+ *
+ * **Known limitation — `og:url` absolutisation drift**: `computeMetaSignature`
+ * deliberately hashes `og:url` *as written* (no absolutisation — see its own
+ * JSDoc), but `page_meta.og_url_id` only ever stores the *absolutised* form
+ * (`derive-flat-from-meta.ts`'s `og_url: absolutizeUrl(og?.url, base)`) — the
+ * archive never keeps the raw, possibly-relative string. For a page whose
+ * template writes `og:url` as an absolute URL (the OGP-recommended, and most
+ * common, form) this is a no-op and the reconstructed `metaSig` matches the
+ * live one exactly. For a template that writes it relative, the replayed
+ * `metaSig` for that shape's already-archived members will not bit-for-bit
+ * match the `metaSig` a *newly*-scraped same-shape page computes live this
+ * session (e.g. a previously-failed page succeeding under `--retry-failed`),
+ * so the Misra-Gries counter may fail to recognise it as a repeat of the
+ * dominant signature. This can only cause under-counting (a missed majority
+ * match, decrementing instead of incrementing), never over-counting — a
+ * `metaSig` mismatch can never falsely trigger a cap — so it degrades this
+ * feature's benefit for relative-`og:url` sites without ever making the cap
+ * fire incorrectly.
+ *
+ * **Known limitation — hash-fragment drift in `ogUrlMismatch`**: the live
+ * call site (`Crawler`'s `'page'` handler) resolves `og:url` against
+ * `result.pageData.url.href` — the page's full URL, hash fragment included
+ * — while `listDedupeCapObservations` reads back `url_refs.url`, which is
+ * always stored as `withoutHashAndAuth` (see `insert-page.ts`); the archive
+ * never keeps the hash. For a page whose own URL carries a `#fragment` and
+ * whose `og:url` resolves to that same URL *without* the fragment (a
+ * correct, common self-reference), the live computation sees a mismatch
+ * (the fragment differs) while the replayed one — comparing two
+ * fragment-less strings — does not. Same failure direction as the
+ * `metaSig` limitation above: this can only suppress a mismatch replay
+ * would otherwise have detected, never fabricate one, so it degrades this
+ * signal's confidence-halving benefit for hash-carrying pages without ever
+ * making the cap fire incorrectly. `bodyHash` and the reconstructed `url`
+ * field are unaffected by this — only the `pageUrl` argument to
+ * `resolveOgUrlMismatch` is hash-stripped.
+ * @param row - One archived page's fields, from `listDedupeCapObservations`.
+ * @returns The reconstructed observation, or `null` if the row carries no
+ *   usable shape or meta signal (mirroring the live site's skip condition).
+ * @example
+ * ```ts
+ * buildDedupeCapObservation({
+ * 	url: 'https://example.com/news/date/2024/',
+ * 	title: 'お知らせ',
+ * 	description: null,
+ * 	ogTitle: null,
+ * 	ogUrl: null,
+ * 	bodyHash: Buffer.from('...'),
+ * });
+ * // => { shapeKey: 'example.com/news/date/{n}/', metaSig: '...', bodyHash, ogUrlMismatch: false, url: 'https://example.com/news/date/2024/' }
+ * ```
+ */
+export function buildDedupeCapObservation(row) {
+    const shapeKey = computeShapeKey(row.url);
+    if (!shapeKey)
+        return null;
+    const meta = {
+        title: row.title ?? '',
+        description: row.description ?? undefined,
+        og: {
+            title: row.ogTitle ?? undefined,
+            url: row.ogUrl ?? undefined,
+        },
+    };
+    const metaSig = computeMetaSignature(meta);
+    if (!metaSig)
+        return null;
+    return {
+        shapeKey,
+        metaSig,
+        bodyHash: row.bodyHash,
+        ogUrlMismatch: resolveOgUrlMismatch(meta, row.url),
+        url: row.url,
+    };
+}

package/lib/crawler/dedupe/compute-meta-signature.d.ts

@@ -1,4 +1,4 @@
-import type { Meta } from '@d-zero/beholder';
+import type { MetaSignatureSource } from './types.js';
 /**
  * Computes a signature of the four meta fields most likely to be identical
  * across a self-generating pager/cluster trap: `<title>`, `<meta
@@ -17,14 +17,16 @@ import type { Meta } from '@d-zero/beholder';
  * for a page-URL argument. Absolutisation only matters when *comparing*
  * `og:url` against the page's own URL (a different signal, computed
  * separately at cap-scoring time).
- * @param meta - Beholder-derived metadata for the page.
+ * @param meta - Beholder-derived metadata for the page, or the equivalent
+ *   `MetaSignatureSource` fields reconstructed from archived `page_meta`
+ *   columns (see `buildDedupeCapObservation`).
  * @returns A hex-encoded SHA-1 signature, or `null` if the page has no title
  *   and no Open Graph tags.
  * @example
  * ```ts
- * computeMetaSignature({ title: 'お知らせ', og: { url: '/news' } } as Meta);
+ * computeMetaSignature({ title: 'お知らせ', og: { url: '/news' } });
  * // => a stable hex digest
- * computeMetaSignature({ title: '', og: {} } as Meta); // => null
+ * computeMetaSignature({ title: '', og: {} }); // => null
  * ```
  */
-export declare function computeMetaSignature(meta: Meta): string | null;
+export declare function computeMetaSignature(meta: MetaSignatureSource): string | null;

package/lib/crawler/dedupe/compute-shape-key.d.ts

@@ -21,6 +21,26 @@
  * pagination/date/id token", so the whole segment is collapsed. Do not share
  * masking logic between the two — they classify the same kind of text for
  * opposite purposes.
+ *
+ * Query keys are additionally run through `flattenQueryKeys` before being
+ * folded into the key, because `ExURL`'s query normalization (`URLSearchParams`
+ * decode-then-re-encode round trip in `@d-zero/shared/parse-url`) means a
+ * site that double-encodes `&`/`=` inside a query value produces a URL whose
+ * query string still arrives here percent-encoded — `decomposeUrl`'s literal
+ * `&`/`=` split then sees ONE key that is itself a percent-encoded
+ * `key=value&key=value...` blob, with any variable content buried inside
+ * that key name rather than in a value. Without flattening, every such URL
+ * gets a distinct key name and therefore a distinct shape key, which
+ * silently defeats `--dedupe-cap` (issue #351) no matter the threshold.
+ *
+ * This changes the shape key computed for URLs that hit the flattening
+ * path. `dedupe_cap_events.shape_key` (see `archive/create-adjunct-tables.ts`)
+ * is an append-only value frozen at crawl time and never rewritten, so an
+ * archive crawled before this fix that happened to already have a
+ * `dedupe_cap_events` row for such a URL (rare, since the bug largely
+ * prevented capping from firing on it in the first place) will not
+ * re-match on the next `backfillDedupeCapEventId` recompute — the same
+ * self-healing-backfill trade-off any shape-key algorithm change makes.
  * @param url - A URL string (protocol-agnostic `//host/...` or full
  *   `https://host/...`), typically `ExURL.withoutHashAndAuth`.
  * @returns The shape key, or `null` if `url` cannot be decomposed.

package/lib/crawler/dedupe/compute-shape-key.js

@@ -1,7 +1,58 @@
+import { decodeURIComponentSafely } from '@d-zero/shared/decode-uri-safely';
 import { decomposeUrl } from '../decompose-url.js';
 const DIGIT_CONTAINING_SEGMENT_PATTERN = /\d/;
 const SEGMENT_PLACEHOLDER = '{n}';
 const VALUE_PLACEHOLDER = '{v}';
+/**
+ * Recursively decodes and re-splits a single query key that may itself be a
+ * percent-encoded `key=value&key=value...` blob (see the "query trap" case
+ * in `computeShapeKey`'s doc comment), peeling one layer of encoding at a
+ * time so a multiply-encoded blob (`%2526` → `%26` → `&`) still converges.
+ *
+ * Each call first decodes via `decodeURIComponentSafely` (falls back to the
+ * input unchanged on malformed `%` sequences). If decoding changed
+ * anything, there may be another layer underneath, so it recurses on the
+ * decoded string to peel further. Once decoding is a no-op (fully decoded,
+ * or undecodable), the value is at a fixed point: split on `&`/`=` if
+ * either is now present and recurse on the pieces (each piece may need its
+ * own decode chain); otherwise it's a leaf.
+ *
+ * Every recursive call operates on a string strictly shorter than its
+ * input — a successful decode always shrinks length (a `%XX` triplet never
+ * expands to more than 1 output character), and splitting on a delimiter
+ * that is present at a fixed point always yields a strictly shorter piece
+ * — so this terminates for any finite input without needing an explicit
+ * depth limit.
+ * @param key - A single raw query key, as returned by `decomposeUrl`.
+ * @returns The one or more sub-key names this key flattens to.
+ */
+function flattenQueryKey(key) {
+    const decoded = decodeURIComponentSafely(key);
+    if (decoded !== key)
+        return flattenQueryKey(decoded);
+    if (!decoded.includes('=') && !decoded.includes('&'))
+        return [decoded];
+    return decoded.split('&').flatMap((pair) => flattenQueryKey(pair.split('=')[0] ?? ''));
+}
+/**
+ * Flattens and normalizes a list of query keys so that keys whose variable
+ * content leaked into the key name itself (via nested percent-encoding)
+ * still converge to the same set of sub-key names regardless of the
+ * variable values or the sub-pairs' original order.
+ *
+ * Deliberately does NOT deduplicate the flattened keys: a legitimate
+ * multi-value query (`?tag=a&tag=b`) must stay distinguishable from a
+ * single-value one (`?tag=a`) — collapsing repeated key names would fold
+ * pages with a different facet count into the same shape, which is a
+ * correctness regression unrelated to the encoding bug this function fixes.
+ * @param keys - Raw query keys, as returned by `decomposeUrl`.
+ * @returns The flattened, sorted key names.
+ */
+function flattenQueryKeys(keys) {
+    return keys
+        .flatMap((key) => flattenQueryKey(key))
+        .toSorted((a, b) => a.localeCompare(b));
+}
 /**
  * Computes a URL "shape" key: the host plus path/query with every path
  * segment that contains a digit collapsed to a fixed placeholder, and every
@@ -25,6 +76,26 @@ const VALUE_PLACEHOLDER = '{v}';
  * pagination/date/id token", so the whole segment is collapsed. Do not share
  * masking logic between the two — they classify the same kind of text for
  * opposite purposes.
+ *
+ * Query keys are additionally run through `flattenQueryKeys` before being
+ * folded into the key, because `ExURL`'s query normalization (`URLSearchParams`
+ * decode-then-re-encode round trip in `@d-zero/shared/parse-url`) means a
+ * site that double-encodes `&`/`=` inside a query value produces a URL whose
+ * query string still arrives here percent-encoded — `decomposeUrl`'s literal
+ * `&`/`=` split then sees ONE key that is itself a percent-encoded
+ * `key=value&key=value...` blob, with any variable content buried inside
+ * that key name rather than in a value. Without flattening, every such URL
+ * gets a distinct key name and therefore a distinct shape key, which
+ * silently defeats `--dedupe-cap` (issue #351) no matter the threshold.
+ *
+ * This changes the shape key computed for URLs that hit the flattening
+ * path. `dedupe_cap_events.shape_key` (see `archive/create-adjunct-tables.ts`)
+ * is an append-only value frozen at crawl time and never rewritten, so an
+ * archive crawled before this fix that happened to already have a
+ * `dedupe_cap_events` row for such a URL (rare, since the bug largely
+ * prevented capping from firing on it in the first place) will not
+ * re-match on the next `backfillDedupeCapEventId` recompute — the same
+ * self-healing-backfill trade-off any shape-key algorithm change makes.
  * @param url - A URL string (protocol-agnostic `//host/...` or full
  *   `https://host/...`), typically `ExURL.withoutHashAndAuth`.
  * @returns The shape key, or `null` if `url` cannot be decomposed.
@@ -49,7 +120,8 @@ export function computeShapeKey(url) {
         key += `/${shapedSegments.join('/')}`;
     }
     if (queryKeys.length > 0) {
-        const pairs = queryKeys.map((k) => `${k}=${VALUE_PLACEHOLDER}`);
+        const flattenedKeys = flattenQueryKeys(queryKeys);
+        const pairs = flattenedKeys.map((k) => `${k}=${VALUE_PLACEHOLDER}`);
         key += `?${pairs.join('&')}`;
     }
     return key;

package/lib/crawler/dedupe/dedupe-cap-tracker.d.ts

@@ -61,7 +61,12 @@ export default class DedupeCapTracker {
      * @param preloadedSticky - Shape keys already confirmed capped in a prior
      *   session (from `dedupe_cap_events.shape_key`), seeded so `--resume` /
      *   `--append` / `--retry-failed` / `--inventory` do not re-admit a trap
-     *   this crawl already paid the cost of discovering once.
+     *   this crawl already paid the cost of discovering once. Only covers
+     *   shapes that already crossed their threshold — a shape that fell
+     *   short still starts this instance's `#state` at nothing; callers
+     *   restore that half via `#observe` (see `Crawler`'s constructor,
+     *   which replays `CrawlerOptions.preloadedDedupeObservations` right
+     *   after constructing this tracker).
      */
     constructor(options: DedupeCapOptions, preloadedSticky?: Iterable<string>);
     /**

package/lib/crawler/dedupe/dedupe-cap-tracker.js

@@ -67,7 +67,12 @@ export default class DedupeCapTracker {
      * @param preloadedSticky - Shape keys already confirmed capped in a prior
      *   session (from `dedupe_cap_events.shape_key`), seeded so `--resume` /
      *   `--append` / `--retry-failed` / `--inventory` do not re-admit a trap
-     *   this crawl already paid the cost of discovering once.
+     *   this crawl already paid the cost of discovering once. Only covers
+     *   shapes that already crossed their threshold — a shape that fell
+     *   short still starts this instance's `#state` at nothing; callers
+     *   restore that half via `#observe` (see `Crawler`'s constructor,
+     *   which replays `CrawlerOptions.preloadedDedupeObservations` right
+     *   after constructing this tracker).
      */
     constructor(options, preloadedSticky = []) {
         this.#options = options;

package/lib/crawler/dedupe/resolve-og-url-mismatch.d.ts

@@ -1,4 +1,4 @@
-import type { Meta } from '@d-zero/beholder';
+import type { MetaSignatureSource } from './types.js';
 /**
  * Determines whether a page's (absolutised) `og:url` points somewhere other
  * than the page itself — one of the two confidence signals that lower the
@@ -15,17 +15,19 @@ import type { Meta } from '@d-zero/beholder';
  * the page's own absolute URL without resolving it first would treat every
  * relative self-reference (e.g. `content="./"`) as a mismatch, inflating
  * this signal on ordinary pages.
- * @param meta - Beholder-derived metadata for the page.
+ * @param meta - Beholder-derived metadata for the page, or the equivalent
+ *   `MetaSignatureSource` fields reconstructed from archived `page_meta`
+ *   columns (see `buildDedupeCapObservation`).
  * @param pageUrl - The page's own absolute URL.
  * @returns `true` if `og:url` is present and resolves to a URL different
  *   from `pageUrl`; `false` if absent (no signal) or if it resolves to the
  *   same URL.
  * @example
  * ```ts
- * resolveOgUrlMismatch({ title: '', og: { url: '/news' } } as Meta, 'https://example.com/news/date/2024/');
+ * resolveOgUrlMismatch({ title: '', og: { url: '/news' } }, 'https://example.com/news/date/2024/');
  * // => true — og:url points at the parent listing, not this page
- * resolveOgUrlMismatch({ title: '', og: { url: './' } } as Meta, 'https://example.com/');
+ * resolveOgUrlMismatch({ title: '', og: { url: './' } }, 'https://example.com/');
  * // => false — relative self-reference resolves to the same URL
  * ```
  */
-export declare function resolveOgUrlMismatch(meta: Meta, pageUrl: string): boolean;
+export declare function resolveOgUrlMismatch(meta: MetaSignatureSource, pageUrl: string): boolean;

package/lib/crawler/dedupe/resolve-og-url-mismatch.js

@@ -14,16 +14,18 @@
  * the page's own absolute URL without resolving it first would treat every
  * relative self-reference (e.g. `content="./"`) as a mismatch, inflating
  * this signal on ordinary pages.
- * @param meta - Beholder-derived metadata for the page.
+ * @param meta - Beholder-derived metadata for the page, or the equivalent
+ *   `MetaSignatureSource` fields reconstructed from archived `page_meta`
+ *   columns (see `buildDedupeCapObservation`).
  * @param pageUrl - The page's own absolute URL.
  * @returns `true` if `og:url` is present and resolves to a URL different
  *   from `pageUrl`; `false` if absent (no signal) or if it resolves to the
  *   same URL.
  * @example
  * ```ts
- * resolveOgUrlMismatch({ title: '', og: { url: '/news' } } as Meta, 'https://example.com/news/date/2024/');
+ * resolveOgUrlMismatch({ title: '', og: { url: '/news' } }, 'https://example.com/news/date/2024/');
  * // => true — og:url points at the parent listing, not this page
- * resolveOgUrlMismatch({ title: '', og: { url: './' } } as Meta, 'https://example.com/');
+ * resolveOgUrlMismatch({ title: '', og: { url: './' } }, 'https://example.com/');
  * // => false — relative self-reference resolves to the same URL
  * ```
  */

package/lib/crawler/dedupe/types.d.ts

@@ -1,3 +1,21 @@
+/**
+ * The structural subset of beholder's `Meta` that `computeMetaSignature` and
+ * `resolveOgUrlMismatch` actually read — narrowed so both functions can
+ * accept a row reconstructed from archived `page_meta` columns (see
+ * `buildDedupeCapObservation`) without a cast. `Meta` itself remains
+ * assignable to this type, so no live crawl-time call site needs to change.
+ */
+export interface MetaSignatureSource {
+    /** See `Meta.title`. */
+    title: string;
+    /** See `Meta.description`. */
+    description?: string;
+    /** See `Meta.og`. Only `title` and `url` are read. */
+    og?: {
+        title?: string;
+        url?: string;
+    };
+}
 /** Options controlling the opt-in same-cluster soft cap (`--dedupe-cap` / `--dedupe-map-cap`). */
 export interface DedupeCapOptions {
     /** Base Misra-Gries majority-vote threshold, before confidence-signal halving. */

package/lib/crawler/fetch-destination.js

@@ -203,6 +203,7 @@ async function _fetchHead(url, isExternal, method, titleBytesLimit, userAgent, t
                     html: '',
                     mainContents: null,
                     scrollHeight: null,
+                    imageScan: { desktop: null, mobile: null },
                     isSkipped: false,
                 };
             };

package/lib/crawler/link-list.d.ts

@@ -126,7 +126,15 @@ export default class LinkList {
      * @param pending - URLs that were pending in the previous session.
      * @param done - URLs that were already completed in the previous session.
      * @param options - URL parsing options for re-parsing the pending URLs.
+     * @param metadataOnlyUrls - The subset of `pending` that was persisted as
+     *   `content_items.is_metadata_only = 1` (see `getCrawlingState`'s
+     *   `pendingMetadataOnly`). Without this, every restored URL re-enters
+     *   the queue as a full-scrape target, silently promoting a URL that was
+     *   only ever meant for a metadata-only scrape (#369) — the in-memory
+     *   `add()` call below restores the same `metadataOnly` flag the
+     *   original discovery set, instead of losing it across the
+     *   resume boundary.
      * @returns The parsed pending URLs that were successfully added to the queue.
      */
-    resume(pending: string[], done: string[], options: ParseURLOptions): ExURL[];
+    resume(pending: string[], done: string[], options: ParseURLOptions, metadataOnlyUrls?: readonly string[]): ExURL[];
 }

package/lib/crawler/link-list.js

@@ -216,9 +216,18 @@ export default class LinkList {
      * @param pending - URLs that were pending in the previous session.
      * @param done - URLs that were already completed in the previous session.
      * @param options - URL parsing options for re-parsing the pending URLs.
+     * @param metadataOnlyUrls - The subset of `pending` that was persisted as
+     *   `content_items.is_metadata_only = 1` (see `getCrawlingState`'s
+     *   `pendingMetadataOnly`). Without this, every restored URL re-enters
+     *   the queue as a full-scrape target, silently promoting a URL that was
+     *   only ever meant for a metadata-only scrape (#369) — the in-memory
+     *   `add()` call below restores the same `metadataOnly` flag the
+     *   original discovery set, instead of losing it across the
+     *   resume boundary.
      * @returns The parsed pending URLs that were successfully added to the queue.
      */
-    resume(pending, done, options) {
+    resume(pending, done, options, metadataOnlyUrls = []) {
+        const metadataOnlyKeys = new Set(metadataOnlyUrls.map((url) => protocolAgnosticKey(url)));
         const parsedPending = [];
         for (const url of done) {
             this.#done.add(protocolAgnosticKey(url));
@@ -228,7 +237,8 @@ export default class LinkList {
             if (!parsedUrl) {
                 continue;
             }
-            this.add(parsedUrl);
+            const isMetadataOnly = metadataOnlyKeys.has(protocolAgnosticKey(parsedUrl.withoutHashAndAuth));
+            this.add(parsedUrl, isMetadataOnly ? { metadataOnly: true } : undefined);
             parsedPending.push(parsedUrl);
         }
         return parsedPending;

package/lib/crawler/link-to-page-data.js

@@ -50,6 +50,7 @@ export function linkToPageData(link) {
         html: '',
         mainContents: null,
         scrollHeight: null,
+        imageScan: { desktop: null, mobile: null },
         isSkipped: false,
     };
 }

package/lib/crawler/resource-to-page-data.js

@@ -56,6 +56,7 @@ export function resourceToPageData(params) {
         html: '',
         mainContents: null,
         scrollHeight: null,
+        imageScan: { desktop: null, mobile: null },
         isSkipped: false,
     };
 }

package/lib/crawler/types.d.ts

@@ -1,8 +1,10 @@
+import type { DedupeCapObservation } from './dedupe/types.js';
 import type { NetworkProbe } from './probe-network.js';
 import type { PageSource } from '../archive/types.js';
 import type { ErrorKind } from '../types.js';
 import type { PageData, CrawlerError, Resource } from '../utils/types/types.js';
 import type { ChangePhaseEvent, ConsoleLogEntry, ScrapeResult } from '@d-zero/beholder';
+import type { Lanes } from '@d-zero/dealer';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 /**
  * Result of resolving a URL that redirects to a destination already rendered
@@ -223,6 +225,79 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * of discovering once. Ignored when {@link dedupeCap} is `null`.
      */
     preloadedStickyShapeKeys: readonly string[];
+    /**
+     * A prior session's per-shape Misra-Gries observations, replayed into
+     * the tracker synchronously in the constructor (see `Crawler`'s own
+     * constructor JSDoc) so a shape that was close to — but short of — its
+     * threshold when the previous session ended does not restart its
+     * counter at 0. Unlike {@link preloadedStickyShapeKeys} (which only
+     * carries shapes already confirmed capped), this carries every
+     * qualifying page's observation regardless of whether it ever
+     * contributed to a cap. Seeded from `archive.listDedupeCapObservations()`
+     * by the same five resuming-session methods, via
+     * `buildDedupeCapObservation`. Ignored when {@link dedupeCap} is `null`.
+     */
+    preloadedDedupeObservations: readonly DedupeCapObservation[];
+    /**
+     * A `Lanes` instance owned by the caller (typically the CLI), reused as
+     * `deal()`'s display instead of letting `deal()` create its own. `null`/
+     * `undefined` falls back to `deal()`'s own `Lanes` (keyed off `verbose`).
+     *
+     * Passing this is what lets a caller running crawl-body `deal()`
+     * alongside its own runtime input UI (e.g. a CLI reading stdin) draw
+     * that UI as a `Lanes` footer without a second `Lanes`/`Display`
+     * instance fighting over the same terminal stream (`@d-zero/dealer`'s
+     * single-instance-per-stream constraint).
+     */
+    lanes?: Lanes;
+}
+/**
+ * A runtime change to apply to an in-progress crawl's tunable options via
+ * {@link Crawler.updateRuntimeOptions}. `parallels`/`interval` replace the
+ * current value; the exclude arrays are additive (new entries merge into the
+ * existing list, duplicates dropped) — there is no way to remove an
+ * already-set exclude pattern.
+ */
+export interface CrawlRuntimeOptionsPatch {
+    /** New concurrency limit (integer, `>= 1`). */
+    parallels?: number;
+    /** New per-URL delay in milliseconds (integer, `>= 0`). */
+    interval?: number;
+    /** Glob patterns to add to {@link CrawlerOptions.excludes}. */
+    excludes?: readonly string[];
+    /** URL prefixes to add to {@link CrawlerOptions.excludeUrls}. */
+    excludeUrls?: readonly string[];
+    /** Keywords to add to {@link CrawlerOptions.excludeKeywords}. */
+    excludeKeywords?: readonly string[];
+}
+/**
+ * Snapshot of the tunable crawl options after applying a
+ * {@link CrawlRuntimeOptionsPatch}, returned by
+ * {@link Crawler.updateRuntimeOptions} so the caller can report what changed
+ * without re-reading `Crawler`'s private state.
+ */
+export interface CrawlRuntimeOptions {
+    readonly parallels: number;
+    readonly interval: number;
+    readonly excludes: readonly string[];
+    readonly excludeUrls: readonly string[];
+    readonly excludeKeywords: readonly string[];
+    /**
+     * The subset of `patch.excludes` that was actually new — i.e. not already
+     * present in {@link excludes} before this patch, and not a duplicate of
+     * an earlier entry in the same patch. Empty when `patch.excludes` was
+     * omitted or every entry it carried was already present. A caller
+     * reporting "what changed" (`format-crawl-console-result.ts`) needs this
+     * distinct from {@link excludes} itself — the additive-only merge already
+     * silently drops duplicates, so echoing back `patch.excludes` verbatim as
+     * "added" would claim a no-op resubmission (e.g. the operator typing the
+     * same `exclude` pattern twice) actually added something.
+     */
+    readonly addedExcludes: readonly string[];
+    /** Same as {@link addedExcludes}, for `patch.excludeUrls`/{@link excludeUrls}. */
+    readonly addedExcludeUrls: readonly string[];
+    /** Same as {@link addedExcludes}, for `patch.excludeKeywords`/{@link excludeKeywords}. */
+    readonly addedExcludeKeywords: readonly string[];
 }
 /**
  * Inventory-mode runtime configuration. Passed from