@nitpicker/crawler 0.15.0 → 0.17.0

package/lib/crawler/generate-predicted-urls.js

@@ -1,11 +1,24 @@
 import { decomposeUrl } from './decompose-url.js';
 import { reconstructUrl } from './reconstruct-url.js';
+const DIGITS_ONLY_PATTERN = /^\d+$/;
 /**
  * Generates predicted URLs by extrapolating the detected pagination pattern.
  *
  * Starting from `currentUrl`, applies the pattern's step `count` times to produce
  * future page URLs (e.g. if step=1 and currentNumber=2, generates page 3, 4, ...).
  * These URLs are pushed into the crawl queue and discarded later if they 404.
+ *
+ * Generation stops at the first token that would lose its original digit-string
+ * shape, rather than skipping it and continuing: beyond `Number.MAX_SAFE_INTEGER`,
+ * `String()` renders scientific notation (e.g. `"1e+21"`), which is not a valid
+ * path/query token and — left unguarded — becomes a self-generated URL that no
+ * page on the target site ever linked to (observed in production: a
+ * `/news/date/{year}/` pager whose per-anchor `step` was miscalculated from
+ * unrelated pages, compounding across rounds until it emitted
+ * `1.7715854126052197e+120`). A token growing far beyond its original digit
+ * count is equally implausible as a next page number. Both trends are
+ * monotonic as `i` increases, so once one prediction is rejected, every later
+ * one in the same batch would be too — there is nothing to skip past.
  * @param pattern - The detected pagination pattern from `detectPaginationPattern()`
  * @param currentUrl - The URL to extrapolate from (protocol-agnostic, without hash/auth)
  * @param count - Number of predicted URLs to generate (typically equals concurrency)
@@ -17,11 +30,29 @@ export function generatePredictedUrls(pattern, currentUrl, count) {
     const decomposed = decomposeUrl(currentUrl);
     if (!decomposed)
         return [];
+    const { pathSegments, queryValues } = decomposed;
+    const originalToken = pattern.tokenIndex < pathSegments.length
+        ? pathSegments[pattern.tokenIndex]
+        : queryValues[pattern.tokenIndex - pathSegments.length];
+    if (originalToken === undefined)
+        return [];
+    // A pager jumping from 4 digits to 5 (e.g. year 9999 → 10000) is plausible;
+    // jumping straight to 6+ digits within the same predicted batch is not.
+    const maxDigits = originalToken.length + 1;
     const results = [];
     for (let i = 1; i <= count; i++) {
         const nextNum = pattern.currentNumber + pattern.step * i;
-        const url = reconstructUrl(decomposed, pattern.tokenIndex, String(nextNum));
-        results.push(url);
+        if (!Number.isSafeInteger(nextNum))
+            break;
+        const rendered = String(nextNum);
+        if (!DIGITS_ONLY_PATTERN.test(rendered))
+            break;
+        // Preserve zero-padding width (e.g. "01" → "02"), without truncating a
+        // value that has legitimately grown past the original width.
+        const padded = rendered.padStart(originalToken.length, '0');
+        if (padded.length > maxDigits)
+            break;
+        results.push(reconstructUrl(decomposed, pattern.tokenIndex, padded));
     }
     return results;
 }

package/lib/crawler/is-puppeteer-fallback-candidate.js

@@ -26,6 +26,9 @@ import { classifyErrorKind } from '../classify-error-kind.js';
  *   as `timeout` via the `NetTimeoutError "Timeout: <url>"` shape.
  * - **local-network** — operator-side connectivity loss; nothing on this
  *   machine will reach the host.
+ * - **redirect-loop** — the redirect chain itself never terminates; a
+ *   puppeteer navigation follows the identical Location-header chain (or
+ *   hits Chromium's own `ERR_TOO_MANY_REDIRECTS`) and loops the same way.
  * - **protocol** — puppeteer lifecycle race; bouncing back to puppeteer
  *   reproduces the same race.
  * - **unknown** — by design. Spinning up a fresh Chromium for every

package/lib/crawler/types.d.ts

@@ -190,6 +190,29 @@ export interface CrawlerOptions extends Required<Pick<ParseURLOptions, 'disableQ
      * without touching the real network.
      */
     networkProbe: NetworkProbe | null;
+    /**
+     * Same-cluster soft-cap threshold (`--dedupe-cap`), or `null` to disable
+     * the feature entirely (the default). When set, {@link Crawler} stops
+     * enqueueing newly-discovered URLs whose shape (see `computeShapeKey`)
+     * has accumulated this many matching-signature observations (see
+     * `DedupeCapTracker`).
+     */
+    dedupeCap: number | null;
+    /**
+     * Hard cap on the number of distinct URL shapes the same-cluster soft
+     * cap tracks at once (`--dedupe-map-cap`); the least-recently-touched
+     * shape is evicted beyond this. Only relevant when {@link dedupeCap} is
+     * non-null.
+     */
+    dedupeMapCap: number;
+    /**
+     * Shape keys already confirmed capped in a prior session
+     * (persisted as `dedupe_cap_events.shape_key`), seeded into the
+     * tracker's sticky set so `--resume` / `--append` / `--retry-failed` /
+     * `--inventory` do not re-admit a trap this crawl already paid the cost
+     * of discovering once. Ignored when {@link dedupeCap} is `null`.
+     */
+    preloadedStickyShapeKeys: readonly string[];
 }
 /**
  * Inventory-mode runtime configuration. Passed from
@@ -436,6 +459,21 @@ export interface CrawlerEventTypes {
         /** Epoch ms the recovery probe first succeeded. */
         endedAt: number;
     };
+    /**
+     * Emitted the instant the opt-in same-cluster soft cap
+     * ({@link CrawlerOptions.dedupeCap}) confirms a URL shape as a trap (see
+     * `DedupeCapTracker`). The orchestrator persists this via
+     * `Archive.insertDedupeCapEvent` and must remember the returned row id so
+     * `rejected_count` can be finalized once at `crawlEnd` (`Crawler` itself
+     * never touches the archive).
+     */
+    dedupeCap: {
+        shapeKey: string;
+        sampleUrl: string;
+        bodyHash: Buffer;
+        effectiveThreshold: number;
+        observedCount: number;
+    };
 }
 /**
  * Tunables for `NetworkOutageDetector`.

package/lib/crawler-orchestrator.d.ts

@@ -69,6 +69,18 @@ interface CrawlConfig extends Config {
      * `options` so an E2E test can inject it via the public API.
      */
     networkProbe: NetworkProbe | null;
+    /** See {@link CrawlerOptions.dedupeCap}. `null`/omitted disables the feature. */
+    dedupeCap: number | null;
+    /** See {@link CrawlerOptions.dedupeMapCap}. Omitted falls through to `Crawler`'s own default. */
+    dedupeMapCap: number;
+    /**
+     * See {@link CrawlerOptions.preloadedStickyShapeKeys}. Set internally by
+     * the four resuming-session static methods
+     * (`append`/`inventory`/`retryFailed`/`resume`), each independently
+     * calling `archive.listDedupeCapShapeKeys()`; not part of the public
+     * options a caller of those methods passes directly.
+     */
+    preloadedStickyShapeKeys: readonly string[];
 }
 /**
  * Callback invoked after the CrawlerOrchestrator instance is fully initialized
@@ -216,9 +228,10 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * 1. Open the archive (writer mode, takes the archive lock).
      * 2. Reject list-mode archives — they hold metadata-only rows that
      *    inventory has no business touching.
-     * 3. Reject archives with unfinished `pending` URLs — those would inherit
-     *    the inventory `source` label by mistake. Operator must resume /
-     *    retry-failed first.
+     * 3. Warn (but proceed) on archives with unfinished `pending` URLs —
+     *    crawled-wins source priority keeps their labels stable; the
+     *    operator can `--resume` first if they want the prior work
+     *    finalized.
      * 4. If `source` is given, archive its exact bytes under
      *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
      *    Done before scope classification so even a run that discards every
@@ -231,17 +244,33 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Split the remaining novel URLs on the effective `excludes` /
+     *    `excludeUrls` (archived config overlaid with this run's
+     *    overrides — the same inputs the crawl's fetch-time
+     *    `shouldSkipUrl` gate uses). Matching URLs are recorded as
+     *    terminal skipped pages (`is_skipped=1`,
+     *    `skip_reason='excluded'`, `source='inventory-seed'`) instead of
+     *    being imported — the same end state a link-discovered excluded
+     *    URL reaches in a normal crawl — and counted as
+     *    `exclude_skipped` (issue #260). Running this after step 6 keeps
+     *    previously crawled rows that newly match the exclusion config
+     *    untouched (crawled-wins). `excludeKeywords` does not
+     *    participate here: it matches rendered page content, which a URL
+     *    list does not have — HTML seeds still get it at render time via
+     *    the browser verdict.
+     * 8. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
-     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
-     *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 9. If any HTML seeds exist, start a Crawler with
+     * 9. Classify each importable novel URL by URL-extension heuristic
+     *    (no probe — see the in-body rationale). HTML-looking URLs are
+     *    queued as Crawler seeds (`'inventory-seed'`); everything else is
+     *    recorded in `resources` directly as `'inventory-seed'` (no
+     *    browser launch, no HEAD).
+     * 10. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 10. Drop the backup on success; restore it on any throw.
+     * 11. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -264,7 +293,7 @@ export declare class CrawlerOrchestrator extends EventEmitter<CrawlEvent> {
      *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
      *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
-     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
      */
     static inventory(archivePath: string, inventoryUrls: string[], options?: Partial<CrawlConfig>, initializedCallback?: CrawlInitializedCallback, source?: InventorySource | null): Promise<CrawlerOrchestrator>;
     /**

package/lib/crawler-orchestrator.js

@@ -16,6 +16,7 @@ import { isLikelyHtmlUrl } from './crawler/is-likely-html-url.js';
 import { networkOutageSummaryCounter } from './crawler/network-outage-summary-counter.js';
 import { PreloadShortCircuitError } from './crawler/preload-short-circuit-error.js';
 import { protocolAgnosticKey } from './crawler/protocol-agnostic-key.js';
+import { shouldSkipUrl } from './crawler/should-skip-url.js';
 import { crawlerLog, log } from './debug.js';
 import { normalizeToArray } from './normalize-to-array.js';
 import { resolveOutputPath } from './resolve-output-path.js';
@@ -64,6 +65,15 @@ export class CrawlerOrchestrator extends EventEmitter {
     #archive;
     /** The crawler engine that discovers and scrapes pages. */
     #crawler;
+    /**
+     * `dedupe_cap_events.id` for each shape confirmed capped this session, so
+     * `crawlEnd` can look up the right row to finalize with
+     * `Crawler#getDedupeCapRejections`'s counts. A `Map` (not a single
+     * scalar like {@link #openNetworkOutageId}) because, unlike a network
+     * outage, more than one shape can be capped simultaneously within one
+     * crawl.
+     */
+    #dedupeCapEventIds = new Map();
     /** Whether the crawl was started from a pre-defined URL list (non-recursive mode). */
     #fromList;
     /**
@@ -158,6 +168,13 @@ export class CrawlerOrchestrator extends EventEmitter {
             networkOutageHostThreshold: options?.networkOutageHostThreshold,
             networkOutageProbeIntervalMs: options?.networkOutageProbeIntervalMs,
             networkProbe: options?.networkProbe ?? null,
+            dedupeCap: options?.dedupeCap ?? null,
+            dedupeMapCap: options?.dedupeMapCap,
+            // Only the four resuming-session static methods
+            // (`append`/`inventory`/`retryFailed`/`resume`) pass this — a
+            // fresh `crawling()` has no archive history to seed from (see
+            // `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc).
+            preloadedStickyShapeKeys: options?.preloadedStickyShapeKeys ?? [],
         });
     }
     /**
@@ -298,6 +315,23 @@ export class CrawlerOrchestrator extends EventEmitter {
                 })
                     .catch((error) => reject(error));
             });
+            this.#crawler.on('dedupeCap', ({ shapeKey, sampleUrl, bodyHash, effectiveThreshold, observedCount }) => {
+                crawlerLog('Dedupe cap reached: shapeKey=%s effectiveThreshold=%d observedCount=%d', shapeKey, effectiveThreshold, observedCount);
+                console.error(`[dedupe-cap] same-cluster trap confirmed: ${shapeKey} (sample: ${sampleUrl})`);
+                writeQueue
+                    .enqueue(async () => {
+                    const id = await this.#archive.insertDedupeCapEvent({
+                        shapeKey,
+                        sampleUrl,
+                        bodyHash,
+                        effectiveThreshold,
+                        observedCount,
+                        detectedAt: Date.now(),
+                    });
+                    this.#dedupeCapEventIds.set(shapeKey, id);
+                })
+                    .catch((error) => reject(error));
+            });
             this.#crawler.on('response', ({ resource, source }) => {
                 writeQueue
                     .enqueue(() => this.#archive.setResources(resource, source))
@@ -314,6 +348,51 @@ export class CrawlerOrchestrator extends EventEmitter {
                     .catch((error) => reject(error));
             });
             this.#crawler.on('crawlEnd', () => {
+                // Deferred to INSIDE a queued closure, not read synchronously
+                // here, for the same reason `networkOutageRecovered`'s handler
+                // defers reading `#openNetworkOutageId`: a `dedupeCap` event's
+                // INSERT closure may still be queued (not yet executed) at the
+                // instant `crawlEnd` fires. `WriteQueue` runs enqueued
+                // operations in submission order, so by the time THIS closure
+                // executes, every earlier-queued `dedupeCap` INSERT has
+                // already completed and `#dedupeCapEventIds` is reliably
+                // populated.
+                writeQueue
+                    .enqueue(async () => {
+                    const rejections = this.#crawler.getDedupeCapRejections();
+                    // Finalize every shape capped THIS session (has an id in
+                    // `#dedupeCapEventIds`), not just the ones with a nonzero
+                    // rejection count — a shape that capped near the end of the
+                    // crawl (or whose remaining anchors all happened to be
+                    // discovered before it capped) never enters `rejections` at
+                    // all, and would otherwise stay `rejected_count: NULL` forever
+                    // despite the crawl completing normally, corrupting the "NULL
+                    // means the crawl never reached crawlEnd" contract
+                    // `list-dedupe-cap-events.ts` documents.
+                    const shapeKeysToFinalize = new Set([
+                        ...this.#dedupeCapEventIds.keys(),
+                        ...rejections.keys(),
+                    ]);
+                    await Promise.all([...shapeKeysToFinalize].map((shapeKey) => {
+                        const rejectedCount = rejections.get(shapeKey) ?? 0;
+                        const id = this.#dedupeCapEventIds.get(shapeKey);
+                        // A shape capped THIS session has an id here (the
+                        // `dedupeCap` event always enqueues an INSERT before any
+                        // rejection for that shape can be counted) and is
+                        // finalized once via its row id. A shape with no id was
+                        // never observed this session at all — it was preloaded
+                        // into `DedupeCapTracker`'s sticky set from an EARLIER
+                        // session's `dedupe_cap_events` row (see
+                        // `CrawlConfig.preloadedStickyShapeKeys`'s JSDoc), so gate
+                        // rejections still accumulate for it but no new row (and
+                        // thus no id) is ever created. That earlier row's count is
+                        // accumulated onto by shape_key instead of overwritten.
+                        return id === undefined
+                            ? this.#archive.accumulateDedupeCapRejectedCount(shapeKey, rejectedCount)
+                            : this.#archive.finalizeDedupeCapEvent(id, rejectedCount);
+                    }));
+                })
+                    .catch((error) => reject(error));
                 writeQueue
                     .drain()
                     .then(() => resolve())
@@ -521,9 +600,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                     scopeMap.set(parsed.hostname, [...existing, parsed]);
                 }
                 await archive.repromoteExternalPages(scopeMap, archived);
+                // Seed the sticky set from prior sessions' confirmed traps so
+                // `--append` does not pay the cost of re-discovering them (see
+                // `DedupeCapTracker`'s constructor JSDoc).
+                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
                 const orchestrator = new CrawlerOrchestrator(archive, {
                     ...mergedConfig,
                     roots: mergedRoots,
+                    preloadedStickyShapeKeys,
                 });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
@@ -575,9 +659,10 @@ export class CrawlerOrchestrator extends EventEmitter {
      * 1. Open the archive (writer mode, takes the archive lock).
      * 2. Reject list-mode archives — they hold metadata-only rows that
      *    inventory has no business touching.
-     * 3. Reject archives with unfinished `pending` URLs — those would inherit
-     *    the inventory `source` label by mistake. Operator must resume /
-     *    retry-failed first.
+     * 3. Warn (but proceed) on archives with unfinished `pending` URLs —
+     *    crawled-wins source priority keeps their labels stable; the
+     *    operator can `--resume` first if they want the prior work
+     *    finalized.
      * 4. If `source` is given, archive its exact bytes under
      *    `inventory/<sha256>.txt` (see {@link Archive.saveInventorySourceList}).
      *    Done before scope classification so even a run that discards every
@@ -590,17 +675,33 @@ export class CrawlerOrchestrator extends EventEmitter {
      * 6. Subtract URLs that already exist in `pages` or `resources` so the
      *    second (and N-th) inventory pass is a no-op for known rows — keeps
      *    `'inventory-seed'` rows from being silently demoted.
-     * 7. Make `<archive>.bak`. Anything thrown beyond this point restores
+     * 7. Split the remaining novel URLs on the effective `excludes` /
+     *    `excludeUrls` (archived config overlaid with this run's
+     *    overrides — the same inputs the crawl's fetch-time
+     *    `shouldSkipUrl` gate uses). Matching URLs are recorded as
+     *    terminal skipped pages (`is_skipped=1`,
+     *    `skip_reason='excluded'`, `source='inventory-seed'`) instead of
+     *    being imported — the same end state a link-discovered excluded
+     *    URL reaches in a normal crawl — and counted as
+     *    `exclude_skipped` (issue #260). Running this after step 6 keeps
+     *    previously crawled rows that newly match the exclusion config
+     *    untouched (crawled-wins). `excludeKeywords` does not
+     *    participate here: it matches rendered page content, which a URL
+     *    list does not have — HTML seeds still get it at render time via
+     *    the browser verdict.
+     * 8. Make `<archive>.bak`. Anything thrown beyond this point restores
      *    from the backup.
-     * 8. HEAD-probe each novel URL. Responses classified as HTML are queued
-     *    as Crawler seeds (`'inventory-seed'`); everything else is recorded
-     *    in `resources` directly as `'inventory-seed'` (no browser launch).
-     * 9. If any HTML seeds exist, start a Crawler with
+     * 9. Classify each importable novel URL by URL-extension heuristic
+     *    (no probe — see the in-body rationale). HTML-looking URLs are
+     *    queued as Crawler seeds (`'inventory-seed'`); everything else is
+     *    recorded in `resources` directly as `'inventory-seed'` (no
+     *    browser launch, no HEAD).
+     * 10. If any HTML seeds exist, start a Crawler with
      *    `inventoryMode = { seedUrls }` so the rendered page and every newly
      *    discovered downstream link is labelled correctly. `resume` is fed
      *    the existing `scraped` / `resources` sets so links into already-
      *    crawled pages stop at the seen-gate without re-rendering.
-     * 10. Drop the backup on success; restore it on any throw.
+     * 11. Drop the backup on success; restore it on any throw.
      *
      * Mutually exclusive with `--append` / `--retry-failed` / `--resume` /
      * `--diff` / `--list` / `--list-file` / `--single` / `--output` — the
@@ -623,7 +724,7 @@ export class CrawlerOrchestrator extends EventEmitter {
      *   `inventoryUrls` in-memory; the audit row's `source_file_sha256`
      *   column will be `NULL` and no source list is archived.
      * @returns The orchestrator instance after a successful inventory pass.
-     * @throws {Error} When `inventoryUrls` is empty, the archive is in list mode, or pending URLs from a previous crawl remain unresolved.
+     * @throws {Error} When `inventoryUrls` is empty or the archive is in list mode. Unresolved pending URLs from a previous crawl do NOT throw — see step 3.
      */
     static async inventory(archivePath, inventoryUrls, options, initializedCallback, source = null) {
         if (inventoryUrls.length === 0) {
@@ -712,17 +813,49 @@ export class CrawlerOrchestrator extends EventEmitter {
             });
             const knownCount = existingPageUrls.size + existingResourceUrls.size;
             log('[inventory] %d in-scope, %d already in archive, %d new', inScope.length, knownCount, novelUrls.length);
+            // Split the novel URLs on the exclusion config BEFORE the
+            // HTML/non-HTML classification, so an exclude-matched URL is
+            // recorded as a terminal skipped page instead of being imported
+            // (issue #260). The inputs mirror the scrape phase's fetch-time
+            // gate (`shouldSkipUrl` in `crawler.ts` fed by the constructor's
+            // merge): archived config overlaid with this run's overrides,
+            // and `DEFAULT_EXCLUDED_EXTERNAL_URLS` merged ahead of the
+            // user's prefixes — classification and gate must never disagree
+            // about the same URL. Running this AFTER the known-URL filter is
+            // deliberate: a previously crawled row that newly matches the
+            // exclusion config stays untouched (crawled-wins), matching how
+            // `getExistingPageUrls` shields known rows from re-labelling.
+            // `excludeKeywords` is deliberately absent: it matches rendered
+            // page content, which a URL list does not have — HTML seeds
+            // still get it at render time via the browser verdict.
+            const effectiveConfig = { ...archived, ...cleanObject(options) };
+            const excludes = normalizeToArray(effectiveConfig.excludes);
+            const excludeUrls = [
+                ...DEFAULT_EXCLUDED_EXTERNAL_URLS,
+                ...normalizeToArray(effectiveConfig.excludeUrls),
+            ];
+            const excludedNovelUrls = [];
+            const importableNovelUrls = [];
+            for (const url of novelUrls) {
+                if (shouldSkipUrl({ url, excludes, excludeUrls, options: effectiveConfig })) {
+                    excludedNovelUrls.push(url);
+                }
+                else {
+                    importableNovelUrls.push(url);
+                }
+            }
+            if (excludedNovelUrls.length > 0) {
+                log('[inventory] %d URL(s) recorded as skipped (matched excludes / excludeUrls)', excludedNovelUrls.length);
+            }
             if (novelUrls.length === 0) {
                 // Nothing to do — release the archive cleanly without taking a
                 // backup. The orchestrator returned here is empty; the caller
-                // should only invoke `close` on it.
-                const noopConfig = {
-                    ...archived,
-                    ...cleanObject(options),
-                };
-                const orchestrator = new CrawlerOrchestrator(archive, noopConfig);
+                // should only invoke `close` on it. `effectiveConfig` is the
+                // same archived-plus-overrides merge every other path in this
+                // method sees.
+                const orchestrator = new CrawlerOrchestrator(archive, effectiveConfig);
                 if (initializedCallback) {
-                    await initializedCallback(orchestrator, noopConfig);
+                    await initializedCallback(orchestrator, effectiveConfig);
                 }
                 return orchestrator;
             }
@@ -738,7 +871,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             // clause). This flag steers the catch below.
             let ingestionComplete = false;
             try {
-                // Classify novel URLs by URL-extension heuristic (no I/O).
+                // Classify importable novel URLs by URL-extension heuristic (no I/O).
                 // Source file lists come from `ls` on the doc-root, so the
                 // extension reflects the real file type — a HEAD pre-flight
                 // here would be pure wasted I/O. Edge cases:
@@ -769,7 +902,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // null as "not probed" rather than "failed".
                 const rawHtmlSeeds = [];
                 const nonHtmlSeeds = [];
-                for (const url of novelUrls) {
+                for (const url of importableNovelUrls) {
                     if (isLikelyHtmlUrl(url)) {
                         rawHtmlSeeds.push(url);
                     }
@@ -809,7 +942,15 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // these rows up on the next `--resume` via the
                 // `OR p.source != 'crawled'` clause.
                 await archive.insertInventorySeeds(htmlSeeds);
-                log('[inventory] %d HTML seed(s), %d non-HTML resource(s) recorded', htmlSeeds.length, nonHtmlSeeds.length);
+                // Record exclude-matched novel URLs as terminal skipped pages
+                // (`is_skipped=1`, `skip_reason='excluded'`,
+                // `source='inventory-seed'`) — the same end state the normal
+                // crawl's fetch-time gate produces for link-discovered
+                // excluded URLs, so the archive looks identical no matter
+                // how the URL was discovered. Inside the `.bak` window for
+                // the same all-or-nothing reason as the seed inserts above.
+                await archive.insertInventorySkippedPages(excludedNovelUrls);
+                log('[inventory] %d HTML seed(s), %d non-HTML resource(s), %d skipped page(s) recorded', htmlSeeds.length, nonHtmlSeeds.length, excludedNovelUrls.length);
                 // Audit row is written *inside* the `.bak` window: a libsql
                 // hiccup or transient lock on the INSERT aborts the ingestion
                 // and the `.bak` restore wipes the pre-inserted seeds too,
@@ -823,6 +964,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                     htmlSeedsCount: htmlSeeds.length,
                     nonHtmlCount: nonHtmlSeeds.length,
                     outOfScope,
+                    excludeSkipped: excludedNovelUrls.length,
                     sourceFileSha256: source?.sha256 ?? null,
                     invalidSkipped: source?.invalidLineCount ?? null,
                 });
@@ -842,8 +984,7 @@ export class CrawlerOrchestrator extends EventEmitter {
                 // (matches the rest of the orchestrator's public surface —
                 // no inventory bookkeeping leaks out).
                 const baseConfig = {
-                    ...archived,
-                    ...cleanObject(options),
+                    ...effectiveConfig,
                     recursive: true,
                     fromList: false,
                 };
@@ -856,6 +997,16 @@ export class CrawlerOrchestrator extends EventEmitter {
                     inventoryMode: { seedUrls: seedSet },
                 };
                 if (htmlSeeds.length > 0) {
+                    // Seed the sticky set from prior sessions' confirmed traps
+                    // so `--inventory` does not pay the cost of
+                    // re-discovering them (see `DedupeCapTracker`'s
+                    // constructor JSDoc). Scoped to this branch only,
+                    // matching `#preloadDnsBurnedHostCache`'s scoping below —
+                    // the fallback (non-HTML-only) branch never calls
+                    // `orchestrator.crawling(...)`, so the tracker is never
+                    // consulted there.
+                    orchestratorOptions.preloadedStickyShapeKeys =
+                        await archive.listDedupeCapShapeKeys();
                     const orchestrator = new CrawlerOrchestrator(archive, orchestratorOptions);
                     // Re-read pending *after* the pre-insert so the strict-
                     // pending set includes the freshly inserted
@@ -1007,7 +1158,14 @@ export class CrawlerOrchestrator extends EventEmitter {
                 log('Start retrying failed pages');
                 log('Archive %s', absFilePath);
                 log('Reset %d failed page(s)', resetUrls.length);
-                const orchestrator = new CrawlerOrchestrator(archive, config);
+                // Seed the sticky set from prior sessions' confirmed traps so
+                // `--retry-failed` does not pay the cost of re-discovering
+                // them (see `DedupeCapTracker`'s constructor JSDoc).
+                const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
+                const orchestrator = new CrawlerOrchestrator(archive, {
+                    ...config,
+                    preloadedStickyShapeKeys,
+                });
                 const { scraped, pending } = await archive.getCrawlingState();
                 const resources = await archive.getResourceUrlList();
                 const pagesScrapedOffset = await archive.getScrapedHtmlPageCount();
@@ -1056,9 +1214,14 @@ export class CrawlerOrchestrator extends EventEmitter {
     static async resume(stubPath, options, initializedCallback) {
         const archive = await Archive.resume(stubPath);
         const archivedConfig = await archive.getConfig();
+        // Seed the sticky set from prior sessions' confirmed traps so
+        // `--resume` does not pay the cost of re-discovering them (see
+        // `DedupeCapTracker`'s constructor JSDoc).
+        const preloadedStickyShapeKeys = await archive.listDedupeCapShapeKeys();
         const config = {
             ...archivedConfig,
             ...cleanObject(options),
+            preloadedStickyShapeKeys,
         };
         const orchestrator = new CrawlerOrchestrator(archive, config);
         const _url = await archive.getUrl();
@@ -1147,6 +1310,7 @@ export class CrawlerOrchestrator extends EventEmitter {
             new_pages: aggregates.htmlSeedsCount,
             new_resources: aggregates.nonHtmlCount,
             scope_skipped: aggregates.outOfScope,
+            exclude_skipped: aggregates.excludeSkipped,
             invalid_skipped: aggregates.invalidSkipped,
         });
     }

package/lib/crawler.d.ts

@@ -33,6 +33,7 @@ export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
 export { decodeStoredBlob } from './archive/decode-html-blob.js';
 export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
 export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
+export { computeShapeKey } from './crawler/dedupe/compute-shape-key.js';
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
 export * from './crawler/types.js';
@@ -46,6 +47,7 @@ export type { NetworkProbe } from './crawler/probe-network.js';
 export { probeNetwork } from './crawler/probe-network.js';
 export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
 export { chooseProbeHost } from './crawler/choose-probe-host.js';
+export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 export { populateEntityTables } from './archive/populate-entity-tables/populate-entities.js';
 export type { PageDomPathResolver } from './archive/populate-entity-tables/populate-image-items.js';

package/lib/crawler.js

@@ -31,6 +31,7 @@ export { computeBodyHash } from './archive/body-hash/compute-body-hash.js';
 export { decodeStoredBlob } from './archive/decode-html-blob.js';
 export { computeTierAAliasKey } from './archive/url-alias/compute-tier-a-alias-key.js';
 export { computeTierBAliasKey } from './archive/url-alias/compute-tier-b-alias-key.js';
+export { computeShapeKey } from './crawler/dedupe/compute-shape-key.js';
 // Core
 export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
 export * from './types.js';
@@ -43,6 +44,7 @@ export { default as NetworkGate } from './crawler/network-gate.js';
 export { probeNetwork } from './crawler/probe-network.js';
 export { computeOutageClampTimestamp } from './archive/db-ops/outages/compute-outage-clamp-timestamp.js';
 export { chooseProbeHost } from './crawler/choose-probe-host.js';
+export { assertChromeIsInstalled } from './crawler/assert-chrome-installed.js';
 export { computeFileSha256 } from './utils/compute-file-sha256.js';
 // 0.13 ref-table population (issue #191, epic #103). Exposed as the
 // public seam that the migration script (`scripts/migrate-to-0.13.mjs`)

package/lib/permanent-error-kinds.d.ts

@@ -7,11 +7,12 @@ import type { ErrorKind } from './types.js';
  * Used by `resetFailedPages` to exclude pages whose latest recorded error
  * falls in this set, so `--retry-failed` actually converges: without the
  * exclusion, NXDOMAIN / TLS mismatch / `ERR_BLOCKED_BY_CLIENT` /
- * `ECONNREFUSED` / HTTP parse-error pages would be reset to pending on every
- * iteration, the crawler would re-attempt them, they would fail again the
- * same way, and the retry-target count would stay constant forever.
+ * `ECONNREFUSED` / HTTP parse-error / redirect-loop pages would be reset to
+ * pending on every iteration, the crawler would re-attempt them, they would
+ * fail again the same way, and the retry-target count would stay constant
+ * forever.
  *
- * Why these five and not others:
+ * Why these six and not others:
  * - **dns** — `ENOTFOUND` / `ERR_NAME_NOT_RESOLVED` are authoritative DNS
  *   answers; the host is gone (or never existed). EAI_AGAIN is split out as
  *   `dns-transient` precisely so it is NOT in this set.
@@ -28,6 +29,10 @@ import type { ErrorKind } from './types.js';
  *   the listener; either no process is listening on the port or its accept
  *   queue rejected the connection. Either way the answer is final until the
  *   server operator intervenes.
+ * - **redirect-loop** — `Maximum number of redirects exceeded` /
+ *   `ERR_TOO_MANY_REDIRECTS` means the site's own redirect chain never
+ *   terminates; the exact same chain is served on every future fetch until
+ *   the site operator fixes it.
  *
  * Notably absent (intentionally retryable):
  * - `connection-reset` / `connection-timeout` — could be middlebox or