@nitpicker/core 0.13.0 → 0.14.0

package/lib/discover-analyze-plugins.js

@@ -7,7 +7,6 @@
 const STANDARD_ANALYZE_PLUGINS = [
     '@nitpicker/analyze-axe',
     '@nitpicker/analyze-lighthouse',
-    '@nitpicker/analyze-main-contents',
     '@nitpicker/analyze-markuplint',
     '@nitpicker/analyze-search',
     '@nitpicker/analyze-textlint',

package/lib/nitpicker.d.ts

@@ -66,7 +66,7 @@ export declare class Nitpicker extends EventEmitter<NitpickerEvent> {
      * declared concurrency or by {@link DEFAULT_CONCURRENCY}. This architecture
      * enables per-plugin progress tracking via Lanes.
      *
-     * The analysis proceeds in two phases:
+     * The analysis proceeds in up to three phases:
      *
      * 1. **`eachPage` phase** - For each plugin with `eachPage`, dispatches
      *    page analysis tasks to the plugin's worker pool. Progress is
@@ -76,10 +76,32 @@ export declare class Nitpicker extends EventEmitter<NitpickerEvent> {
      *    sequentially in the main thread. These are lightweight checks
      *    that don't need DOM access.
      *
-     * On completion, three data entries are stored in the archive:
+     * 3. **Template classification phase** (opt-in via
+     *    {@link AnalyzeOptions.classifyTemplates}) - Classifies every internal
+     *    HTML page into a template group by DOM-structure similarity. Unlike
+     *    phases 1-2 this is not a discovered `@nitpicker/analyze-*` plugin; it
+     *    runs once, globally, after every batch from phases 1-2 has been
+     *    accumulated (see `accumulatedPages` below for why it cannot run
+     *    per-batch).
+     *
+     * On completion, data entries are stored in the archive:
      * - `analysis/report` - Full {@link Report} with headers and data
      * - `analysis/table` - The raw {@link Table} instance (serialized)
      * - SQL-backed analysis violation tables - Flat {@link Violation} records
+     * - `page_templates` SQL table - `templateKey` per page, written via
+     *   `Archive.replacePageTemplates` and never round-tripped through
+     *   `Table`/`Report`
+     *
+     * When `filter` resolves to zero plugins (only reachable when
+     * `classifyTemplates` is the sole reason for this call — see
+     * `cli/src/commands/analyze.ts`'s `--templates`-only bypass), this call
+     * contributes no plugin data of its own. To avoid silently wiping every
+     * column and violation from a *previous* run's plugins, this case skips
+     * `replaceAnalysisViolations` entirely and seeds the new `Table` from the
+     * archive's existing `analysis/report` (if any). This does not apply when
+     * one or more plugins ran: running a different plugin subset than a prior
+     * call has always fully replaced the report/table/violations with that
+     * subset's output, and that established contract is unchanged here.
      * @param filter - Optional list of plugin module names to run.
      *   If omitted, all configured plugins are executed.
      * @param options - Optional settings for progress display.

package/lib/nitpicker.js

@@ -7,6 +7,7 @@ import c from 'ansi-colors';
 import { importModules } from './import-modules.js';
 import { loadPluginSettings } from './load-plugin-settings.js';
 import { Table } from './table.js';
+import { classifyPageTemplates } from './template-classification/classify-page-templates.js';
 import { UrlEventBus } from './url-event-bus.js';
 import { WorkerPool } from './worker/worker-pool.js';
 export { definePlugin } from './hooks/define-plugin.js';
@@ -29,6 +30,34 @@ const workerPath = path.resolve(__dirname, 'worker/worker.js');
  * Tracks the number of CPU cores so that worker pools scale with the host.
  */
 const DEFAULT_CONCURRENCY = Math.max(1, os.cpus().length);
+/**
+ * Renders a `@d-zero/page-cluster` `ProgressEvent` as a short, human-readable
+ * fragment for the Lanes template-classification lane, including a
+ * done/total percentage wherever the event carries both halves of one
+ * (matching the `N/M (X%)` convention every other lane in this file uses).
+ * `pass0-signals` and `stage-b-start` carry only a running count with no
+ * corpus-wide total, so those two render count-only.
+ * @param event - Progress event forwarded from `resolvePageClusterKeys`.
+ * @returns A short status fragment, e.g. `"42/100 blocks (42%)"`.
+ */
+function formatTemplateClassificationProgress(event) {
+    switch (event.phase) {
+        case 'pass0-signals': {
+            return `reading pages (${event.pagesSeen})`;
+        }
+        case 'pass1-block-complete': {
+            const percent = Math.round((event.blocksProcessed / event.totalBlocks) * 100);
+            return `${event.blocksProcessed}/${event.totalBlocks} blocks (${percent}%)`;
+        }
+        case 'pass1b-assign': {
+            const percent = Math.round((event.pagesAssigned / event.pagesToAssign) * 100);
+            return `${event.pagesAssigned}/${event.pagesToAssign} pages (${percent}%)`;
+        }
+        case 'stage-b-start': {
+            return `merging ${event.unitCount} units`;
+        }
+    }
+}
 /**
  * Core orchestrator for running analyze plugins against a `.nitpicker` archive.
  *
@@ -112,7 +141,7 @@ export class Nitpicker extends EventEmitter {
      * declared concurrency or by {@link DEFAULT_CONCURRENCY}. This architecture
      * enables per-plugin progress tracking via Lanes.
      *
-     * The analysis proceeds in two phases:
+     * The analysis proceeds in up to three phases:
      *
      * 1. **`eachPage` phase** - For each plugin with `eachPage`, dispatches
      *    page analysis tasks to the plugin's worker pool. Progress is
@@ -122,10 +151,32 @@ export class Nitpicker extends EventEmitter {
      *    sequentially in the main thread. These are lightweight checks
      *    that don't need DOM access.
      *
-     * On completion, three data entries are stored in the archive:
+     * 3. **Template classification phase** (opt-in via
+     *    {@link AnalyzeOptions.classifyTemplates}) - Classifies every internal
+     *    HTML page into a template group by DOM-structure similarity. Unlike
+     *    phases 1-2 this is not a discovered `@nitpicker/analyze-*` plugin; it
+     *    runs once, globally, after every batch from phases 1-2 has been
+     *    accumulated (see `accumulatedPages` below for why it cannot run
+     *    per-batch).
+     *
+     * On completion, data entries are stored in the archive:
      * - `analysis/report` - Full {@link Report} with headers and data
      * - `analysis/table` - The raw {@link Table} instance (serialized)
      * - SQL-backed analysis violation tables - Flat {@link Violation} records
+     * - `page_templates` SQL table - `templateKey` per page, written via
+     *   `Archive.replacePageTemplates` and never round-tripped through
+     *   `Table`/`Report`
+     *
+     * When `filter` resolves to zero plugins (only reachable when
+     * `classifyTemplates` is the sole reason for this call — see
+     * `cli/src/commands/analyze.ts`'s `--templates`-only bypass), this call
+     * contributes no plugin data of its own. To avoid silently wiping every
+     * column and violation from a *previous* run's plugins, this case skips
+     * `replaceAnalysisViolations` entirely and seeds the new `Table` from the
+     * archive's existing `analysis/report` (if any). This does not apply when
+     * one or more plugins ran: running a different plugin subset than a prior
+     * call has always fully replaced the report/table/violations with that
+     * subset's output, and that established contract is unchanged here.
      * @param filter - Optional list of plugin module names to run.
      *   If omitted, all configured plugins are executed.
      * @param options - Optional settings for progress display.
@@ -149,6 +200,34 @@ export class Nitpicker extends EventEmitter {
         const analyzeMods = await importModules(plugins);
         const lanes = options?.lanes;
         const table = new Table();
+        if (plugins.length === 0) {
+            // Only reachable via `--templates` with zero configured/selected
+            // plugins (see this method's JSDoc). Seed from whatever
+            // `analysis/report` already exists so this run — which contributes
+            // no plugin data of its own — doesn't wipe a previous run's columns.
+            try {
+                const previousReport = await this.archive.getData('analysis/report');
+                if (previousReport.pageData) {
+                    table.addHeaders(previousReport.pageData.headers);
+                    table.addData(previousReport.pageData.data);
+                }
+            }
+            catch (error) {
+                // ENOENT (no previous `analysis/report` — e.g. first analyze()
+                // call ever on this archive) is expected and starting from an
+                // empty table is correct. Anything else (corrupted JSON,
+                // permission error) is not silently swallowed: emit it so the
+                // user knows a previous run's data may not have been preserved,
+                // while still proceeding rather than blocking this run entirely.
+                if (error?.code !== 'ENOENT') {
+                    const message = error instanceof Error ? error.message : String(error);
+                    await this.emit('error', {
+                        message: `Failed to read previous analysis/report before a zero-plugin run: ${message}`,
+                        error: error instanceof Error ? error : null,
+                    });
+                }
+            }
+        }
         for (const mod of analyzeMods) {
             if (!mod.headers) {
                 continue;
@@ -180,7 +259,32 @@ export class Nitpicker extends EventEmitter {
             const label = pluginLabels.get(name) ?? name;
             lanes?.update(id, c.dim(`${label}: Waiting...`));
         }
+        // Template classification (opt-in, `--templates`) is a corpus-wide batch
+        // computation, not a per-page `AnalyzePlugin`, so it has its own lane
+        // outside `pluginLaneIds` and does not go through `analyzeMods`/`Worker`.
+        const templateClassificationLaneId = options?.classifyTemplates
+            ? eachPagePlugins.length
+            : null;
+        if (templateClassificationLaneId != null) {
+            lanes?.update(templateClassificationLaneId, c.dim('Template classification: Waiting...'));
+        }
+        // Accumulated across every `getPagesWithRefs` batch so template
+        // classification runs once, globally, after the loop below completes —
+        // never per-batch. Per-batch classification would produce template keys
+        // that are only comparable within their own batch (see
+        // `classifyPageTemplates`'s caller contract). `Page` instances are
+        // lightweight handles (HTML is fetched lazily via `getHtml()`), so
+        // accumulating every page across a 100,000-page batch size is not an
+        // OOM risk even for archives with hundreds of thousands of pages.
+        const accumulatedPages = [];
         await this.archive.getPagesWithRefs(100_000, async (pages) => {
+            if (options?.classifyTemplates) {
+                // Avoid `push(...pages)`: a 100,000-page batch can overflow V8's
+                // argument-spread limit even though the data itself fits in memory.
+                for (const page of pages) {
+                    accumulatedPages.push(page);
+                }
+            }
             const urlEmitter = new UrlEventBus();
             // Phase 1: eachPage plugins (sequentially, pages in parallel)
             for (const [pluginSeqIndex, { plugin, modIndex }] of eachPagePlugins.entries()) {
@@ -354,13 +458,60 @@ export class Nitpicker extends EventEmitter {
         }, {
             withRefs: false,
         });
+        // Phase 3: template classification (opt-in). Deliberately outside the
+        // `getPagesWithRefs` loop above — it must run once, globally, over every
+        // accumulated page, not once per 100,000-page batch (see
+        // `accumulatedPages`'s JSDoc comment).
+        //
+        // Wrapped in try/catch (unlike phases 1-2, whose failures are already
+        // caught per-page/per-mod) because this phase has no finer-grained
+        // unit to catch around: a failure here must not discard the
+        // already-computed `table`/`allViolations` from phases 1-2, which are
+        // persisted in the unconditional tail below regardless of this
+        // phase's outcome.
+        if (templateClassificationLaneId != null) {
+            try {
+                const templateKeys = await classifyPageTemplates({
+                    archive: this.archive,
+                    pages: accumulatedPages,
+                    // Only worth paying for when there's a lane to update —
+                    // passing a defined callback at all (even a no-op one)
+                    // demotes `resolvePageClusterKeys` off its byte-identical,
+                    // yield-overhead-free sync path for corpora at or below its
+                    // inline threshold (see @d-zero/page-cluster's own docs).
+                    onProgress: lanes
+                        ? (event) => {
+                            lanes.update(templateClassificationLaneId, `Template classification: ${formatTemplateClassificationProgress(event)}%braille%`);
+                        }
+                        : undefined,
+                });
+                if (templateKeys.size > 0) {
+                    await this.archive.replacePageTemplates(templateKeys);
+                }
+                lanes?.update(templateClassificationLaneId, c.green(`Template classification: Done (${templateKeys.size} pages)`));
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                lanes?.update(templateClassificationLaneId, c.red(`Template classification: Failed (${message})`));
+                await this.emit('error', {
+                    message: `[template classification] ${message}`,
+                    error: error instanceof Error ? error : null,
+                });
+            }
+        }
         const report = {
             name: 'general',
             pageData: table.toJSON(),
         };
         await this.archive.setData('analysis/report', report);
         await this.archive.setData('analysis/table', table);
-        await this.archive.replaceAnalysisViolations(allViolations);
+        // See this method's JSDoc: a zero-plugin call (only reachable via
+        // `--templates` alone) contributes no violations of its own, so
+        // replacing the whole table with an empty array would silently erase
+        // every violation a previous run's plugins stored.
+        if (plugins.length > 0) {
+            await this.archive.replaceAnalysisViolations(allViolations);
+        }
     }
     /**
      * Loads and caches the plugin configuration from the user's config file.

package/lib/template-classification/classify-page-templates.d.ts

@@ -0,0 +1,64 @@
+import type { ProgressEvent } from '@d-zero/page-cluster/resolve-page-cluster-keys';
+import type { Archive, Page } from '@nitpicker/crawler';
+/**
+ * Options for {@link classifyPageTemplates}.
+ */
+export interface ClassifyPageTemplatesOptions {
+    /** The archive to read pages/stylesheets from and to derive the cache key from. */
+    archive: Archive;
+    /**
+     * All pages already loaded for this `analyze()` run (see
+     * `Nitpicker.analyze`'s accumulation across `getPagesWithRefs` batches —
+     * classification must run once, globally, not per batch).
+     */
+    pages: readonly Page[];
+    /**
+     * Progress callback forwarded to `@d-zero/page-cluster`'s
+     * `resolvePageClusterKeys`, so long-running classification on large
+     * archives isn't silently unresponsive.
+     */
+    onProgress?: (event: ProgressEvent) => void;
+}
+/**
+ * Classifies every internal HTML page in the archive into a template group
+ * by DOM-structure similarity, using `@d-zero/page-cluster`.
+ *
+ * Result is cached, keyed by the archive file's path + size + mtime + page
+ * count (see {@link buildArchiveCacheKey}). **This only helps one narrow
+ * case**: a crash or Ctrl-C after this function completes but before
+ * `nitpicker.write()` finishes (writing back a multi-gigabyte archive can
+ * itself take a while) — re-running on the same, not-yet-rewritten archive
+ * file hits the cache instead of re-running the whole multi-pass clustering.
+ * It does **not** help a *second, fully successful* `analyze --templates`
+ * run on the same crawled data: `nitpicker.write()` always re-packages the
+ * archive (new mtime, and usually a different size), which changes the key
+ * on every completed run. There is no explicit `--force` escape hatch —
+ * any change to the archive changes the key, which is the correctness
+ * signal this cache relies on. The cache is **not** cleared at the start of
+ * a run (unlike the per-plugin `Cache` in `nitpicker.ts`) — its entire
+ * purpose is surviving the one crash-before-write window described above.
+ *
+ * There is no mid-computation checkpoint/resume: `resolvePageClusterKeys`
+ * holds state across its own internal multi-pass read and has no resume
+ * API, so a crash *during* classification means the next run classifies
+ * from scratch (this is a limitation of the upstream library, not something
+ * this function works around).
+ *
+ * Internally: the external library's `resolvePageClusterKeys` returns a
+ * `clusterKey` per page; this function's public naming is `templateKey`,
+ * to avoid colliding with nitpicker's unrelated pre-existing "isolated
+ * cluster" concept (`@nitpicker/query`'s `compute-isolated-clusters.ts`,
+ * link-reachability graph components — a completely different kind of
+ * grouping from DOM-structure similarity).
+ * @param options - See {@link ClassifyPageTemplatesOptions}.
+ * @returns Map from page URL (`page.url.href`) to its template key. Only
+ *   internal HTML pages with retrievable HTML have an entry.
+ * @example
+ * ```ts
+ * const templateKeys = await classifyPageTemplates({ archive, pages });
+ * for (const [url, templateKey] of templateKeys) {
+ *   table.addData({ [url]: { templateKey: { value: templateKey } } });
+ * }
+ * ```
+ */
+export declare function classifyPageTemplates(options: ClassifyPageTemplatesOptions): Promise<Map<string, string>>;

package/lib/template-classification/classify-page-templates.js

@@ -0,0 +1,119 @@
+import { stat } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { resolvePageClusterKeys } from '@d-zero/page-cluster/resolve-page-cluster-keys';
+import { Cache } from '@d-zero/shared/cache';
+import { collectPageStylesheetUrls } from './collect-page-stylesheet-urls.js';
+import { createPageClusterFactory } from './create-page-cluster-factory.js';
+/**
+ * Classifies every internal HTML page in the archive into a template group
+ * by DOM-structure similarity, using `@d-zero/page-cluster`.
+ *
+ * Result is cached, keyed by the archive file's path + size + mtime + page
+ * count (see {@link buildArchiveCacheKey}). **This only helps one narrow
+ * case**: a crash or Ctrl-C after this function completes but before
+ * `nitpicker.write()` finishes (writing back a multi-gigabyte archive can
+ * itself take a while) — re-running on the same, not-yet-rewritten archive
+ * file hits the cache instead of re-running the whole multi-pass clustering.
+ * It does **not** help a *second, fully successful* `analyze --templates`
+ * run on the same crawled data: `nitpicker.write()` always re-packages the
+ * archive (new mtime, and usually a different size), which changes the key
+ * on every completed run. There is no explicit `--force` escape hatch —
+ * any change to the archive changes the key, which is the correctness
+ * signal this cache relies on. The cache is **not** cleared at the start of
+ * a run (unlike the per-plugin `Cache` in `nitpicker.ts`) — its entire
+ * purpose is surviving the one crash-before-write window described above.
+ *
+ * There is no mid-computation checkpoint/resume: `resolvePageClusterKeys`
+ * holds state across its own internal multi-pass read and has no resume
+ * API, so a crash *during* classification means the next run classifies
+ * from scratch (this is a limitation of the upstream library, not something
+ * this function works around).
+ *
+ * Internally: the external library's `resolvePageClusterKeys` returns a
+ * `clusterKey` per page; this function's public naming is `templateKey`,
+ * to avoid colliding with nitpicker's unrelated pre-existing "isolated
+ * cluster" concept (`@nitpicker/query`'s `compute-isolated-clusters.ts`,
+ * link-reachability graph components — a completely different kind of
+ * grouping from DOM-structure similarity).
+ * @param options - See {@link ClassifyPageTemplatesOptions}.
+ * @returns Map from page URL (`page.url.href`) to its template key. Only
+ *   internal HTML pages with retrievable HTML have an entry.
+ * @example
+ * ```ts
+ * const templateKeys = await classifyPageTemplates({ archive, pages });
+ * for (const [url, templateKey] of templateKeys) {
+ *   table.addData({ [url]: { templateKey: { value: templateKey } } });
+ * }
+ * ```
+ */
+export async function classifyPageTemplates(options) {
+    const { archive, pages, onProgress } = options;
+    const cache = getTemplateCache();
+    const cacheKey = await buildArchiveCacheKey(archive.filePath, pages.length);
+    if (cacheKey) {
+        const cached = await cache.load(cacheKey);
+        if (cached) {
+            return new Map(Object.entries(cached));
+        }
+    }
+    const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
+    const { factory, getYieldedUrls } = createPageClusterFactory(pages, stylesheetsByUrl);
+    const clusterKeys = await resolvePageClusterKeys(factory, onProgress ? { onProgress } : undefined);
+    // Safe only after `resolvePageClusterKeys` has resolved — see
+    // `createPageClusterFactory`'s JSDoc for why.
+    const yieldedUrls = getYieldedUrls();
+    if (clusterKeys.length !== yieldedUrls.length) {
+        // Would silently mis-map template keys to the wrong URLs if allowed
+        // through — see createPageClusterFactory's JSDoc for why this should
+        // be structurally impossible, but a loud failure here is far
+        // preferable to a quiet mis-assignment.
+        throw new Error(`classifyPageTemplates: resolvePageClusterKeys returned ${clusterKeys.length} keys for ${yieldedUrls.length} yielded pages.`);
+    }
+    const result = new Map();
+    for (const [index, url] of yieldedUrls.entries()) {
+        result.set(url, clusterKeys[index]);
+    }
+    if (cacheKey) {
+        await cache.store(cacheKey, Object.fromEntries(result));
+    }
+    return result;
+}
+/**
+ * Lazily constructs the persistent template-classification cache. Kept as a
+ * function (rather than a module-level constant) so tests can point it at
+ * an isolated tmpDir-per-test without module-mocking `os.tmpdir`.
+ */
+function getTemplateCache() {
+    return new Cache('nitpicker-templates', path.join(os.tmpdir(), 'nitpicker/cache/templates'));
+}
+/**
+ * Derives a cache key from the archive file's identity: its absolute path,
+ * byte size, mtime, and internal HTML page count. Byte size + mtime alone is
+ * a coarse signal — a same-path overwrite that happens to land on the same
+ * byte size within the filesystem's mtime tick (coarse mtime resolution,
+ * or two writes close enough together) would otherwise be indistinguishable
+ * from "unchanged". Page count is folded in as a cheap (already-computed,
+ * no extra query), second independent signal that most re-crawls
+ * (`--append` / `--inventory`) change; it does **not** catch every
+ * possible content change (e.g. `--retry-failed` replacing a failed page's
+ * content without changing the total page count), which is an accepted,
+ * documented gap rather than an attempt at a full content hash — hashing
+ * every page's HTML on every `analyze()` call would cost more than the
+ * cache is meant to save.
+ * @param archiveFilePath - `Archive.filePath` of the archive being analyzed.
+ * @param pageCount - Number of pages passed to {@link classifyPageTemplates}
+ *   for this run (`pages.length`, before internal/HTML filtering).
+ * @returns The cache key, or `null` if the file's stats cannot be read
+ *   (falls back to always recomputing rather than caching under the wrong
+ *   key).
+ */
+async function buildArchiveCacheKey(archiveFilePath, pageCount) {
+    try {
+        const stats = await stat(archiveFilePath);
+        return `${archiveFilePath}:${stats.size}:${stats.mtimeMs}:${pageCount}`;
+    }
+    catch {
+        return null;
+    }
+}

package/lib/template-classification/collect-page-stylesheet-urls.d.ts

@@ -0,0 +1,32 @@
+import type { Archive } from '@nitpicker/crawler';
+/**
+ * Collects, for every page that references at least one CSS stylesheet, the
+ * set of stylesheet URLs it references.
+ *
+ * Reads in three narrow passes instead of one wide JOIN, to avoid
+ * materializing the full page×stylesheet edge set at once — a real
+ * 486,000-page archive has ~2.5 million such edges, and a single JOIN
+ * projecting both URL strings for all of them is a transient allocation
+ * spike well beyond what any single page's data actually needs:
+ *
+ * 1. Resolve which `resource_items` rows are classified `'css'`
+ *    (`content_type_refs.category`) and their stylesheet URL.
+ * 2. Resolve `resource_ref_edges` for just those CSS resource ids, in
+ *    chunks, projecting only the two integer ids per row.
+ * 3. Resolve the page ids collected in step 2 to their URLs.
+ *
+ * Identical stylesheet-URL arrays are interned across pages: pages sharing a
+ * template share the same CSS set, so the number of distinct arrays
+ * converges to a few hundred even across hundreds of thousands of pages.
+ * @param archive - The archive to read from.
+ * @returns Map from page URL (parseUrl-normalized `href`, matching
+ *   `Page.url.href`) to the stylesheet URLs it references. Pages that
+ *   reference no CSS resource have no entry — callers should default to an
+ *   empty array on a missed lookup.
+ * @example
+ * ```ts
+ * const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
+ * const hrefs = stylesheetsByUrl.get(page.url.href) ?? [];
+ * ```
+ */
+export declare function collectPageStylesheetUrls(archive: Archive): Promise<Map<string, readonly string[]>>;

package/lib/template-classification/collect-page-stylesheet-urls.js

@@ -0,0 +1,130 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { eachSplitted } from '@nitpicker/crawler';
+/**
+ * SQLite has a hard cap of 999 bound variables per statement. A real archive
+ * can have far more than 999 CSS-referencing edges or pages, so every
+ * `whereIn` below is chunked at this size.
+ *
+ * Independently duplicated (not shared/imported) from `@nitpicker/query`'s
+ * `compute-isolated-clusters.ts`, which defines the identical
+ * `SQLITE_IN_CHUNK = 500` for the same reason. If this margin below 999 ever
+ * needs revisiting, update both — there is no shared module either package
+ * depends on to hold one copy.
+ */
+const SQLITE_IN_CHUNK = 500;
+/**
+ * Collects, for every page that references at least one CSS stylesheet, the
+ * set of stylesheet URLs it references.
+ *
+ * Reads in three narrow passes instead of one wide JOIN, to avoid
+ * materializing the full page×stylesheet edge set at once — a real
+ * 486,000-page archive has ~2.5 million such edges, and a single JOIN
+ * projecting both URL strings for all of them is a transient allocation
+ * spike well beyond what any single page's data actually needs:
+ *
+ * 1. Resolve which `resource_items` rows are classified `'css'`
+ *    (`content_type_refs.category`) and their stylesheet URL.
+ * 2. Resolve `resource_ref_edges` for just those CSS resource ids, in
+ *    chunks, projecting only the two integer ids per row.
+ * 3. Resolve the page ids collected in step 2 to their URLs.
+ *
+ * Identical stylesheet-URL arrays are interned across pages: pages sharing a
+ * template share the same CSS set, so the number of distinct arrays
+ * converges to a few hundred even across hundreds of thousands of pages.
+ * @param archive - The archive to read from.
+ * @returns Map from page URL (parseUrl-normalized `href`, matching
+ *   `Page.url.href`) to the stylesheet URLs it references. Pages that
+ *   reference no CSS resource have no entry — callers should default to an
+ *   empty array on a missed lookup.
+ * @example
+ * ```ts
+ * const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
+ * const hrefs = stylesheetsByUrl.get(page.url.href) ?? [];
+ * ```
+ */
+export async function collectPageStylesheetUrls(archive) {
+    const knex = archive.getKnex();
+    const cssResources = (await knex('resource_items as ri')
+        .join('content_type_refs as ctr', 'ctr.id', 'ri.content_type_id')
+        .where('ctr.category', 'css')
+        .select('ri.id as id', 'ri.url_id as urlId'));
+    // A CSS resource identified by a `data:` URI (routed to `blob_refs`
+    // instead of `url_refs`, `resource_items.url_blob_id`) has no stylesheet
+    // URL to report — skip it rather than passing a `null` into `whereIn`.
+    const cssResourcesWithUrl = cssResources.filter((r) => r.urlId != null);
+    if (cssResourcesWithUrl.length === 0) {
+        return new Map();
+    }
+    const cssUrlById = new Map();
+    await eachSplitted(cssResourcesWithUrl.map((r) => r.urlId), SQLITE_IN_CHUNK, async (chunk) => {
+        const rows = (await knex('url_refs').whereIn('id', chunk).select('id', 'url'));
+        for (const row of rows) {
+            cssUrlById.set(row.id, row.url);
+        }
+    });
+    const cssUrlByResourceId = new Map();
+    for (const r of cssResourcesWithUrl) {
+        const url = cssUrlById.get(r.urlId);
+        if (url) {
+            cssUrlByResourceId.set(r.id, url);
+        }
+    }
+    const resourceIdsByPageId = new Map();
+    await eachSplitted(cssResourcesWithUrl.map((r) => r.id), SQLITE_IN_CHUNK, async (chunk) => {
+        const rows = (await knex('resource_ref_edges')
+            .whereIn('resource_id', chunk)
+            .select('resource_id as resourceId', 'page_id as pageId'));
+        // Avoid `push(...rows)`: on large real archives this chunk array can
+        // be large enough to overflow V8's argument-spread limit even though
+        // the underlying data itself fits in memory.
+        for (const row of rows) {
+            const existing = resourceIdsByPageId.get(row.pageId);
+            if (existing) {
+                existing.push(row.resourceId);
+            }
+            else {
+                resourceIdsByPageId.set(row.pageId, [row.resourceId]);
+            }
+        }
+    });
+    if (resourceIdsByPageId.size === 0) {
+        return new Map();
+    }
+    const pageUrlById = new Map();
+    await eachSplitted([...resourceIdsByPageId.keys()], SQLITE_IN_CHUNK, async (chunk) => {
+        const rows = (await knex('content_items as ci')
+            .join('url_refs as ur', 'ur.id', 'ci.url_id')
+            .whereIn('ci.id', chunk)
+            .select('ci.id as id', 'ur.url as url'));
+        for (const row of rows) {
+            pageUrlById.set(row.id, row.url);
+        }
+    });
+    const internedSets = new Map();
+    const result = new Map();
+    for (const [pageId, resourceIds] of resourceIdsByPageId) {
+        const pageUrlRaw = pageUrlById.get(pageId);
+        if (!pageUrlRaw) {
+            continue;
+        }
+        const parsed = parseUrl(pageUrlRaw);
+        if (!parsed) {
+            continue;
+        }
+        const urls = resourceIds
+            .map((id) => cssUrlByResourceId.get(id))
+            .filter((url) => url != null)
+            .toSorted();
+        if (urls.length === 0) {
+            continue;
+        }
+        const internKey = urls.join(' ');
+        let interned = internedSets.get(internKey);
+        if (!interned) {
+            interned = Object.freeze(urls);
+            internedSets.set(internKey, interned);
+        }
+        result.set(parsed.href, interned);
+    }
+    return result;
+}

package/lib/template-classification/create-page-cluster-factory.d.ts

@@ -0,0 +1,55 @@
+import type { PageFactory } from '@d-zero/page-cluster/resolve-page-cluster-keys';
+import type { Page } from '@nitpicker/crawler';
+/**
+ * Builds the `PageFactory` that `@d-zero/page-cluster`'s
+ * `resolvePageClusterKeys` reads the corpus through, plus a way to read back
+ * the URL list it actually yielded (see below for why that can't be known
+ * synchronously up front).
+ *
+ * Only internal HTML pages (`Page.isInternalPage()` — content-type based,
+ * matching the "a page is judged by content-type, not `isTarget`" archive
+ * invariant) with retrievable HTML are yielded. A page can be an internal
+ * HTML page and still have no retrievable HTML (e.g. a skipped or
+ * self-heal-affected row) — `page.getHtml()` returning falsy is only known
+ * by actually calling it, not from any synchronous field.
+ *
+ * `resolvePageClusterKeys` calls the returned factory one or more times
+ * (once for corpora at or below its inline threshold, twice above it — see
+ * its own docs) to re-read the corpus. Whichever pages end up excluded
+ * (empty HTML) must be excluded **identically** on every call, or the
+ * pass-to-pass page sequence drifts and the library's same-order contract
+ * breaks. Excluding by `isInternalPage()` is safe (synchronous,
+ * deterministic per page). Excluding by empty `getHtml()` is also safe,
+ * but only because the archive is immutable for the duration of one
+ * `analyze()` run — the same page's HTML never changes between calls.
+ *
+ * `getYieldedUrls()` reflects the URLs yielded by the **most recently
+ * completed** full iteration of the factory's generator, deliberately not a
+ * separate up-front probe pass over every candidate's HTML: every documented
+ * call shape of `resolvePageClusterKeys` fully drains the factory it's given
+ * (that's what "reading the factory" means in its own docs), so by the time
+ * `resolvePageClusterKeys` resolves, at least one full drain has already
+ * happened and `getYieldedUrls()` reflects it — a dedicated probe pass would
+ * only add a full corpus-wide decompression sweep beyond the library's own
+ * 1-2, for no benefit. `classifyPageTemplates` still defends against a
+ * violated assumption here with a hard length check against
+ * `resolvePageClusterKeys`'s result array.
+ * @param pages - Candidate pages (as already loaded by `Nitpicker.analyze()`
+ *   — this function does not re-query the archive).
+ * @param stylesheetsByUrl - Output of `collectPageStylesheetUrls`, keyed by
+ *   `page.url.href`.
+ * @returns `factory` to pass to `resolvePageClusterKeys`, and
+ *   `getYieldedUrls()` — call this only *after* `resolvePageClusterKeys`
+ *   resolves; its result array has exactly the length and order of
+ *   `getYieldedUrls()` at that point.
+ * @example
+ * ```ts
+ * const { factory, getYieldedUrls } = createPageClusterFactory(pages, stylesheetsByUrl);
+ * const clusterKeys = await resolvePageClusterKeys(factory);
+ * const yieldedUrls = getYieldedUrls(); // safe to read now
+ * ```
+ */
+export declare function createPageClusterFactory(pages: readonly Page[], stylesheetsByUrl: ReadonlyMap<string, readonly string[]>): {
+    factory: PageFactory;
+    getYieldedUrls: () => readonly string[];
+};

package/lib/template-classification/create-page-cluster-factory.js

@@ -0,0 +1,88 @@
+/**
+ * Builds the `PageFactory` that `@d-zero/page-cluster`'s
+ * `resolvePageClusterKeys` reads the corpus through, plus a way to read back
+ * the URL list it actually yielded (see below for why that can't be known
+ * synchronously up front).
+ *
+ * Only internal HTML pages (`Page.isInternalPage()` — content-type based,
+ * matching the "a page is judged by content-type, not `isTarget`" archive
+ * invariant) with retrievable HTML are yielded. A page can be an internal
+ * HTML page and still have no retrievable HTML (e.g. a skipped or
+ * self-heal-affected row) — `page.getHtml()` returning falsy is only known
+ * by actually calling it, not from any synchronous field.
+ *
+ * `resolvePageClusterKeys` calls the returned factory one or more times
+ * (once for corpora at or below its inline threshold, twice above it — see
+ * its own docs) to re-read the corpus. Whichever pages end up excluded
+ * (empty HTML) must be excluded **identically** on every call, or the
+ * pass-to-pass page sequence drifts and the library's same-order contract
+ * breaks. Excluding by `isInternalPage()` is safe (synchronous,
+ * deterministic per page). Excluding by empty `getHtml()` is also safe,
+ * but only because the archive is immutable for the duration of one
+ * `analyze()` run — the same page's HTML never changes between calls.
+ *
+ * `getYieldedUrls()` reflects the URLs yielded by the **most recently
+ * completed** full iteration of the factory's generator, deliberately not a
+ * separate up-front probe pass over every candidate's HTML: every documented
+ * call shape of `resolvePageClusterKeys` fully drains the factory it's given
+ * (that's what "reading the factory" means in its own docs), so by the time
+ * `resolvePageClusterKeys` resolves, at least one full drain has already
+ * happened and `getYieldedUrls()` reflects it — a dedicated probe pass would
+ * only add a full corpus-wide decompression sweep beyond the library's own
+ * 1-2, for no benefit. `classifyPageTemplates` still defends against a
+ * violated assumption here with a hard length check against
+ * `resolvePageClusterKeys`'s result array.
+ * @param pages - Candidate pages (as already loaded by `Nitpicker.analyze()`
+ *   — this function does not re-query the archive).
+ * @param stylesheetsByUrl - Output of `collectPageStylesheetUrls`, keyed by
+ *   `page.url.href`.
+ * @returns `factory` to pass to `resolvePageClusterKeys`, and
+ *   `getYieldedUrls()` — call this only *after* `resolvePageClusterKeys`
+ *   resolves; its result array has exactly the length and order of
+ *   `getYieldedUrls()` at that point.
+ * @example
+ * ```ts
+ * const { factory, getYieldedUrls } = createPageClusterFactory(pages, stylesheetsByUrl);
+ * const clusterKeys = await resolvePageClusterKeys(factory);
+ * const yieldedUrls = getYieldedUrls(); // safe to read now
+ * ```
+ */
+export function createPageClusterFactory(pages, stylesheetsByUrl) {
+    const candidates = pages.filter((page) => page.isInternalPage());
+    let yieldedUrls = [];
+    const factory = () => yieldPageClusterSignals(candidates, stylesheetsByUrl, (urls) => {
+        yieldedUrls = urls;
+    });
+    return { factory, getYieldedUrls: () => yieldedUrls };
+}
+/**
+ * The actual per-call iterator body for {@link createPageClusterFactory}'s
+ * `PageFactory`. HTML is fetched lazily, one page at a time, so the archive
+ * never holds more than one page's decompressed HTML in memory regardless
+ * of corpus size — matching `@d-zero/page-cluster`'s own memory-bounding
+ * design for its multi-pass reads.
+ * @param candidates - Pre-filtered internal HTML pages, in fixed order.
+ * @param stylesheetsByUrl - See {@link createPageClusterFactory}.
+ * @param onDrained - Called with every yielded URL, in order, once this
+ *   particular iteration reaches the end of `candidates` — never called if
+ *   the consumer stops iterating early.
+ * @yields {PageClusterSignals} One per candidate page with retrievable HTML.
+ */
+async function* yieldPageClusterSignals(candidates, stylesheetsByUrl, onDrained) {
+    const yieldedUrls = [];
+    for (const page of candidates) {
+        const html = await page.getHtml();
+        if (!html) {
+            continue;
+        }
+        const url = page.url;
+        yieldedUrls.push(url.href);
+        yield {
+            paths: url.paths,
+            stylesheetHrefs: stylesheetsByUrl.get(url.href) ?? [],
+            html,
+            host: url.port ? `${url.hostname}:${url.port}` : url.hostname,
+        };
+    }
+    onDrained(yieldedUrls);
+}

package/lib/types.d.ts

@@ -44,6 +44,17 @@ export interface AnalyzeOptions {
     readonly lanes?: Lanes;
     /** When `true`, outputs plain-text progress lines instead of animated Lanes. */
     readonly verbose?: boolean;
+    /**
+     * When `true`, runs an additional, opt-in classification phase that
+     * groups internal HTML pages by DOM-structure similarity (using
+     * `@d-zero/page-cluster`), independent of the configured
+     * `@nitpicker/analyze-*` plugins. Results are added to the report table
+     * as a `templateKey` column. Defaults to `false` — the algorithm's
+     * runtime on very large archives is not yet characterized, so it is
+     * never run implicitly.
+     * @see {@link ./template-classification/classify-page-templates.ts!classifyPageTemplates}
+     */
+    readonly classifyTemplates?: boolean;
 }
 /**
  * CLI-specified overrides for individual plugin settings.
@@ -70,11 +81,6 @@ export interface PluginOverrides {
         /** CSS selector to narrow the search scope. */
         scope?: string;
     };
-    /** Overrides for `@nitpicker/analyze-main-contents` plugin settings. */
-    '@nitpicker/analyze-main-contents'?: {
-        /** CSS selector for main content detection. */
-        mainContentSelector?: string;
-    };
     /** Overrides for `@nitpicker/analyze-axe` plugin settings. */
     '@nitpicker/analyze-axe'?: {
         /** BCP 47 language tag for locale. */

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/core",
-	"version": "0.13.0",
+	"version": "0.14.0",
 	"description": "Plugin-based page analysis engine for Nitpicker",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -28,12 +28,13 @@
 	},
 	"dependencies": {
 		"@d-zero/dealer": "1.9.4",
+		"@d-zero/page-cluster": "0.3.1",
 		"@d-zero/shared": "0.22.2",
-		"@nitpicker/crawler": "0.13.0",
-		"@nitpicker/types": "0.13.0",
+		"@nitpicker/crawler": "0.14.0",
+		"@nitpicker/types": "0.14.0",
 		"ansi-colors": "4.1.3",
 		"cosmiconfig": "9.0.2",
 		"jsdom": "29.1.1"
 	},
-	"gitHead": "2cd462ea6b183f4462b7caba27b4db04aaba7c70"
+	"gitHead": "d6e1894a29aa29ca94785d9db0acfc3acc74764b"
 }