@nitpicker/core 0.22.2 → 0.23.0

package/lib/nitpicker.d.ts

@@ -71,7 +71,7 @@ export declare class Nitpicker extends EventEmitter<NitpickerEvent> {
      * declared concurrency or by {@link DEFAULT_CONCURRENCY}. This architecture
      * enables per-plugin progress tracking via Lanes.
      *
-     * The analysis proceeds in up to three phases:
+     * The analysis proceeds in up to two phases:
      *
      * 1. **`eachPage` phase** - For each plugin with `eachPage`, dispatches
      *    page analysis tasks to the plugin's worker pool. Progress is
@@ -81,32 +81,10 @@ export declare class Nitpicker extends EventEmitter<NitpickerEvent> {
      *    sequentially in the main thread. These are lightweight checks
      *    that don't need DOM access.
      *
-     * 3. **Template classification phase** (opt-in via
-     *    {@link AnalyzeOptions.classifyTemplates}) - Classifies every internal
-     *    HTML page into a template group by DOM-structure similarity. Unlike
-     *    phases 1-2 this is not a discovered `@nitpicker/analyze-*` plugin; it
-     *    runs once, globally, after every batch from phases 1-2 has been
-     *    accumulated (see `accumulatedPages` below for why it cannot run
-     *    per-batch).
-     *
      * On completion, data entries are stored in the archive:
      * - `analysis/report` - Full {@link Report} with headers and data
      * - `analysis/table` - The raw {@link Table} instance (serialized)
      * - SQL-backed analysis violation tables - Flat {@link Violation} records
-     * - `page_templates` SQL table - `templateKey` per page, written via
-     *   `Archive.replacePageTemplates` and never round-tripped through
-     *   `Table`/`Report`
-     *
-     * When `filter` resolves to zero plugins (only reachable when
-     * `classifyTemplates` is the sole reason for this call — see
-     * `cli/src/commands/analyze.ts`'s `--templates`-only bypass), this call
-     * contributes no plugin data of its own. To avoid silently wiping every
-     * column and violation from a *previous* run's plugins, this case skips
-     * `replaceAnalysisViolations` entirely and seeds the new `Table` from the
-     * archive's existing `analysis/report` (if any). This does not apply when
-     * one or more plugins ran: running a different plugin subset than a prior
-     * call has always fully replaced the report/table/violations with that
-     * subset's output, and that established contract is unchanged here.
      * @param filter - Optional list of plugin module names to run.
      *   If omitted, all configured plugins are executed.
      * @param options - Optional settings for progress display.

package/lib/nitpicker.js

@@ -8,7 +8,6 @@ import { getTableCacheRoot } from './get-table-cache-root.js';
 import { importModules } from './import-modules.js';
 import { loadPluginSettings } from './load-plugin-settings.js';
 import { Table } from './table.js';
-import { classifyPageTemplates } from './template-classification/classify-page-templates.js';
 import { UrlEventBus } from './url-event-bus.js';
 import { WorkerPool } from './worker/worker-pool.js';
 export { definePlugin } from './hooks/define-plugin.js';
@@ -31,34 +30,6 @@ const workerPath = path.resolve(__dirname, 'worker/worker.js');
  * Tracks the number of CPU cores so that worker pools scale with the host.
  */
 const DEFAULT_CONCURRENCY = Math.max(1, os.cpus().length);
-/**
- * Renders a `@d-zero/page-cluster` `ProgressEvent` as a short, human-readable
- * fragment for the Lanes template-classification lane, including a
- * done/total percentage wherever the event carries both halves of one
- * (matching the `N/M (X%)` convention every other lane in this file uses).
- * `pass0-signals` and `stage-b-start` carry only a running count with no
- * corpus-wide total, so those two render count-only.
- * @param event - Progress event forwarded from `resolvePageClusterKeys`.
- * @returns A short status fragment, e.g. `"42/100 blocks (42%)"`.
- */
-function formatTemplateClassificationProgress(event) {
-    switch (event.phase) {
-        case 'pass0-signals': {
-            return `reading pages (${event.pagesSeen})`;
-        }
-        case 'pass1-block-complete': {
-            const percent = Math.round((event.blocksProcessed / event.totalBlocks) * 100);
-            return `${event.blocksProcessed}/${event.totalBlocks} blocks (${percent}%)`;
-        }
-        case 'pass1b-assign': {
-            const percent = Math.round((event.pagesAssigned / event.pagesToAssign) * 100);
-            return `${event.pagesAssigned}/${event.pagesToAssign} pages (${percent}%)`;
-        }
-        case 'stage-b-start': {
-            return `merging ${event.unitCount} units`;
-        }
-    }
-}
 /**
  * Core orchestrator for running analyze plugins against a `.nitpicker` archive.
  *
@@ -149,7 +120,7 @@ export class Nitpicker extends EventEmitter {
      * declared concurrency or by {@link DEFAULT_CONCURRENCY}. This architecture
      * enables per-plugin progress tracking via Lanes.
      *
-     * The analysis proceeds in up to three phases:
+     * The analysis proceeds in up to two phases:
      *
      * 1. **`eachPage` phase** - For each plugin with `eachPage`, dispatches
      *    page analysis tasks to the plugin's worker pool. Progress is
@@ -159,32 +130,10 @@ export class Nitpicker extends EventEmitter {
      *    sequentially in the main thread. These are lightweight checks
      *    that don't need DOM access.
      *
-     * 3. **Template classification phase** (opt-in via
-     *    {@link AnalyzeOptions.classifyTemplates}) - Classifies every internal
-     *    HTML page into a template group by DOM-structure similarity. Unlike
-     *    phases 1-2 this is not a discovered `@nitpicker/analyze-*` plugin; it
-     *    runs once, globally, after every batch from phases 1-2 has been
-     *    accumulated (see `accumulatedPages` below for why it cannot run
-     *    per-batch).
-     *
      * On completion, data entries are stored in the archive:
      * - `analysis/report` - Full {@link Report} with headers and data
      * - `analysis/table` - The raw {@link Table} instance (serialized)
      * - SQL-backed analysis violation tables - Flat {@link Violation} records
-     * - `page_templates` SQL table - `templateKey` per page, written via
-     *   `Archive.replacePageTemplates` and never round-tripped through
-     *   `Table`/`Report`
-     *
-     * When `filter` resolves to zero plugins (only reachable when
-     * `classifyTemplates` is the sole reason for this call — see
-     * `cli/src/commands/analyze.ts`'s `--templates`-only bypass), this call
-     * contributes no plugin data of its own. To avoid silently wiping every
-     * column and violation from a *previous* run's plugins, this case skips
-     * `replaceAnalysisViolations` entirely and seeds the new `Table` from the
-     * archive's existing `analysis/report` (if any). This does not apply when
-     * one or more plugins ran: running a different plugin subset than a prior
-     * call has always fully replaced the report/table/violations with that
-     * subset's output, and that established contract is unchanged here.
      * @param filter - Optional list of plugin module names to run.
      *   If omitted, all configured plugins are executed.
      * @param options - Optional settings for progress display.
@@ -208,34 +157,6 @@ export class Nitpicker extends EventEmitter {
         const analyzeMods = await importModules(plugins);
         const lanes = options?.lanes;
         const table = new Table();
-        if (plugins.length === 0) {
-            // Only reachable via `--templates` with zero configured/selected
-            // plugins (see this method's JSDoc). Seed from whatever
-            // `analysis/report` already exists so this run — which contributes
-            // no plugin data of its own — doesn't wipe a previous run's columns.
-            try {
-                const previousReport = await this.archive.getData('analysis/report');
-                if (previousReport.pageData) {
-                    table.addHeaders(previousReport.pageData.headers);
-                    table.addData(previousReport.pageData.data);
-                }
-            }
-            catch (error) {
-                // ENOENT (no previous `analysis/report` — e.g. first analyze()
-                // call ever on this archive) is expected and starting from an
-                // empty table is correct. Anything else (corrupted JSON,
-                // permission error) is not silently swallowed: emit it so the
-                // user knows a previous run's data may not have been preserved,
-                // while still proceeding rather than blocking this run entirely.
-                if (error?.code !== 'ENOENT') {
-                    const message = error instanceof Error ? error.message : String(error);
-                    await this.emit('error', {
-                        message: `Failed to read previous analysis/report before a zero-plugin run: ${message}`,
-                        error: error instanceof Error ? error : null,
-                    });
-                }
-            }
-        }
         for (const mod of analyzeMods) {
             if (!mod.headers) {
                 continue;
@@ -267,32 +188,7 @@ export class Nitpicker extends EventEmitter {
             const label = pluginLabels.get(name) ?? name;
             lanes?.update(id, c.dim(`${label}: Waiting...`));
         }
-        // Template classification (opt-in, `--templates`) is a corpus-wide batch
-        // computation, not a per-page `AnalyzePlugin`, so it has its own lane
-        // outside `pluginLaneIds` and does not go through `analyzeMods`/`Worker`.
-        const templateClassificationLaneId = options?.classifyTemplates
-            ? eachPagePlugins.length
-            : null;
-        if (templateClassificationLaneId != null) {
-            lanes?.update(templateClassificationLaneId, c.dim('Template classification: Waiting...'));
-        }
-        // Accumulated across every `getPagesWithRefs` batch so template
-        // classification runs once, globally, after the loop below completes —
-        // never per-batch. Per-batch classification would produce template keys
-        // that are only comparable within their own batch (see
-        // `classifyPageTemplates`'s caller contract). `Page` instances are
-        // lightweight handles (HTML is fetched lazily via `getHtml()`), so
-        // accumulating every page across a 100,000-page batch size is not an
-        // OOM risk even for archives with hundreds of thousands of pages.
-        const accumulatedPages = [];
         await this.archive.getPagesWithRefs(100_000, async (pages) => {
-            if (options?.classifyTemplates) {
-                // Avoid `push(...pages)`: a 100,000-page batch can overflow V8's
-                // argument-spread limit even though the data itself fits in memory.
-                for (const page of pages) {
-                    accumulatedPages.push(page);
-                }
-            }
             const urlEmitter = new UrlEventBus();
             // Phase 1: eachPage plugins (sequentially, pages in parallel)
             for (const [pluginSeqIndex, { plugin, modIndex }] of eachPagePlugins.entries()) {
@@ -466,63 +362,13 @@ export class Nitpicker extends EventEmitter {
         }, {
             withRefs: false,
         });
-        // Phase 3: template classification (opt-in). Deliberately outside the
-        // `getPagesWithRefs` loop above — it must run once, globally, over every
-        // accumulated page, not once per 100,000-page batch (see
-        // `accumulatedPages`'s JSDoc comment).
-        //
-        // Wrapped in try/catch (unlike phases 1-2, whose failures are already
-        // caught per-page/per-mod) because this phase has no finer-grained
-        // unit to catch around: a failure here must not discard the
-        // already-computed `table`/`allViolations` from phases 1-2, which are
-        // persisted in the unconditional tail below regardless of this
-        // phase's outcome.
-        if (templateClassificationLaneId != null) {
-            try {
-                const classification = await classifyPageTemplates({
-                    archive: this.archive,
-                    pages: accumulatedPages,
-                    // Only worth paying for when there's a lane to update —
-                    // passing a defined callback at all (even a no-op one)
-                    // demotes `resolvePageClusterKeys` off its byte-identical,
-                    // yield-overhead-free sync path for corpora at or below its
-                    // inline threshold (see @d-zero/page-cluster's own docs).
-                    // `classifyPageTemplates` always requests cluster reasons too,
-                    // independent of this — see its own JSDoc for why that no
-                    // longer costs the sync-path demotion this comment warns about.
-                    onProgress: lanes
-                        ? (event) => {
-                            lanes.update(templateClassificationLaneId, `Template classification: ${formatTemplateClassificationProgress(event)}%braille%`);
-                        }
-                        : undefined,
-                });
-                if (classification.templateKeysByUrl.size > 0) {
-                    await this.archive.replacePageTemplates(classification.templateKeysByUrl, classification.clusterReasonsByTemplateKey);
-                }
-                lanes?.update(templateClassificationLaneId, c.green(`Template classification: Done (${classification.templateKeysByUrl.size} pages)`));
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                lanes?.update(templateClassificationLaneId, c.red(`Template classification: Failed (${message})`));
-                await this.emit('error', {
-                    message: `[template classification] ${message}`,
-                    error: error instanceof Error ? error : null,
-                });
-            }
-        }
         const report = {
             name: 'general',
             pageData: table.toJSON(),
         };
         await this.archive.setData('analysis/report', report);
         await this.archive.setData('analysis/table', table);
-        // See this method's JSDoc: a zero-plugin call (only reachable via
-        // `--templates` alone) contributes no violations of its own, so
-        // replacing the whole table with an empty array would silently erase
-        // every violation a previous run's plugins stored.
-        if (plugins.length > 0) {
-            await this.archive.replaceAnalysisViolations(allViolations);
-        }
+        await this.archive.replaceAnalysisViolations(allViolations);
     }
     /**
      * Loads and caches the plugin configuration from the user's config file.

package/lib/types.d.ts

@@ -44,17 +44,6 @@ export interface AnalyzeOptions {
     readonly lanes?: Lanes;
     /** When `true`, outputs plain-text progress lines instead of animated Lanes. */
     readonly verbose?: boolean;
-    /**
-     * When `true`, runs an additional, opt-in classification phase that
-     * groups internal HTML pages by DOM-structure similarity (using
-     * `@d-zero/page-cluster`), independent of the configured
-     * `@nitpicker/analyze-*` plugins. Results are added to the report table
-     * as a `templateKey` column. Defaults to `false` — the algorithm's
-     * runtime on very large archives is not yet characterized, so it is
-     * never run implicitly.
-     * @see {@link ./template-classification/classify-page-templates.ts!classifyPageTemplates}
-     */
-    readonly classifyTemplates?: boolean;
 }
 /**
  * CLI-specified overrides for individual plugin settings.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/core",
-	"version": "0.22.2",
+	"version": "0.23.0",
 	"description": "Plugin-based page analysis engine for Nitpicker",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -31,13 +31,12 @@
 	},
 	"dependencies": {
 		"@d-zero/dealer": "1.14.0",
-		"@d-zero/page-cluster": "0.6.4",
 		"@d-zero/shared": "0.23.0",
-		"@nitpicker/crawler": "0.22.2",
-		"@nitpicker/types": "0.22.0",
+		"@nitpicker/crawler": "0.23.0",
+		"@nitpicker/types": "0.23.0",
 		"ansi-colors": "4.1.3",
 		"cosmiconfig": "10.0.1",
 		"jsdom": "30.0.1"
 	},
-	"gitHead": "27db72abdebd19dc6f83a1b5087d73d8b574f229"
+	"gitHead": "c62b266e166b1a1e98f0572c8579012abb25e3a7"
 }

package/lib/template-classification/classify-page-templates.d.ts

@@ -1,49 +0,0 @@
-import type { ClassifyPageTemplatesOptions, PageTemplateClassification } from './types.js';
-/**
- * Classifies every internal HTML page in the archive into a template group
- * by DOM-structure similarity, using `@d-zero/page-cluster`.
- *
- * Result is cached, keyed by the archive file's path + size + mtime + page
- * count (see {@link buildArchiveCacheKey}). **This only helps one narrow
- * case**: a crash or Ctrl-C after this function completes but before
- * `nitpicker.write()` finishes (writing back a multi-gigabyte archive can
- * itself take a while) — re-running on the same, not-yet-rewritten archive
- * file hits the cache instead of re-running the whole multi-pass clustering.
- * It does **not** help a *second, fully successful* `analyze --templates`
- * run on the same crawled data: `nitpicker.write()` always re-packages the
- * archive (new mtime, and usually a different size), which changes the key
- * on every completed run. There is no explicit `--force` escape hatch —
- * any change to the archive changes the key, which is the correctness
- * signal this cache relies on. The cache is **not** cleared at the start of
- * a run (unlike the per-plugin `Cache` in `nitpicker.ts`) — its entire
- * purpose is surviving the one crash-before-write window described above.
- *
- * There is no mid-computation checkpoint/resume: `resolvePageClusterKeys`
- * holds state across its own internal multi-pass read and has no resume
- * API, so a crash *during* classification means the next run classifies
- * from scratch (this is a limitation of the upstream library, not something
- * this function works around).
- *
- * Internally: the external library's `resolvePageClusterKeys` returns a
- * `clusterKey` per page; this function's public naming is `templateKey`,
- * to avoid colliding with nitpicker's unrelated pre-existing "isolated
- * cluster" concept (`@nitpicker/query`'s `compute-isolated-clusters.ts`,
- * link-reachability graph components — a completely different kind of
- * grouping from DOM-structure similarity).
- *
- * `onClusterReason` is always passed to `resolvePageClusterKeys` (not made
- * conditional the way `onProgress` is) — as of `@d-zero/page-cluster` 0.5.3,
- * requesting cluster reasons no longer forces small corpora off the
- * progress-emitting path (see that option's own JSDoc), so there is no
- * responsiveness cost to always capturing them.
- * @param options - See {@link ClassifyPageTemplatesOptions}.
- * @returns See {@link PageTemplateClassification}.
- * @example
- * ```ts
- * const { templateKeysByUrl } = await classifyPageTemplates({ archive, pages });
- * for (const [url, templateKey] of templateKeysByUrl) {
- *   table.addData({ [url]: { templateKey: { value: templateKey } } });
- * }
- * ```
- */
-export declare function classifyPageTemplates(options: ClassifyPageTemplatesOptions): Promise<PageTemplateClassification>;

package/lib/template-classification/classify-page-templates.js

@@ -1,159 +0,0 @@
-import { stat } from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
-import { resolvePageClusterKeys } from '@d-zero/page-cluster/resolve-page-cluster-keys';
-import { Cache } from '@d-zero/shared/cache';
-import { collectPageStylesheetUrls } from './collect-page-stylesheet-urls.js';
-import { createPageClusterFactory } from './create-page-cluster-factory.js';
-/**
- * Narrows a loaded cache value to the current {@link TemplateClassificationCacheEntry}
- * shape, so a pre-existing on-disk cache entry from before cluster-reason
- * capture was added (a bare `Record<string, string>`, no `templateKeys` key)
- * is treated as a miss instead of being misread as `{ templateKeys: undefined,
- * clusterReasons: undefined }`. No version counter is used for this — the
- * structural check is sufficient and avoids a magic number that would need
- * bumping on every future cache-shape change.
- * @param value - The value loaded from the persistent cache.
- */
-function isTemplateClassificationCacheEntry(value) {
-    if (typeof value !== 'object' || value === null) {
-        return false;
-    }
-    const candidate = value;
-    return (typeof candidate.templateKeys === 'object' &&
-        candidate.templateKeys !== null &&
-        typeof candidate.clusterReasons === 'object' &&
-        candidate.clusterReasons !== null);
-}
-/**
- * Classifies every internal HTML page in the archive into a template group
- * by DOM-structure similarity, using `@d-zero/page-cluster`.
- *
- * Result is cached, keyed by the archive file's path + size + mtime + page
- * count (see {@link buildArchiveCacheKey}). **This only helps one narrow
- * case**: a crash or Ctrl-C after this function completes but before
- * `nitpicker.write()` finishes (writing back a multi-gigabyte archive can
- * itself take a while) — re-running on the same, not-yet-rewritten archive
- * file hits the cache instead of re-running the whole multi-pass clustering.
- * It does **not** help a *second, fully successful* `analyze --templates`
- * run on the same crawled data: `nitpicker.write()` always re-packages the
- * archive (new mtime, and usually a different size), which changes the key
- * on every completed run. There is no explicit `--force` escape hatch —
- * any change to the archive changes the key, which is the correctness
- * signal this cache relies on. The cache is **not** cleared at the start of
- * a run (unlike the per-plugin `Cache` in `nitpicker.ts`) — its entire
- * purpose is surviving the one crash-before-write window described above.
- *
- * There is no mid-computation checkpoint/resume: `resolvePageClusterKeys`
- * holds state across its own internal multi-pass read and has no resume
- * API, so a crash *during* classification means the next run classifies
- * from scratch (this is a limitation of the upstream library, not something
- * this function works around).
- *
- * Internally: the external library's `resolvePageClusterKeys` returns a
- * `clusterKey` per page; this function's public naming is `templateKey`,
- * to avoid colliding with nitpicker's unrelated pre-existing "isolated
- * cluster" concept (`@nitpicker/query`'s `compute-isolated-clusters.ts`,
- * link-reachability graph components — a completely different kind of
- * grouping from DOM-structure similarity).
- *
- * `onClusterReason` is always passed to `resolvePageClusterKeys` (not made
- * conditional the way `onProgress` is) — as of `@d-zero/page-cluster` 0.5.3,
- * requesting cluster reasons no longer forces small corpora off the
- * progress-emitting path (see that option's own JSDoc), so there is no
- * responsiveness cost to always capturing them.
- * @param options - See {@link ClassifyPageTemplatesOptions}.
- * @returns See {@link PageTemplateClassification}.
- * @example
- * ```ts
- * const { templateKeysByUrl } = await classifyPageTemplates({ archive, pages });
- * for (const [url, templateKey] of templateKeysByUrl) {
- *   table.addData({ [url]: { templateKey: { value: templateKey } } });
- * }
- * ```
- */
-export async function classifyPageTemplates(options) {
-    const { archive, pages, onProgress } = options;
-    const cache = getTemplateCache();
-    const cacheKey = await buildArchiveCacheKey(archive.filePath, pages.length);
-    if (cacheKey) {
-        const cached = await cache.load(cacheKey);
-        if (cached && isTemplateClassificationCacheEntry(cached)) {
-            return {
-                templateKeysByUrl: new Map(Object.entries(cached.templateKeys)),
-                clusterReasonsByTemplateKey: new Map(Object.entries(cached.clusterReasons)),
-            };
-        }
-    }
-    const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
-    const { factory, getYieldedUrls } = createPageClusterFactory(pages, stylesheetsByUrl);
-    const clusterReasonsByTemplateKey = new Map();
-    const clusterKeys = await resolvePageClusterKeys(factory, {
-        ...(onProgress ? { onProgress } : {}),
-        onClusterReason: (clusterKey, reason) => {
-            clusterReasonsByTemplateKey.set(clusterKey, reason);
-        },
-    });
-    // Safe only after `resolvePageClusterKeys` has resolved — see
-    // `createPageClusterFactory`'s JSDoc for why.
-    const yieldedUrls = getYieldedUrls();
-    if (clusterKeys.length !== yieldedUrls.length) {
-        // Would silently mis-map template keys to the wrong URLs if allowed
-        // through — see createPageClusterFactory's JSDoc for why this should
-        // be structurally impossible, but a loud failure here is far
-        // preferable to a quiet mis-assignment. No equivalent check exists for
-        // `clusterReasonsByTemplateKey` — see `PageTemplateClassification`'s
-        // own JSDoc for why that map is a best-effort side channel, not a
-        // per-page guarantee.
-        throw new Error(`classifyPageTemplates: resolvePageClusterKeys returned ${clusterKeys.length} keys for ${yieldedUrls.length} yielded pages.`);
-    }
-    const templateKeysByUrl = new Map();
-    for (const [index, url] of yieldedUrls.entries()) {
-        templateKeysByUrl.set(url, clusterKeys[index]);
-    }
-    if (cacheKey) {
-        await cache.store(cacheKey, {
-            templateKeys: Object.fromEntries(templateKeysByUrl),
-            clusterReasons: Object.fromEntries(clusterReasonsByTemplateKey),
-        });
-    }
-    return { templateKeysByUrl, clusterReasonsByTemplateKey };
-}
-/**
- * Lazily constructs the persistent template-classification cache. Kept as a
- * function (rather than a module-level constant) so tests can point it at
- * an isolated tmpDir-per-test without module-mocking `os.tmpdir`.
- */
-function getTemplateCache() {
-    return new Cache('nitpicker-templates', path.join(os.tmpdir(), 'nitpicker/cache/templates'));
-}
-/**
- * Derives a cache key from the archive file's identity: its absolute path,
- * byte size, mtime, and internal HTML page count. Byte size + mtime alone is
- * a coarse signal — a same-path overwrite that happens to land on the same
- * byte size within the filesystem's mtime tick (coarse mtime resolution,
- * or two writes close enough together) would otherwise be indistinguishable
- * from "unchanged". Page count is folded in as a cheap (already-computed,
- * no extra query), second independent signal that most re-crawls
- * (`--append` / `--inventory`) change; it does **not** catch every
- * possible content change (e.g. `--retry-failed` replacing a failed page's
- * content without changing the total page count), which is an accepted,
- * documented gap rather than an attempt at a full content hash — hashing
- * every page's HTML on every `analyze()` call would cost more than the
- * cache is meant to save.
- * @param archiveFilePath - `Archive.filePath` of the archive being analyzed.
- * @param pageCount - Number of pages passed to {@link classifyPageTemplates}
- *   for this run (`pages.length`, before internal/HTML filtering).
- * @returns The cache key, or `null` if the file's stats cannot be read
- *   (falls back to always recomputing rather than caching under the wrong
- *   key).
- */
-async function buildArchiveCacheKey(archiveFilePath, pageCount) {
-    try {
-        const stats = await stat(archiveFilePath);
-        return `${archiveFilePath}:${stats.size}:${stats.mtimeMs}:${pageCount}`;
-    }
-    catch {
-        return null;
-    }
-}

package/lib/template-classification/collect-page-stylesheet-urls.d.ts

@@ -1,32 +0,0 @@
-import type { Archive } from '@nitpicker/crawler';
-/**
- * Collects, for every page that references at least one CSS stylesheet, the
- * set of stylesheet URLs it references.
- *
- * Reads in three narrow passes instead of one wide JOIN, to avoid
- * materializing the full page×stylesheet edge set at once — a real
- * 486,000-page archive has ~2.5 million such edges, and a single JOIN
- * projecting both URL strings for all of them is a transient allocation
- * spike well beyond what any single page's data actually needs:
- *
- * 1. Resolve which `resource_items` rows are classified `'css'`
- *    (`content_type_refs.category`) and their stylesheet URL.
- * 2. Resolve `resource_ref_edges` for just those CSS resource ids, in
- *    chunks, projecting only the two integer ids per row.
- * 3. Resolve the page ids collected in step 2 to their URLs.
- *
- * Identical stylesheet-URL arrays are interned across pages: pages sharing a
- * template share the same CSS set, so the number of distinct arrays
- * converges to a few hundred even across hundreds of thousands of pages.
- * @param archive - The archive to read from.
- * @returns Map from page URL (parseUrl-normalized `href`, matching
- *   `Page.url.href`) to the stylesheet URLs it references. Pages that
- *   reference no CSS resource have no entry — callers should default to an
- *   empty array on a missed lookup.
- * @example
- * ```ts
- * const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
- * const hrefs = stylesheetsByUrl.get(page.url.href) ?? [];
- * ```
- */
-export declare function collectPageStylesheetUrls(archive: Archive): Promise<Map<string, readonly string[]>>;

package/lib/template-classification/collect-page-stylesheet-urls.js

@@ -1,130 +0,0 @@
-import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
-import { eachSplitted } from '@nitpicker/crawler';
-/**
- * SQLite has a hard cap of 999 bound variables per statement. A real archive
- * can have far more than 999 CSS-referencing edges or pages, so every
- * `whereIn` below is chunked at this size.
- *
- * Independently duplicated (not shared/imported) from `@nitpicker/query`'s
- * `compute-isolated-clusters.ts`, which defines the identical
- * `SQLITE_IN_CHUNK = 500` for the same reason. If this margin below 999 ever
- * needs revisiting, update both — there is no shared module either package
- * depends on to hold one copy.
- */
-const SQLITE_IN_CHUNK = 500;
-/**
- * Collects, for every page that references at least one CSS stylesheet, the
- * set of stylesheet URLs it references.
- *
- * Reads in three narrow passes instead of one wide JOIN, to avoid
- * materializing the full page×stylesheet edge set at once — a real
- * 486,000-page archive has ~2.5 million such edges, and a single JOIN
- * projecting both URL strings for all of them is a transient allocation
- * spike well beyond what any single page's data actually needs:
- *
- * 1. Resolve which `resource_items` rows are classified `'css'`
- *    (`content_type_refs.category`) and their stylesheet URL.
- * 2. Resolve `resource_ref_edges` for just those CSS resource ids, in
- *    chunks, projecting only the two integer ids per row.
- * 3. Resolve the page ids collected in step 2 to their URLs.
- *
- * Identical stylesheet-URL arrays are interned across pages: pages sharing a
- * template share the same CSS set, so the number of distinct arrays
- * converges to a few hundred even across hundreds of thousands of pages.
- * @param archive - The archive to read from.
- * @returns Map from page URL (parseUrl-normalized `href`, matching
- *   `Page.url.href`) to the stylesheet URLs it references. Pages that
- *   reference no CSS resource have no entry — callers should default to an
- *   empty array on a missed lookup.
- * @example
- * ```ts
- * const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
- * const hrefs = stylesheetsByUrl.get(page.url.href) ?? [];
- * ```
- */
-export async function collectPageStylesheetUrls(archive) {
-    const knex = archive.getKnex();
-    const cssResources = (await knex('resource_items as ri')
-        .join('content_type_refs as ctr', 'ctr.id', 'ri.content_type_id')
-        .where('ctr.category', 'css')
-        .select('ri.id as id', 'ri.url_id as urlId'));
-    // A CSS resource identified by a `data:` URI (routed to `blob_refs`
-    // instead of `url_refs`, `resource_items.url_blob_id`) has no stylesheet
-    // URL to report — skip it rather than passing a `null` into `whereIn`.
-    const cssResourcesWithUrl = cssResources.filter((r) => r.urlId != null);
-    if (cssResourcesWithUrl.length === 0) {
-        return new Map();
-    }
-    const cssUrlById = new Map();
-    await eachSplitted(cssResourcesWithUrl.map((r) => r.urlId), SQLITE_IN_CHUNK, async (chunk) => {
-        const rows = (await knex('url_refs').whereIn('id', chunk).select('id', 'url'));
-        for (const row of rows) {
-            cssUrlById.set(row.id, row.url);
-        }
-    });
-    const cssUrlByResourceId = new Map();
-    for (const r of cssResourcesWithUrl) {
-        const url = cssUrlById.get(r.urlId);
-        if (url) {
-            cssUrlByResourceId.set(r.id, url);
-        }
-    }
-    const resourceIdsByPageId = new Map();
-    await eachSplitted(cssResourcesWithUrl.map((r) => r.id), SQLITE_IN_CHUNK, async (chunk) => {
-        const rows = (await knex('resource_ref_edges')
-            .whereIn('resource_id', chunk)
-            .select('resource_id as resourceId', 'page_id as pageId'));
-        // Avoid `push(...rows)`: on large real archives this chunk array can
-        // be large enough to overflow V8's argument-spread limit even though
-        // the underlying data itself fits in memory.
-        for (const row of rows) {
-            const existing = resourceIdsByPageId.get(row.pageId);
-            if (existing) {
-                existing.push(row.resourceId);
-            }
-            else {
-                resourceIdsByPageId.set(row.pageId, [row.resourceId]);
-            }
-        }
-    });
-    if (resourceIdsByPageId.size === 0) {
-        return new Map();
-    }
-    const pageUrlById = new Map();
-    await eachSplitted([...resourceIdsByPageId.keys()], SQLITE_IN_CHUNK, async (chunk) => {
-        const rows = (await knex('content_items as ci')
-            .join('url_refs as ur', 'ur.id', 'ci.url_id')
-            .whereIn('ci.id', chunk)
-            .select('ci.id as id', 'ur.url as url'));
-        for (const row of rows) {
-            pageUrlById.set(row.id, row.url);
-        }
-    });
-    const internedSets = new Map();
-    const result = new Map();
-    for (const [pageId, resourceIds] of resourceIdsByPageId) {
-        const pageUrlRaw = pageUrlById.get(pageId);
-        if (!pageUrlRaw) {
-            continue;
-        }
-        const parsed = parseUrl(pageUrlRaw);
-        if (!parsed) {
-            continue;
-        }
-        const urls = resourceIds
-            .map((id) => cssUrlByResourceId.get(id))
-            .filter((url) => url != null)
-            .toSorted();
-        if (urls.length === 0) {
-            continue;
-        }
-        const internKey = urls.join(' ');
-        let interned = internedSets.get(internKey);
-        if (!interned) {
-            interned = Object.freeze(urls);
-            internedSets.set(internKey, interned);
-        }
-        result.set(parsed.href, interned);
-    }
-    return result;
-}

package/lib/template-classification/create-page-cluster-factory.d.ts

@@ -1,55 +0,0 @@
-import type { PageFactory } from '@d-zero/page-cluster/resolve-page-cluster-keys';
-import type { Page } from '@nitpicker/crawler';
-/**
- * Builds the `PageFactory` that `@d-zero/page-cluster`'s
- * `resolvePageClusterKeys` reads the corpus through, plus a way to read back
- * the URL list it actually yielded (see below for why that can't be known
- * synchronously up front).
- *
- * Only internal HTML pages (`Page.isInternalPage()` — content-type based,
- * matching the "a page is judged by content-type, not `isTarget`" archive
- * invariant) with retrievable HTML are yielded. A page can be an internal
- * HTML page and still have no retrievable HTML (e.g. a skipped or
- * self-heal-affected row) — `page.getHtml()` returning falsy is only known
- * by actually calling it, not from any synchronous field.
- *
- * `resolvePageClusterKeys` calls the returned factory one or more times
- * (once for corpora at or below its inline threshold, twice above it — see
- * its own docs) to re-read the corpus. Whichever pages end up excluded
- * (empty HTML) must be excluded **identically** on every call, or the
- * pass-to-pass page sequence drifts and the library's same-order contract
- * breaks. Excluding by `isInternalPage()` is safe (synchronous,
- * deterministic per page). Excluding by empty `getHtml()` is also safe,
- * but only because the archive is immutable for the duration of one
- * `analyze()` run — the same page's HTML never changes between calls.
- *
- * `getYieldedUrls()` reflects the URLs yielded by the **most recently
- * completed** full iteration of the factory's generator, deliberately not a
- * separate up-front probe pass over every candidate's HTML: every documented
- * call shape of `resolvePageClusterKeys` fully drains the factory it's given
- * (that's what "reading the factory" means in its own docs), so by the time
- * `resolvePageClusterKeys` resolves, at least one full drain has already
- * happened and `getYieldedUrls()` reflects it — a dedicated probe pass would
- * only add a full corpus-wide decompression sweep beyond the library's own
- * 1-2, for no benefit. `classifyPageTemplates` still defends against a
- * violated assumption here with a hard length check against
- * `resolvePageClusterKeys`'s result array.
- * @param pages - Candidate pages (as already loaded by `Nitpicker.analyze()`
- *   — this function does not re-query the archive).
- * @param stylesheetsByUrl - Output of `collectPageStylesheetUrls`, keyed by
- *   `page.url.href`.
- * @returns `factory` to pass to `resolvePageClusterKeys`, and
- *   `getYieldedUrls()` — call this only *after* `resolvePageClusterKeys`
- *   resolves; its result array has exactly the length and order of
- *   `getYieldedUrls()` at that point.
- * @example
- * ```ts
- * const { factory, getYieldedUrls } = createPageClusterFactory(pages, stylesheetsByUrl);
- * const clusterKeys = await resolvePageClusterKeys(factory);
- * const yieldedUrls = getYieldedUrls(); // safe to read now
- * ```
- */
-export declare function createPageClusterFactory(pages: readonly Page[], stylesheetsByUrl: ReadonlyMap<string, readonly string[]>): {
-    factory: PageFactory;
-    getYieldedUrls: () => readonly string[];
-};

package/lib/template-classification/create-page-cluster-factory.js

@@ -1,88 +0,0 @@
-/**
- * Builds the `PageFactory` that `@d-zero/page-cluster`'s
- * `resolvePageClusterKeys` reads the corpus through, plus a way to read back
- * the URL list it actually yielded (see below for why that can't be known
- * synchronously up front).
- *
- * Only internal HTML pages (`Page.isInternalPage()` — content-type based,
- * matching the "a page is judged by content-type, not `isTarget`" archive
- * invariant) with retrievable HTML are yielded. A page can be an internal
- * HTML page and still have no retrievable HTML (e.g. a skipped or
- * self-heal-affected row) — `page.getHtml()` returning falsy is only known
- * by actually calling it, not from any synchronous field.
- *
- * `resolvePageClusterKeys` calls the returned factory one or more times
- * (once for corpora at or below its inline threshold, twice above it — see
- * its own docs) to re-read the corpus. Whichever pages end up excluded
- * (empty HTML) must be excluded **identically** on every call, or the
- * pass-to-pass page sequence drifts and the library's same-order contract
- * breaks. Excluding by `isInternalPage()` is safe (synchronous,
- * deterministic per page). Excluding by empty `getHtml()` is also safe,
- * but only because the archive is immutable for the duration of one
- * `analyze()` run — the same page's HTML never changes between calls.
- *
- * `getYieldedUrls()` reflects the URLs yielded by the **most recently
- * completed** full iteration of the factory's generator, deliberately not a
- * separate up-front probe pass over every candidate's HTML: every documented
- * call shape of `resolvePageClusterKeys` fully drains the factory it's given
- * (that's what "reading the factory" means in its own docs), so by the time
- * `resolvePageClusterKeys` resolves, at least one full drain has already
- * happened and `getYieldedUrls()` reflects it — a dedicated probe pass would
- * only add a full corpus-wide decompression sweep beyond the library's own
- * 1-2, for no benefit. `classifyPageTemplates` still defends against a
- * violated assumption here with a hard length check against
- * `resolvePageClusterKeys`'s result array.
- * @param pages - Candidate pages (as already loaded by `Nitpicker.analyze()`
- *   — this function does not re-query the archive).
- * @param stylesheetsByUrl - Output of `collectPageStylesheetUrls`, keyed by
- *   `page.url.href`.
- * @returns `factory` to pass to `resolvePageClusterKeys`, and
- *   `getYieldedUrls()` — call this only *after* `resolvePageClusterKeys`
- *   resolves; its result array has exactly the length and order of
- *   `getYieldedUrls()` at that point.
- * @example
- * ```ts
- * const { factory, getYieldedUrls } = createPageClusterFactory(pages, stylesheetsByUrl);
- * const clusterKeys = await resolvePageClusterKeys(factory);
- * const yieldedUrls = getYieldedUrls(); // safe to read now
- * ```
- */
-export function createPageClusterFactory(pages, stylesheetsByUrl) {
-    const candidates = pages.filter((page) => page.isInternalPage());
-    let yieldedUrls = [];
-    const factory = () => yieldPageClusterSignals(candidates, stylesheetsByUrl, (urls) => {
-        yieldedUrls = urls;
-    });
-    return { factory, getYieldedUrls: () => yieldedUrls };
-}
-/**
- * The actual per-call iterator body for {@link createPageClusterFactory}'s
- * `PageFactory`. HTML is fetched lazily, one page at a time, so the archive
- * never holds more than one page's decompressed HTML in memory regardless
- * of corpus size — matching `@d-zero/page-cluster`'s own memory-bounding
- * design for its multi-pass reads.
- * @param candidates - Pre-filtered internal HTML pages, in fixed order.
- * @param stylesheetsByUrl - See {@link createPageClusterFactory}.
- * @param onDrained - Called with every yielded URL, in order, once this
- *   particular iteration reaches the end of `candidates` — never called if
- *   the consumer stops iterating early.
- * @yields {PageClusterSignals} One per candidate page with retrievable HTML.
- */
-async function* yieldPageClusterSignals(candidates, stylesheetsByUrl, onDrained) {
-    const yieldedUrls = [];
-    for (const page of candidates) {
-        const html = await page.getHtml();
-        if (!html) {
-            continue;
-        }
-        const url = page.url;
-        yieldedUrls.push(url.href);
-        yield {
-            paths: url.paths,
-            stylesheetHrefs: stylesheetsByUrl.get(url.href) ?? [],
-            html,
-            host: url.port ? `${url.hostname}:${url.port}` : url.hostname,
-        };
-    }
-    onDrained(yieldedUrls);
-}

package/lib/template-classification/types.d.ts

@@ -1,44 +0,0 @@
-import type { ProgressEvent } from '@d-zero/page-cluster/resolve-page-cluster-keys';
-import type { Archive, Page, TemplateClusterReason } from '@nitpicker/crawler';
-/**
- * Options for {@link import('./classify-page-templates.js').classifyPageTemplates}.
- */
-export interface ClassifyPageTemplatesOptions {
-    /** The archive to read pages/stylesheets from and to derive the cache key from. */
-    archive: Archive;
-    /**
-     * All pages already loaded for this `analyze()` run (see
-     * `Nitpicker.analyze`'s accumulation across `getPagesWithRefs` batches —
-     * classification must run once, globally, not per batch).
-     */
-    pages: readonly Page[];
-    /**
-     * Progress callback forwarded to `@d-zero/page-cluster`'s
-     * `resolvePageClusterKeys`, so long-running classification on large
-     * archives isn't silently unresponsive. Independent of cluster-reason
-     * capture — `@d-zero/page-cluster` 0.5.3+ composes `onProgress` and
-     * `onClusterReason` without either one demoting the corpus off its
-     * progress-emitting path (see `classifyPageTemplates`'s own JSDoc).
-     */
-    onProgress?: (event: ProgressEvent) => void;
-}
-/**
- * Result of {@link import('./classify-page-templates.js').classifyPageTemplates}.
- */
-export interface PageTemplateClassification {
-    /**
-     * Page URL (`page.url.href`) → template key. Only internal HTML pages
-     * with retrievable HTML have an entry.
-     */
-    readonly templateKeysByUrl: ReadonlyMap<string, string>;
-    /**
-     * Template key → `@d-zero/page-cluster`'s cluster-selection evidence for
-     * that key. **Not guaranteed to cover every key in `templateKeysByUrl`**:
-     * `@d-zero/page-cluster` only emits a reason for a final cluster it still
-     * holds full grouping state for at the moment `onClusterReason` fires,
-     * which is a best-effort side channel rather than a per-page guarantee
-     * (unlike `templateKeysByUrl`, which is verified 1:1 against the yielded
-     * page set — see the hard length check in `classifyPageTemplates`).
-     */
-    readonly clusterReasonsByTemplateKey: ReadonlyMap<string, TemplateClusterReason>;
-}

package/lib/template-classification/types.js

@@ -1 +0,0 @@
-export {};