@nitpicker/archive 0.0.0 → 0.25.0

package/lib/retarget-legacy-fk-tables.js

@@ -0,0 +1,137 @@
+import { createAdjunctTables } from './create-adjunct-tables.js';
+import { convertLegacyPageTagsToInserts } from './meta/technologies/convert-legacy-page-tags-to-inserts.js';
+import { eachSplitted } from './utils/array/each-splitted.js';
+/**
+ * The adjunct tables whose FK declarations may still point at the legacy
+ * `pages(id)` on migrated archives. Only names are listed — the column
+ * set is derived at runtime from the canonical table recreated by
+ * {@link createAdjunctTables}, so a future column addition there cannot
+ * silently drift from a hardcoded list (a missing column in the staged
+ * copy fails the copy-back INSERT loudly instead of dropping data).
+ *
+ * `page_tags` is deliberately NOT here — it has no current-schema
+ * equivalent to retarget onto (removed in favor of `technology_signals` /
+ * `page_technologies`). {@link retargetLegacyFkTables} handles it as a
+ * special case: convert its rows, then drop it outright rather than
+ * recreating it. See that function's JSDoc.
+ *
+ * `analysis_violations` / `analysis_text_refs` are likewise not listed: the
+ * current schema has no equivalent, so {@link retargetLegacyFkTables} drops
+ * them (and their rows) instead of retargeting.
+ */
+const RETARGET_TABLES = [
+    'page_html_ref',
+    'page_jsonld',
+    'page_errors',
+];
+/**
+ * Rewrites the FK declarations of the adjunct tables from the legacy
+ * `pages(id)` to `content_items(id)` by rebuilding each table, converts
+ * `page_tags` (if present) into `technology_signals` / `page_technologies`,
+ * AND drops the legacy `analysis_violations` / `analysis_text_refs` tables
+ * (analyze output that that no reader or writer uses).
+ * SQLite has no `ALTER TABLE … DROP CONSTRAINT`, so the only way to change
+ * an FK target is to recreate the table:
+ *
+ * 1. Stage each {@link RETARGET_TABLES} table's rows into a constraint-free
+ *    `<table>__retarget` copy (`CREATE TABLE … AS SELECT *`) and drop the
+ *    original, freeing its index names. `page_tags`, if present, is read
+ *    into memory here too (converted via `convertLegacyPageTagsToInserts` —
+ *    the same helper `migrate-page-tags-to-page-technologies.ts` uses for
+ *    already-0.13+ archives) but NOT staged the same way: it has no
+ *    current-schema table to retarget onto, so there is nothing to copy
+ *    back — only something to drop, once its data lives in the tables
+ *    from step 2.
+ * 2. Recreate every {@link RETARGET_TABLES} table through
+ *    {@link createAdjunctTables} — the same DDL fresh archives get, so
+ *    migrated archives end up identical in shape and index naming.
+ *    Rebuilding the DDL inline here instead would recreate the exact
+ *    drift this function repairs. This same call also creates
+ *    `technology_signals` / `page_technologies` (idempotent
+ *    `hasTable`-guarded DDL — a no-op if a prior partial run already
+ *    created them).
+ * 3. Copy the staged {@link RETARGET_TABLES} rows back using the recreated
+ *    table's own column list (read via `pragma_table_info`). A column the
+ *    canonical DDL has but the input archive lacks aborts the INSERT with
+ *    a clear error — never a silent data drop. The converted `page_tags`
+ *    signals/technologies are inserted here too, then `page_tags` itself
+ *    is dropped (not recreated).
+ *
+ * A rename-based recipe (`ALTER TABLE x RENAME TO x__retarget`, skip
+ * copy #1) is deliberately NOT used: SQLite keeps the renamed table's
+ * indexes attached under their original names, which collide with the
+ * index names {@link createAdjunctTables} declares — and the old index
+ * names vary by the era of the archive's original provisioning, so
+ * dropping them first would require enumerating unknown names. The
+ * copy-out is cheap at these tables' scale (a handful of rows per page).
+ *
+ * Adjunct tables that do not exist on the input archive are simply created
+ * empty by step 2.
+ *
+ * The migrated-archive PK-preservation contract makes the copy safe:
+ * `content_items.id` reuses the legacy `pages.id` values verbatim, so
+ * every staged `pageId` / `page_id` resolves to the same row under the
+ * new FK target. Run inside a transaction with `PRAGMA foreign_keys = ON`
+ * so the copy-back doubles as a row-level FK validation — a stale id
+ * aborts the transaction here rather than surfacing later as a
+ * `foreign_key_check` finding.
+ * @param trx - Knex transaction on the archive being migrated
+ *   (`scripts/migrate-to-0.13.mjs` is the caller).
+ * @example
+ * await db.transaction(async (trx) => {
+ *   await retargetLegacyFkTables(trx);
+ * });
+ * // pragma_foreign_key_list('page_errors') now reports content_items.
+ */
+export async function retargetLegacyFkTables(trx) {
+    // Legacy analyze output: no current-schema counterpart, so the rows are
+    // discarded. The child (`analysis_violations`, which references
+    // `analysis_text_refs`) goes first so the drop is legal under
+    // `PRAGMA foreign_keys = ON`.
+    await trx.raw('DROP TABLE IF EXISTS "analysis_violations"');
+    await trx.raw('DROP TABLE IF EXISTS "analysis_text_refs"');
+    const staged = [];
+    for (const table of RETARGET_TABLES) {
+        if (!(await trx.schema.hasTable(table))) {
+            continue;
+        }
+        await trx.raw(`CREATE TABLE "${table}__retarget" AS SELECT * FROM "${table}"`);
+        await trx.raw(`DROP TABLE "${table}"`);
+        staged.push(table);
+    }
+    // `page_tags` has no current-schema table to retarget onto (removed —
+    // see the module JSDoc), so its rows are converted to
+    // `technology_signals` / `page_technologies` shape HERE, while
+    // `page_tags` still exists, then only dropped (not staged/recreated)
+    // after `createAdjunctTables` below provisions the tables that will
+    // receive them.
+    const hasLegacyPageTags = await trx.schema.hasTable('page_tags');
+    const legacyTagRows = hasLegacyPageTags
+        ? await trx
+            .select('pageId', 'provider', 'version', 'confidence', 'categories')
+            .from('page_tags')
+        : [];
+    await createAdjunctTables(trx);
+    for (const table of staged) {
+        const columns = await trx
+            .select('name')
+            .from(trx.raw('pragma_table_info(?)', [table]));
+        const insertColumnList = columns.map((column) => `"${column.name}"`).join(', ');
+        await trx.raw(`INSERT INTO "${table}" (${insertColumnList}) SELECT ${insertColumnList} FROM "${table}__retarget"`);
+        await trx.raw(`DROP TABLE "${table}__retarget"`);
+    }
+    if (hasLegacyPageTags) {
+        const { signalInserts, technologyInserts } = convertLegacyPageTagsToInserts(legacyTagRows);
+        if (signalInserts.length > 0) {
+            await eachSplitted(signalInserts, 100, async (chunk) => {
+                await trx('technology_signals').insert(chunk);
+            });
+        }
+        if (technologyInserts.length > 0) {
+            await eachSplitted(technologyInserts, 100, async (chunk) => {
+                await trx('page_technologies').insert(chunk);
+            });
+        }
+        await trx.raw('DROP TABLE "page_tags"');
+    }
+}

package/lib/safe-path.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Resolves and validates a file path to prevent path traversal attacks.
+ * Ensures the resolved path stays within the specified base directory.
+ * @param base - The base directory that all paths must stay within.
+ * @param segments - Path segments to resolve relative to the base.
+ * @returns The resolved absolute path.
+ * @throws {Error} If the resolved path escapes the base directory.
+ */
+export declare function safePath(base: string, ...segments: string[]): string;

package/lib/safe-path.js

@@ -0,0 +1,17 @@
+import path from 'node:path';
+/**
+ * Resolves and validates a file path to prevent path traversal attacks.
+ * Ensures the resolved path stays within the specified base directory.
+ * @param base - The base directory that all paths must stay within.
+ * @param segments - Path segments to resolve relative to the base.
+ * @returns The resolved absolute path.
+ * @throws {Error} If the resolved path escapes the base directory.
+ */
+export function safePath(base, ...segments) {
+    const resolvedBase = path.resolve(base);
+    const resolved = path.resolve(base, ...segments);
+    if (!resolved.startsWith(resolvedBase + path.sep) && resolved !== resolvedBase) {
+        throw new Error(`Path traversal detected: ${segments.join('/')}`);
+    }
+    return resolved;
+}

package/lib/scope/build-scope-map.d.ts

@@ -0,0 +1,21 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Builds a hostname-indexed scope map from a list of root URL strings, for
+ * use with {@link findScopeEntry}.
+ *
+ * Each root that fails to parse is silently skipped (a malformed root can
+ * never match anything anyway) rather than throwing, matching the existing
+ * inline construction this replaces (`@nitpicker/crawler`'s `crawler-orchestrator.ts` `append`
+ * and inventory-classification call sites, `Crawler`'s own scope build).
+ * Roots sharing a hostname are grouped under one array entry so
+ * `findScopeEntry` can pick the most specific match among them.
+ * @param roots - Root URL strings (e.g. `Config.roots`).
+ * @param options - URL parsing options forwarded to {@link parseUrl}.
+ * @returns A hostname-indexed map of parsed root URLs.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(['https://example.com/blog/', 'https://example.com/docs/']);
+ * findScopeEntry(url, scope); // deepest matching root, or null if external
+ * ```
+ */
+export declare function buildScopeMap(roots: readonly string[], options?: ParseURLOptions): Map<string, ExURL[]>;

package/lib/scope/build-scope-map.js

@@ -0,0 +1,32 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+/**
+ * Builds a hostname-indexed scope map from a list of root URL strings, for
+ * use with {@link findScopeEntry}.
+ *
+ * Each root that fails to parse is silently skipped (a malformed root can
+ * never match anything anyway) rather than throwing, matching the existing
+ * inline construction this replaces (`@nitpicker/crawler`'s `crawler-orchestrator.ts` `append`
+ * and inventory-classification call sites, `Crawler`'s own scope build).
+ * Roots sharing a hostname are grouped under one array entry so
+ * `findScopeEntry` can pick the most specific match among them.
+ * @param roots - Root URL strings (e.g. `Config.roots`).
+ * @param options - URL parsing options forwarded to {@link parseUrl}.
+ * @returns A hostname-indexed map of parsed root URLs.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(['https://example.com/blog/', 'https://example.com/docs/']);
+ * findScopeEntry(url, scope); // deepest matching root, or null if external
+ * ```
+ */
+export function buildScopeMap(roots, options) {
+    const scope = new Map();
+    for (const raw of roots) {
+        const parsed = parseUrl(raw, options);
+        if (!parsed) {
+            continue;
+        }
+        const existing = scope.get(parsed.hostname) ?? [];
+        scope.set(parsed.hostname, [...existing, parsed]);
+    }
+    return scope;
+}

package/lib/scope/find-scope-entry.d.ts

@@ -0,0 +1,25 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+/**
+ * Find the most-specific scope entry that contains the given URL.
+ *
+ * A scope entry is an `(hostname, port, path)` triple. The target URL belongs
+ * to a scope entry when its hostname AND port match and its path is at the
+ * same level or deeper than the scope entry's path. Among all matching entries,
+ * the one with the greatest depth wins (e.g. `/blog/2024/` is preferred over
+ * `/blog/`). Port comparison uses the WHATWG-normalized `port` field, so
+ * default ports (`80` for http, `443` for https) collapse to an empty string
+ * and match each other regardless of whether the user wrote them explicitly.
+ * A non-default port like `:3000` only matches scope entries that also carry
+ * the same port — this prevents credentialed dev sites (`localhost:3000`)
+ * from leaking auth into siblings on the same hostname (`localhost:8080`).
+ *
+ * Returns `null` if no scope entry contains the URL — i.e. the URL is external.
+ * This single function replaces the previous trio of `isExternalUrl`,
+ * `isInAnyLowerLayer`, and `findBestMatchingScope`, performing a single
+ * hostname lookup and a single pass over the scope array.
+ * @param url - The target URL to classify.
+ * @param scope - Hostname-indexed map of scope URLs.
+ * @param options - URL parsing options forwarded to {@link isLowerLayer}.
+ * @returns The deepest matching scope URL, or `null` when the URL is external.
+ */
+export declare function findScopeEntry(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): ExURL | null;

package/lib/scope/find-scope-entry.js

@@ -0,0 +1,45 @@
+import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
+/**
+ * Find the most-specific scope entry that contains the given URL.
+ *
+ * A scope entry is an `(hostname, port, path)` triple. The target URL belongs
+ * to a scope entry when its hostname AND port match and its path is at the
+ * same level or deeper than the scope entry's path. Among all matching entries,
+ * the one with the greatest depth wins (e.g. `/blog/2024/` is preferred over
+ * `/blog/`). Port comparison uses the WHATWG-normalized `port` field, so
+ * default ports (`80` for http, `443` for https) collapse to an empty string
+ * and match each other regardless of whether the user wrote them explicitly.
+ * A non-default port like `:3000` only matches scope entries that also carry
+ * the same port — this prevents credentialed dev sites (`localhost:3000`)
+ * from leaking auth into siblings on the same hostname (`localhost:8080`).
+ *
+ * Returns `null` if no scope entry contains the URL — i.e. the URL is external.
+ * This single function replaces the previous trio of `isExternalUrl`,
+ * `isInAnyLowerLayer`, and `findBestMatchingScope`, performing a single
+ * hostname lookup and a single pass over the scope array.
+ * @param url - The target URL to classify.
+ * @param scope - Hostname-indexed map of scope URLs.
+ * @param options - URL parsing options forwarded to {@link isLowerLayer}.
+ * @returns The deepest matching scope URL, or `null` when the URL is external.
+ */
+export function findScopeEntry(url, scope, options) {
+    const scopes = scope.get(url.hostname);
+    if (!scopes) {
+        return null;
+    }
+    let bestMatch = null;
+    let maxDepth = -1;
+    for (const entry of scopes) {
+        if (entry.port !== url.port) {
+            continue;
+        }
+        if (!isLowerLayer(url, entry, options)) {
+            continue;
+        }
+        if (entry.depth > maxDepth) {
+            bestMatch = entry;
+            maxDepth = entry.depth;
+        }
+    }
+    return bestMatch;
+}

package/lib/template-classification/classify-archive-page-templates.d.ts

@@ -0,0 +1,42 @@
+import type { ClassifyArchivePageTemplatesOptions, ClassifyArchivePageTemplatesResult } from './types.js';
+import type Archive from '../archive.js';
+/**
+ * Classifies every internal HTML page of the archive into a template group
+ * by DOM-structure similarity and persists the result to `page_templates`,
+ * `page_template_clusters` and `page_template_labels`.
+ *
+ * This is the crawl-end derived-data step (alongside the JS-resource scan
+ * and the viewer read-model build): the crawl command runs it before the
+ * archive is written, and `viewer-build` / `concat` / `split` re-run it on
+ * their archive. It is a corpus-wide batch computation (template keys are
+ * only comparable within one run), not a per-page one.
+ *
+ * The signals of every internal HTML page with a stored snapshot are loaded
+ * before clustering starts (`loadPageClusterCandidates`: ids and the few
+ * columns the classifier reads, no `Page` objects) and clustered **once**,
+ * never per loading batch: per-batch keys would only be comparable within
+ * their own batch.
+ *
+ * A run that classifies no page at all (no internal page has a stored HTML
+ * snapshot to cluster) leaves the stored classification untouched instead of
+ * replacing it with an empty one: nothing could be re-derived, and erasing a
+ * previous classification (e.g. rows `concat` copied from its sources) would
+ * lose data for no gain. Whenever at least one page is classified the stored
+ * classification is replaced wholesale; labels survive either way —
+ * `replacePageTemplates` carries them forward across runs and keeps retired
+ * ones.
+ * @param archive - A writable archive (crawl-end orchestrator archive, or one
+ *   opened for `viewer-build` / created by `concat` / `split`).
+ * @param options - See {@link ClassifyArchivePageTemplatesOptions}.
+ * @returns See {@link ClassifyArchivePageTemplatesResult}.
+ * @throws {Error} Whatever page loading, clustering or the write throws — callers
+ *   that treat classification as best-effort (all CLI paths) catch it.
+ * @example
+ * ```ts
+ * const { classifiedPageCount, templateCount } = await classifyArchivePageTemplates(
+ *   archive,
+ *   { onProgress: (event) => console.error(event.phase) },
+ * );
+ * ```
+ */
+export declare function classifyArchivePageTemplates(archive: Archive, options?: ClassifyArchivePageTemplatesOptions): Promise<ClassifyArchivePageTemplatesResult>;

package/lib/template-classification/classify-archive-page-templates.js

@@ -0,0 +1,70 @@
+import { classifyPageTemplates } from './classify-page-templates.js';
+import { collectPageIdsWithHtml } from './collect-page-ids-with-html.js';
+import { loadPageClusterCandidates } from './load-page-cluster-candidates.js';
+/**
+ * Classifies every internal HTML page of the archive into a template group
+ * by DOM-structure similarity and persists the result to `page_templates`,
+ * `page_template_clusters` and `page_template_labels`.
+ *
+ * This is the crawl-end derived-data step (alongside the JS-resource scan
+ * and the viewer read-model build): the crawl command runs it before the
+ * archive is written, and `viewer-build` / `concat` / `split` re-run it on
+ * their archive. It is a corpus-wide batch computation (template keys are
+ * only comparable within one run), not a per-page one.
+ *
+ * The signals of every internal HTML page with a stored snapshot are loaded
+ * before clustering starts (`loadPageClusterCandidates`: ids and the few
+ * columns the classifier reads, no `Page` objects) and clustered **once**,
+ * never per loading batch: per-batch keys would only be comparable within
+ * their own batch.
+ *
+ * A run that classifies no page at all (no internal page has a stored HTML
+ * snapshot to cluster) leaves the stored classification untouched instead of
+ * replacing it with an empty one: nothing could be re-derived, and erasing a
+ * previous classification (e.g. rows `concat` copied from its sources) would
+ * lose data for no gain. Whenever at least one page is classified the stored
+ * classification is replaced wholesale; labels survive either way —
+ * `replacePageTemplates` carries them forward across runs and keeps retired
+ * ones.
+ * @param archive - A writable archive (crawl-end orchestrator archive, or one
+ *   opened for `viewer-build` / created by `concat` / `split`).
+ * @param options - See {@link ClassifyArchivePageTemplatesOptions}.
+ * @returns See {@link ClassifyArchivePageTemplatesResult}.
+ * @throws {Error} Whatever page loading, clustering or the write throws — callers
+ *   that treat classification as best-effort (all CLI paths) catch it.
+ * @example
+ * ```ts
+ * const { classifiedPageCount, templateCount } = await classifyArchivePageTemplates(
+ *   archive,
+ *   { onProgress: (event) => console.error(event.phase) },
+ * );
+ * ```
+ */
+export async function classifyArchivePageTemplates(archive, options = {}) {
+    const { memoryBudget, onProgress } = options;
+    onProgress?.({ phase: 'listing-html-snapshots' });
+    const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+    onProgress?.({ phase: 'loading-pages-start' });
+    const candidates = await loadPageClusterCandidates(archive, {
+        pageIdsWithHtml,
+        ...(onProgress ? { onProgress } : {}),
+    });
+    const classification = await classifyPageTemplates({
+        archive,
+        candidates,
+        ...(memoryBudget === undefined ? {} : { memoryBudget }),
+        onProgress,
+    });
+    if (classification.templateKeysByUrl.size > 0) {
+        onProgress?.({
+            phase: 'writing-results',
+            templateCount: new Set(classification.templateKeysByUrl.values()).size,
+        });
+        await archive.replacePageTemplates(classification.templateKeysByUrl, classification.clusterReasonsByTemplateKey);
+    }
+    return {
+        classifiedPageCount: classification.templateKeysByUrl.size,
+        templateCount: new Set(classification.templateKeysByUrl.values()).size,
+        sampledBlocks: classification.sampledBlocks,
+    };
+}

package/lib/template-classification/classify-page-templates.d.ts

@@ -0,0 +1,33 @@
+import type { ClassifyPageTemplatesOptions, PageTemplateClassification } from './types.js';
+/**
+ * Classifies every internal HTML page in the archive into a template group
+ * by DOM-structure similarity, using the classifier `./page-cluster/` drives
+ * in `@nitpicker/core`.
+ *
+ * The result is deliberately **not cached**. A cache keyed on the archive
+ * file's identity (path + size + mtime + page count) cannot be correct at the
+ * only point this runs — the end of a crawl, before the archive is written
+ * back: `archive.filePath` then still names the previous tar (or nothing), so
+ * a `--retry-failed` run that replaces a failed page's HTML without changing
+ * the page count would be served the previous classification. A crash during
+ * classification leaves the crawl's stub directory in place, and resuming it
+ * re-runs this step from scratch; the classifier has no mid-computation
+ * checkpoint to resume from either.
+ *
+ * Internally: the driver's `resolvePageClusterKeys` returns a `clusterKey`
+ * per page; this function's public naming is `templateKey`, to avoid
+ * colliding with nitpicker's unrelated pre-existing "isolated cluster"
+ * concept (`@nitpicker/query`'s `compute-isolated-clusters.ts`,
+ * link-reachability graph components — a completely different kind of
+ * grouping from DOM-structure similarity).
+ * @param options - See {@link ClassifyPageTemplatesOptions}.
+ * @returns See {@link PageTemplateClassification}.
+ * @example
+ * ```ts
+ * const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+ * const candidates = await loadPageClusterCandidates(archive, { pageIdsWithHtml });
+ * const { templateKeysByUrl } = await classifyPageTemplates({ archive, candidates });
+ * await archive.replacePageTemplates(templateKeysByUrl);
+ * ```
+ */
+export declare function classifyPageTemplates(options: ClassifyPageTemplatesOptions): Promise<PageTemplateClassification>;

package/lib/template-classification/classify-page-templates.js

@@ -0,0 +1,60 @@
+import { collectPageStylesheetUrls } from './collect-page-stylesheet-urls.js';
+import { createPageClusterFactory } from './create-page-cluster-factory.js';
+import { resolvePageClusterKeys } from './page-cluster/resolve-page-cluster-keys.js';
+/**
+ * Classifies every internal HTML page in the archive into a template group
+ * by DOM-structure similarity, using the classifier `./page-cluster/` drives
+ * in `@nitpicker/core`.
+ *
+ * The result is deliberately **not cached**. A cache keyed on the archive
+ * file's identity (path + size + mtime + page count) cannot be correct at the
+ * only point this runs — the end of a crawl, before the archive is written
+ * back: `archive.filePath` then still names the previous tar (or nothing), so
+ * a `--retry-failed` run that replaces a failed page's HTML without changing
+ * the page count would be served the previous classification. A crash during
+ * classification leaves the crawl's stub directory in place, and resuming it
+ * re-runs this step from scratch; the classifier has no mid-computation
+ * checkpoint to resume from either.
+ *
+ * Internally: the driver's `resolvePageClusterKeys` returns a `clusterKey`
+ * per page; this function's public naming is `templateKey`, to avoid
+ * colliding with nitpicker's unrelated pre-existing "isolated cluster"
+ * concept (`@nitpicker/query`'s `compute-isolated-clusters.ts`,
+ * link-reachability graph components — a completely different kind of
+ * grouping from DOM-structure similarity).
+ * @param options - See {@link ClassifyPageTemplatesOptions}.
+ * @returns See {@link PageTemplateClassification}.
+ * @example
+ * ```ts
+ * const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+ * const candidates = await loadPageClusterCandidates(archive, { pageIdsWithHtml });
+ * const { templateKeysByUrl } = await classifyPageTemplates({ archive, candidates });
+ * await archive.replacePageTemplates(templateKeysByUrl);
+ * ```
+ */
+export async function classifyPageTemplates(options) {
+    const { archive, candidates, memoryBudget, onProgress } = options;
+    onProgress?.({ phase: 'collecting-stylesheets' });
+    const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
+    const { factory, urls } = createPageClusterFactory({
+        archive,
+        candidates,
+        stylesheetsByUrl,
+    });
+    const { keys, reasons, sampledBlocks } = await resolvePageClusterKeys(factory, {
+        ...(memoryBudget === undefined ? {} : { memoryBudget }),
+        ...(onProgress ? { onProgress } : {}),
+    });
+    if (keys.length !== urls.length) {
+        // Would silently mis-map template keys to the wrong URLs if allowed
+        // through; the driver returns one key per page read, so this guards a
+        // broken invariant loudly rather than assigning quietly.
+        throw new Error(`classifyPageTemplates: resolvePageClusterKeys returned ${keys.length} keys for ${urls.length} pages.`);
+    }
+    const templateKeysByUrl = new Map();
+    for (const [index, url] of urls.entries()) {
+        templateKeysByUrl.set(url, keys[index]);
+    }
+    const clusterReasonsByTemplateKey = reasons;
+    return { templateKeysByUrl, clusterReasonsByTemplateKey, sampledBlocks };
+}

package/lib/template-classification/collect-page-ids-with-html.d.ts

@@ -0,0 +1,24 @@
+import type Archive from '../archive.js';
+/**
+ * Collects the ids (`Page.id`) of the pages that have a non-empty HTML
+ * snapshot stored, without decompressing any of them.
+ *
+ * Template classification reads only pages with HTML, and its engine reads
+ * the corpus several times, decompressing each page only on the reads that
+ * use it. Which pages take part has to be known before the first read, the
+ * same on every read: deciding it by calling `Page.getHtml()` would
+ * decompress every page on every read. A snapshot is non-empty exactly when
+ * its raw UTF-8 size (`page_html_blobs.size_raw`, recorded when it was
+ * written) is above zero, and a page whose `page_html_ref` points at no blob
+ * reads back as `null` — both match what `Page.getHtml()` returns. Keyed by
+ * id, not URL, so the answer is the page's own, whatever its URL normalizes
+ * to.
+ * @param archive - The archive to read from.
+ * @returns The ids, in no particular order.
+ * @example
+ * ```ts
+ * const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+ * const pagesWithHtml = pages.filter((page) => pageIdsWithHtml.has(page.id));
+ * ```
+ */
+export declare function collectPageIdsWithHtml(archive: Archive): Promise<Set<number>>;

package/lib/template-classification/collect-page-ids-with-html.js

@@ -0,0 +1,30 @@
+/**
+ * Collects the ids (`Page.id`) of the pages that have a non-empty HTML
+ * snapshot stored, without decompressing any of them.
+ *
+ * Template classification reads only pages with HTML, and its engine reads
+ * the corpus several times, decompressing each page only on the reads that
+ * use it. Which pages take part has to be known before the first read, the
+ * same on every read: deciding it by calling `Page.getHtml()` would
+ * decompress every page on every read. A snapshot is non-empty exactly when
+ * its raw UTF-8 size (`page_html_blobs.size_raw`, recorded when it was
+ * written) is above zero, and a page whose `page_html_ref` points at no blob
+ * reads back as `null` — both match what `Page.getHtml()` returns. Keyed by
+ * id, not URL, so the answer is the page's own, whatever its URL normalizes
+ * to.
+ * @param archive - The archive to read from.
+ * @returns The ids, in no particular order.
+ * @example
+ * ```ts
+ * const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+ * const pagesWithHtml = pages.filter((page) => pageIdsWithHtml.has(page.id));
+ * ```
+ */
+export async function collectPageIdsWithHtml(archive) {
+    const rows = (await archive
+        .getKnex()('page_html_ref as ref')
+        .join('page_html_blobs as blob', 'blob.hash', 'ref.hash')
+        .where('blob.size_raw', '>', 0)
+        .select('ref.page_id as pageId'));
+    return new Set(rows.map((row) => row.pageId));
+}

package/lib/template-classification/collect-page-stylesheet-urls.d.ts

@@ -0,0 +1,32 @@
+import type Archive from '../archive.js';
+/**
+ * Collects, for every page that references at least one CSS stylesheet, the
+ * set of stylesheet URLs it references.
+ *
+ * Reads in three narrow passes instead of one wide JOIN, to avoid
+ * materializing the full page×stylesheet edge set at once — a real
+ * 486,000-page archive has ~2.5 million such edges, and a single JOIN
+ * projecting both URL strings for all of them is a transient allocation
+ * spike well beyond what any single page's data actually needs:
+ *
+ * 1. Resolve which `resource_items` rows are classified `'css'`
+ *    (`content_type_refs.category`) and their stylesheet URL.
+ * 2. Resolve `resource_ref_edges` for just those CSS resource ids, in
+ *    chunks, projecting only the two integer ids per row.
+ * 3. Resolve the page ids collected in step 2 to their URLs.
+ *
+ * Identical stylesheet-URL arrays are interned across pages: pages sharing a
+ * template share the same CSS set, so the number of distinct arrays
+ * converges to a few hundred even across hundreds of thousands of pages.
+ * @param archive - The archive to read from.
+ * @returns Map from page URL (parseUrl-normalized `href`, matching
+ *   `Page.url.href`) to the stylesheet URLs it references. Pages that
+ *   reference no CSS resource have no entry — callers should default to an
+ *   empty array on a missed lookup.
+ * @example
+ * ```ts
+ * const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
+ * const hrefs = stylesheetsByUrl.get(page.url.href) ?? [];
+ * ```
+ */
+export declare function collectPageStylesheetUrls(archive: Archive): Promise<Map<string, readonly string[]>>;