@nitpicker/archive 0.0.0 → 0.25.1

package/lib/template-classification/collect-page-stylesheet-urls.js

@@ -0,0 +1,132 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { eachSplitted } from '../utils/array/each-splitted.js';
+/**
+ * SQLite has a hard cap of 999 bound variables per statement. A real archive
+ * can have far more than 999 CSS-referencing edges or pages, so every
+ * `whereIn` below is chunked at this size.
+ *
+ * Independently duplicated (not shared/imported) from `@nitpicker/query`'s
+ * `compute-isolated-clusters.ts`, which defines the identical
+ * `SQLITE_IN_CHUNK = 500` for the same reason. If this margin below 999 ever
+ * needs revisiting, update both — there is no shared module either package
+ * depends on to hold one copy.
+ */
+const SQLITE_IN_CHUNK = 500;
+/**
+ * Collects, for every page that references at least one CSS stylesheet, the
+ * set of stylesheet URLs it references.
+ *
+ * Reads in three narrow passes instead of one wide JOIN, to avoid
+ * materializing the full page×stylesheet edge set at once — a real
+ * 486,000-page archive has ~2.5 million such edges, and a single JOIN
+ * projecting both URL strings for all of them is a transient allocation
+ * spike well beyond what any single page's data actually needs:
+ *
+ * 1. Resolve which `resource_items` rows are classified `'css'`
+ *    (`content_type_refs.category`) and their stylesheet URL.
+ * 2. Resolve `resource_ref_edges` for just those CSS resource ids, in
+ *    chunks, projecting only the two integer ids per row.
+ * 3. Resolve the page ids collected in step 2 to their URLs.
+ *
+ * Identical stylesheet-URL arrays are interned across pages: pages sharing a
+ * template share the same CSS set, so the number of distinct arrays
+ * converges to a few hundred even across hundreds of thousands of pages.
+ * @param archive - The archive to read from.
+ * @returns Map from page URL (parseUrl-normalized `href`, matching
+ *   `Page.url.href`) to the stylesheet URLs it references. Pages that
+ *   reference no CSS resource have no entry — callers should default to an
+ *   empty array on a missed lookup.
+ * @example
+ * ```ts
+ * const stylesheetsByUrl = await collectPageStylesheetUrls(archive);
+ * const hrefs = stylesheetsByUrl.get(page.url.href) ?? [];
+ * ```
+ */
+export async function collectPageStylesheetUrls(archive) {
+    const knex = archive.getKnex();
+    const cssResources = await knex('resource_items as ri')
+        .join('content_type_refs as ctr', 'ctr.id', 'ri.content_type_id')
+        .where('ctr.category', 'css')
+        .select('ri.id as id', 'ri.url_id as urlId');
+    // A CSS resource identified by a `data:` URI (routed to `blob_refs`
+    // instead of `url_refs`, `resource_items.url_blob_id`) has no stylesheet
+    // URL to report — skip it rather than passing a `null` into `whereIn`.
+    const cssResourcesWithUrl = cssResources.filter((r) => r.urlId != null);
+    if (cssResourcesWithUrl.length === 0) {
+        return new Map();
+    }
+    const cssUrlById = new Map();
+    await eachSplitted(cssResourcesWithUrl.map((r) => r.urlId), SQLITE_IN_CHUNK, async (chunk) => {
+        const rows = await knex('url_refs')
+            .whereIn('id', chunk)
+            .select('id', 'url');
+        for (const row of rows) {
+            cssUrlById.set(row.id, row.url);
+        }
+    });
+    const cssUrlByResourceId = new Map();
+    for (const r of cssResourcesWithUrl) {
+        const url = cssUrlById.get(r.urlId);
+        if (url) {
+            cssUrlByResourceId.set(r.id, url);
+        }
+    }
+    const resourceIdsByPageId = new Map();
+    await eachSplitted(cssResourcesWithUrl.map((r) => r.id), SQLITE_IN_CHUNK, async (chunk) => {
+        const rows = await knex('resource_ref_edges')
+            .whereIn('resource_id', chunk)
+            .select('resource_id as resourceId', 'page_id as pageId');
+        // Avoid `push(...rows)`: on large real archives this chunk array can
+        // be large enough to overflow V8's argument-spread limit even though
+        // the underlying data itself fits in memory.
+        for (const row of rows) {
+            const existing = resourceIdsByPageId.get(row.pageId);
+            if (existing) {
+                existing.push(row.resourceId);
+            }
+            else {
+                resourceIdsByPageId.set(row.pageId, [row.resourceId]);
+            }
+        }
+    });
+    if (resourceIdsByPageId.size === 0) {
+        return new Map();
+    }
+    const pageUrlById = new Map();
+    await eachSplitted([...resourceIdsByPageId.keys()], SQLITE_IN_CHUNK, async (chunk) => {
+        const rows = await knex('content_items as ci')
+            .join('url_refs as ur', 'ur.id', 'ci.url_id')
+            .whereIn('ci.id', chunk)
+            .select('ci.id as id', 'ur.url as url');
+        for (const row of rows) {
+            pageUrlById.set(row.id, row.url);
+        }
+    });
+    const internedSets = new Map();
+    const result = new Map();
+    for (const [pageId, resourceIds] of resourceIdsByPageId) {
+        const pageUrlRaw = pageUrlById.get(pageId);
+        if (!pageUrlRaw) {
+            continue;
+        }
+        const parsed = parseUrl(pageUrlRaw);
+        if (!parsed) {
+            continue;
+        }
+        const urls = resourceIds
+            .map((id) => cssUrlByResourceId.get(id))
+            .filter((url) => url != null)
+            .toSorted();
+        if (urls.length === 0) {
+            continue;
+        }
+        const internKey = urls.join(' ');
+        let interned = internedSets.get(internKey);
+        if (!interned) {
+            interned = Object.freeze(urls);
+            internedSets.set(internKey, interned);
+        }
+        result.set(parsed.href, interned);
+    }
+    return result;
+}

package/lib/template-classification/create-content-root-hint.d.ts

@@ -0,0 +1,41 @@
+import type { ContentRootSource } from './types.js';
+import type { ContentRootHint } from '@nitpicker/core/types';
+/**
+ * Builds the `contentRoot` hint page-cluster anchors its
+ * content-depth cap on, from the main-content element the crawler recorded
+ * for a page (`page_meta.main_content_*`, detected at crawl time by
+ * `@d-zero/beholder` or forced with `--main-content-selector`).
+ *
+ * Sites without `<main>` / `role="main"` — the case the hint exists for —
+ * otherwise leave the library with no anchor for its cap, so freeform body
+ * content dominates the comparison and one template fragments into many
+ * clusters. The crawler already knows which element holds the page-specific
+ * content; this hands that element's identity over instead of having the
+ * library guess it.
+ *
+ * Fields the crawler left empty are dropped: `nodeName` is lower-cased (the
+ * DOM reports `'DIV'`, the library compares against the parser's `'div'`), and
+ * an empty id / role / class list is omitted, since the library treats
+ * "given but empty" and "absent" the same and a hint with nothing left
+ * matches nothing. Returns `undefined` when the page has no detected main
+ * content (never rendered, or no candidate element) or nothing usable is
+ * left, so the factory can leave the field off the page entirely.
+ *
+ * Deliberately not built from `mainContentSelector`: it is a diagnostic
+ * `tag#id.class` string that is not guaranteed unique and would need a CSS
+ * selector parser on the library side; the four structured columns carry the
+ * same information without parsing.
+ * @param source - The four main-content values of a page; a `Page` satisfies it.
+ * @returns The hint, or `undefined` when there is nothing to anchor on.
+ * @example
+ * ```ts
+ * createContentRootHint({
+ * 	mainContentNodeName: 'DIV',
+ * 	mainContentId: 'main',
+ * 	mainContentRole: null,
+ * 	mainContentClassList: ['spc'],
+ * });
+ * // { tagName: 'div', id: 'main', classList: ['spc'] }
+ * ```
+ */
+export declare function createContentRootHint(source: ContentRootSource): ContentRootHint | undefined;

package/lib/template-classification/create-content-root-hint.js

@@ -0,0 +1,51 @@
+/**
+ * Builds the `contentRoot` hint page-cluster anchors its
+ * content-depth cap on, from the main-content element the crawler recorded
+ * for a page (`page_meta.main_content_*`, detected at crawl time by
+ * `@d-zero/beholder` or forced with `--main-content-selector`).
+ *
+ * Sites without `<main>` / `role="main"` — the case the hint exists for —
+ * otherwise leave the library with no anchor for its cap, so freeform body
+ * content dominates the comparison and one template fragments into many
+ * clusters. The crawler already knows which element holds the page-specific
+ * content; this hands that element's identity over instead of having the
+ * library guess it.
+ *
+ * Fields the crawler left empty are dropped: `nodeName` is lower-cased (the
+ * DOM reports `'DIV'`, the library compares against the parser's `'div'`), and
+ * an empty id / role / class list is omitted, since the library treats
+ * "given but empty" and "absent" the same and a hint with nothing left
+ * matches nothing. Returns `undefined` when the page has no detected main
+ * content (never rendered, or no candidate element) or nothing usable is
+ * left, so the factory can leave the field off the page entirely.
+ *
+ * Deliberately not built from `mainContentSelector`: it is a diagnostic
+ * `tag#id.class` string that is not guaranteed unique and would need a CSS
+ * selector parser on the library side; the four structured columns carry the
+ * same information without parsing.
+ * @param source - The four main-content values of a page; a `Page` satisfies it.
+ * @returns The hint, or `undefined` when there is nothing to anchor on.
+ * @example
+ * ```ts
+ * createContentRootHint({
+ * 	mainContentNodeName: 'DIV',
+ * 	mainContentId: 'main',
+ * 	mainContentRole: null,
+ * 	mainContentClassList: ['spc'],
+ * });
+ * // { tagName: 'div', id: 'main', classList: ['spc'] }
+ * ```
+ */
+export function createContentRootHint(source) {
+    const tagName = source.mainContentNodeName?.toLowerCase();
+    const id = source.mainContentId;
+    const role = source.mainContentRole;
+    const classList = source.mainContentClassList;
+    const hint = {
+        ...(tagName ? { tagName } : {}),
+        ...(id ? { id } : {}),
+        ...(role ? { role } : {}),
+        ...(classList && classList.length > 0 ? { classList } : {}),
+    };
+    return Object.keys(hint).length > 0 ? hint : undefined;
+}

package/lib/template-classification/create-page-cluster-factory.d.ts

@@ -0,0 +1,36 @@
+import type { PageFactory } from './page-cluster/types.js';
+import type { CreatePageClusterFactoryOptions } from './types.js';
+/**
+ * Builds the `PageFactory` that page-cluster's `resolvePageClusterKeys`
+ * reads the corpus through, and the URLs of the pages it yields, in order.
+ *
+ * It yields exactly the candidates it is given — internal HTML pages with a
+ * non-empty stored HTML snapshot, decided upfront by
+ * `loadPageClusterCandidates` from `collectPageIdsWithHtml` (one query that
+ * decompresses nothing). So the set is fixed before the first read and
+ * identical on every read, as the engine requires, and each page's HTML is
+ * decompressed only on the reads that use it (`getHtml`, by page id) instead
+ * of on every read just to see whether it is empty.
+ *
+ * Each yielded page carries its `paths`, `stylesheetHrefs`, `host`, and —
+ * when the crawler detected a main-content element for it — the `contentRoot`
+ * hint the candidate holds, which the classifier anchors its content-depth
+ * cap on. The factory holds each page's id and these signals only.
+ * @param options - See {@link CreatePageClusterFactoryOptions}.
+ * @returns `factory` to pass to `resolvePageClusterKeys`, and `urls`: the
+ *   yielded pages' `href`, in the order of the keys it returns.
+ * @example
+ * ```ts
+ * const { factory, urls } = createPageClusterFactory({
+ * 	archive,
+ * 	candidates,
+ * 	stylesheetsByUrl,
+ * });
+ * const { keys } = await resolvePageClusterKeys(factory);
+ * // keys[i] is the template key of urls[i]
+ * ```
+ */
+export declare function createPageClusterFactory(options: CreatePageClusterFactoryOptions): {
+    factory: PageFactory;
+    urls: readonly string[];
+};

package/lib/template-classification/create-page-cluster-factory.js

@@ -0,0 +1,52 @@
+/**
+ * Builds the `PageFactory` that page-cluster's `resolvePageClusterKeys`
+ * reads the corpus through, and the URLs of the pages it yields, in order.
+ *
+ * It yields exactly the candidates it is given — internal HTML pages with a
+ * non-empty stored HTML snapshot, decided upfront by
+ * `loadPageClusterCandidates` from `collectPageIdsWithHtml` (one query that
+ * decompresses nothing). So the set is fixed before the first read and
+ * identical on every read, as the engine requires, and each page's HTML is
+ * decompressed only on the reads that use it (`getHtml`, by page id) instead
+ * of on every read just to see whether it is empty.
+ *
+ * Each yielded page carries its `paths`, `stylesheetHrefs`, `host`, and —
+ * when the crawler detected a main-content element for it — the `contentRoot`
+ * hint the candidate holds, which the classifier anchors its content-depth
+ * cap on. The factory holds each page's id and these signals only.
+ * @param options - See {@link CreatePageClusterFactoryOptions}.
+ * @returns `factory` to pass to `resolvePageClusterKeys`, and `urls`: the
+ *   yielded pages' `href`, in the order of the keys it returns.
+ * @example
+ * ```ts
+ * const { factory, urls } = createPageClusterFactory({
+ * 	archive,
+ * 	candidates,
+ * 	stylesheetsByUrl,
+ * });
+ * const { keys } = await resolvePageClusterKeys(factory);
+ * // keys[i] is the template key of urls[i]
+ * ```
+ */
+export function createPageClusterFactory(options) {
+    const { archive, candidates, stylesheetsByUrl } = options;
+    const corpus = candidates.map((candidate) => ({
+        paths: candidate.paths,
+        stylesheetHrefs: stylesheetsByUrl.get(candidate.href) ?? [],
+        host: candidate.host,
+        ...(candidate.contentRoot ? { contentRoot: candidate.contentRoot } : {}),
+        getHtml: async () => {
+            const html = await archive.getHtmlOfPage(candidate.id);
+            if (!html) {
+                // The archive does not change during a classification, so a
+                // page found with HTML upfront always reads back with it.
+                throw new Error(`createPageClusterFactory: ${candidate.href} has no stored HTML, but collectPageIdsWithHtml listed it`);
+            }
+            return html;
+        },
+    }));
+    return {
+        factory: () => corpus,
+        urls: candidates.map((candidate) => candidate.href),
+    };
+}

package/lib/template-classification/load-page-cluster-candidates.d.ts

@@ -0,0 +1,32 @@
+import type { LoadPageClusterCandidatesOptions, PageClusterCandidate } from './types.js';
+import type Archive from '../archive.js';
+/**
+ * Loads the internal HTML pages of the archive that have a stored HTML
+ * snapshot as {@link PageClusterCandidate}s: each page's id and the signals
+ * template classification reads without HTML, and nothing else.
+ *
+ * It reads only the page's id, its url and the four `main_content_*` columns
+ * the content-root hint is built from, straight from `content_items` /
+ * `page_meta`, in `content_items.id` order, one keyset batch at a time, and
+ * keeps a page only when its id is in `pageIdsWithHtml`. It deliberately
+ * builds no `Page`: a `Page` carries the page's whole row (every meta column,
+ * joined from many meta tables), which is far more than classification reads and
+ * would stay resident for every page until the clustering ends.
+ *
+ * "Internal HTML page" is `Page.isInternalPage()` evaluated in SQL — content
+ * type `text/html` and not external, the same selection as `getPages`'
+ * `'internal-page'` filter (`load-page-cluster-candidates.spec.ts` asserts
+ * the two return the same pages). The URL is parsed as
+ * `Page.url` does with queries kept (`disableQueries: false`).
+ * @param archive - The archive to read from.
+ * @param options - See {@link LoadPageClusterCandidatesOptions}.
+ * @returns The candidates, in `content_items.id` order.
+ * @throws {Error} When a stored URL cannot be parsed, or a stored
+ *   `main_content_class_list` is not a JSON array of strings.
+ * @example
+ * ```ts
+ * const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+ * const candidates = await loadPageClusterCandidates(archive, { pageIdsWithHtml });
+ * ```
+ */
+export declare function loadPageClusterCandidates(archive: Archive, options: LoadPageClusterCandidatesOptions): Promise<PageClusterCandidate[]>;

package/lib/template-classification/load-page-cluster-candidates.js

@@ -0,0 +1,93 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { parseMainContentClassList } from '../meta/parse-main-content-class-list.js';
+import { createContentRootHint } from './create-content-root-hint.js';
+/**
+ * Rows read per query. Each batch's rows are turned into candidates and
+ * dropped before the next is read, so this bounds the transient garbage of
+ * one query, not what stays resident.
+ */
+const DEFAULT_BATCH_SIZE = 10_000;
+/**
+ * Loads the internal HTML pages of the archive that have a stored HTML
+ * snapshot as {@link PageClusterCandidate}s: each page's id and the signals
+ * template classification reads without HTML, and nothing else.
+ *
+ * It reads only the page's id, its url and the four `main_content_*` columns
+ * the content-root hint is built from, straight from `content_items` /
+ * `page_meta`, in `content_items.id` order, one keyset batch at a time, and
+ * keeps a page only when its id is in `pageIdsWithHtml`. It deliberately
+ * builds no `Page`: a `Page` carries the page's whole row (every meta column,
+ * joined from many meta tables), which is far more than classification reads and
+ * would stay resident for every page until the clustering ends.
+ *
+ * "Internal HTML page" is `Page.isInternalPage()` evaluated in SQL — content
+ * type `text/html` and not external, the same selection as `getPages`'
+ * `'internal-page'` filter (`load-page-cluster-candidates.spec.ts` asserts
+ * the two return the same pages). The URL is parsed as
+ * `Page.url` does with queries kept (`disableQueries: false`).
+ * @param archive - The archive to read from.
+ * @param options - See {@link LoadPageClusterCandidatesOptions}.
+ * @returns The candidates, in `content_items.id` order.
+ * @throws {Error} When a stored URL cannot be parsed, or a stored
+ *   `main_content_class_list` is not a JSON array of strings.
+ * @example
+ * ```ts
+ * const pageIdsWithHtml = await collectPageIdsWithHtml(archive);
+ * const candidates = await loadPageClusterCandidates(archive, { pageIdsWithHtml });
+ * ```
+ */
+export async function loadPageClusterCandidates(archive, options) {
+    const { pageIdsWithHtml, onProgress, batchSize = DEFAULT_BATCH_SIZE } = options;
+    const knex = archive.getKnex();
+    const internalHtml = () => knex('content_items as ci')
+        .join('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
+        .where('ctr.raw', 'text/html')
+        .andWhere('ci.is_external', 0);
+    // Only for progress: counted once up front. The archive is not written
+    // while it is classified, so `done` ends at `total`.
+    const countRow = (await internalHtml().count({ count: '*' }).first());
+    const total = Number(countRow?.count ?? 0);
+    const candidates = [];
+    let lastId = 0;
+    let done = 0;
+    while (true) {
+        const rows = await internalHtml()
+            .join('url_refs as ur', 'ur.id', 'ci.url_id')
+            .leftJoin('page_meta as pm', 'pm.page_id', 'ci.id')
+            .andWhere('ci.id', '>', lastId)
+            .orderBy('ci.id')
+            .limit(batchSize)
+            .select('ci.id as id', 'ur.url as url', 'pm.main_content_node_name as main_content_node_name', 'pm.main_content_id as main_content_id', 'pm.main_content_role as main_content_role', 'pm.main_content_class_list as main_content_class_list');
+        if (rows.length === 0) {
+            break;
+        }
+        for (const row of rows) {
+            if (!pageIdsWithHtml.has(row.id)) {
+                continue;
+            }
+            const url = parseUrl(row.url, { disableQueries: false });
+            if (!url) {
+                throw new Error(`loadPageClusterCandidates: cannot parse the stored URL ${row.url}`);
+            }
+            const contentRoot = createContentRootHint({
+                mainContentNodeName: row.main_content_node_name,
+                mainContentId: row.main_content_id,
+                mainContentRole: row.main_content_role,
+                mainContentClassList: parseMainContentClassList(row.main_content_class_list),
+            });
+            candidates.push({
+                id: row.id,
+                href: url.href,
+                paths: url.paths,
+                host: url.port ? `${url.hostname}:${url.port}` : url.hostname,
+                // Left off entirely (not `undefined`) for pages with no detected
+                // main content, so the candidate carries no undefined-valued key.
+                ...(contentRoot ? { contentRoot } : {}),
+            });
+        }
+        lastId = rows.at(-1).id;
+        done += rows.length;
+        onProgress?.({ phase: 'loading-pages', done, total });
+    }
+    return candidates;
+}

package/lib/template-classification/page-cluster/resolve-page-cluster-keys.d.ts

@@ -0,0 +1,34 @@
+import type { PageClusterKeys, PageFactory, ResolvePageClusterKeysOptions } from './types.js';
+/**
+ * Clusters pages into templates by DOM structure, in the native addon
+ * (`@nitpicker/core`'s `createTemplateClassificationRun`):
+ *
+ * - **Pass 0** splits the corpus into blocks by URL section and first-party
+ *   stylesheets, from the signals alone — no HTML.
+ * - **Stage A** clusters each block: landmark chrome is cut out, the content
+ *   depth is capped at the block's knee, and complete-linkage clustering
+ *   with a max-gap cut splits the block.
+ * - **Stage B** merges clusters across blocks and explains each final one.
+ *
+ * A block is clustered whole unless its Stage A similarity table
+ * (`n * (n - 1) / 2 * 8` bytes for `n` pages) exceeds `options.memoryBudget`,
+ * or the blocks clustered whole would hold more than
+ * `TEMPLATE_EXACT_PAGE_LIMIT` pages together (the largest are then sampled
+ * until they fit); a sampled block is clustered on a sample and a further
+ * read assigns its other pages. The factory is read twice (signals, HTML)
+ * or, with a sampled block that has pages outside its sample, three times; each page's HTML is read
+ * once, and JavaScript never holds the whole corpus's HTML.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const { keys, reasons } = await resolvePageClusterKeys(() => [
+ * 	{ paths: ['news', '1'], stylesheetHrefs: [], getHtml: () => '<body><article>one</article></body>' },
+ * 	{ paths: ['news', '2'], stylesheetHrefs: [], getHtml: () => '<body><article>two</article></body>' },
+ * 	{ paths: ['about'], stylesheetHrefs: [], getHtml: () => '<body><section>about</section></body>' },
+ * ]);
+ * // keys: ['["path:news","cluster:0"]', '["path:news","cluster:0"]', '["path:about","cluster:0"]']
+ * // reasons: one per cluster key, e.g. reasons.get('["path:about","cluster:0"]')?.memberCount === 1
+ * ```
+ */
+export declare function resolvePageClusterKeys(pages: PageFactory, options?: Partial<ResolvePageClusterKeysOptions>): Promise<PageClusterKeys>;