@nitpicker/archive 0.0.0 → 0.25.0

package/lib/db-ops/templates/assign-template-labels.js

@@ -0,0 +1,187 @@
+/**
+ * Assigns every cluster of a fresh template classification its
+ * {@link TemplateLabel}, carrying labels forward from the previous
+ * classification wherever the same template survived the re-run.
+ *
+ * ## Inheritance
+ *
+ * A new cluster inherits an old cluster's label when the two share a
+ * **mutual majority** of pages: the overlap is more than half of the new
+ * cluster *and* more than half of the old one. Mutual majority means at
+ * most one new cluster can claim a given old label, and a cluster that
+ * merely absorbed a few pages of a much larger old template does not steal
+ * its name. A split keeps the old label on the half that holds most of the
+ * old pages; the other half is named afresh.
+ *
+ * The old cluster's size counts only pages that are classified in the new
+ * run too. Pages the crawl has since dropped (`--recrawl` turned them into
+ * redirects, errors, …) are not evidence that the template changed, so
+ * they must not dilute the old half of the majority — otherwise an old
+ * 7-page cluster whose 4 missing pages left 3 unchanged ones would fail
+ * `3 > 7/2` and be renamed for no reason.
+ *
+ * Page ids are the identity used for overlap (not URLs) because they are
+ * what `page_templates` stores and they survive `--append` / `--recrawl`
+ * without renumbering.
+ *
+ * ## Fresh labels
+ *
+ * Clusters left without an inherited label are numbered in page-count
+ * order (largest first, template key as a tie-break) so a first run reads
+ * naturally: `A` is the section's biggest template. Each takes the next
+ * ordinal **after every ordinal in `previousLabels` for its section**, so a
+ * letter is never re-issued to a different template — which would silently
+ * change what "events template B" means between two people's archives.
+ * That guarantee is only as good as `previousLabels`: `replacePageTemplates`
+ * keeps the rows of clusters that disappeared (retired labels) precisely so
+ * they stay in this set across any number of later runs.
+ *
+ * ## Section
+ *
+ * A cluster's section is the first path segment every member URL shares
+ * (`events` for `/events/...`). Members spread over several segments, or
+ * all at the site root, yield `null`: such a cluster is numbered in the
+ * site-wide sequence. An inherited label keeps its old section even if the
+ * cluster's members have since drifted — stability of the name wins over
+ * keeping the section literally accurate.
+ * @param params - See {@link AssignTemplateLabelsParams}.
+ * @returns Template key → label, one entry per cluster in `params.clusters`.
+ * @example
+ * ```ts
+ * const labels = assignTemplateLabels({
+ *   clusters: new Map([['["path:events","cluster:0"]', { pageIds: [1, 2], urls: ['https://example.com/events/a', 'https://example.com/events/b'] }]]),
+ *   previousMembership: new Map(),
+ *   previousLabels: new Map(),
+ * });
+ * labels.get('["path:events","cluster:0"]'); // { section: 'events', ordinal: 1 }
+ * ```
+ */
+export function assignTemplateLabels(params) {
+    const { clusters, previousMembership, previousLabels } = params;
+    const currentPageIds = new Set();
+    for (const cluster of clusters.values()) {
+        for (const pageId of cluster.pageIds) {
+            currentPageIds.add(pageId);
+        }
+    }
+    const previousKeyByPageId = new Map();
+    const previousSizeByKey = new Map();
+    for (const [key, pageIds] of previousMembership) {
+        let size = 0;
+        for (const pageId of pageIds) {
+            previousKeyByPageId.set(pageId, key);
+            if (currentPageIds.has(pageId)) {
+                size += 1;
+            }
+        }
+        previousSizeByKey.set(key, size);
+    }
+    // Every ordinal ever issued per section, seeded from the previous run so
+    // a vanished cluster's letter is retired rather than re-issued.
+    const usedOrdinals = new Map();
+    const markUsed = (label) => {
+        let set = usedOrdinals.get(label.section);
+        if (!set) {
+            set = new Set();
+            usedOrdinals.set(label.section, set);
+        }
+        set.add(label.ordinal);
+    };
+    for (const label of previousLabels.values()) {
+        markUsed(label);
+    }
+    const ordered = [...clusters].toSorted(([keyA, a], [keyB, b]) => b.pageIds.length - a.pageIds.length || keyA.localeCompare(keyB));
+    const labels = new Map();
+    const claimedPreviousKeys = new Set();
+    const unlabeled = [];
+    for (const entry of ordered) {
+        const [key, cluster] = entry;
+        const inherited = findInheritedLabel(cluster.pageIds, {
+            previousKeyByPageId,
+            previousSizeByKey,
+            previousLabels,
+            claimedPreviousKeys,
+        });
+        if (inherited) {
+            claimedPreviousKeys.add(inherited.previousKey);
+            labels.set(key, inherited.label);
+        }
+        else {
+            unlabeled.push(entry);
+        }
+    }
+    for (const [key, cluster] of unlabeled) {
+        const section = deriveSection(cluster.urls);
+        const used = usedOrdinals.get(section);
+        const ordinal = used ? Math.max(...used) + 1 : 1;
+        const label = { section, ordinal };
+        markUsed(label);
+        labels.set(key, label);
+    }
+    return labels;
+}
+/**
+ * Finds the previous cluster whose label the given members inherit, if any
+ * — the unclaimed, labeled previous cluster sharing a mutual majority of
+ * pages with them.
+ * @param pageIds - The new cluster's member page ids.
+ * @param context - Lookups built once by `assignTemplateLabels`.
+ * @param context.previousKeyByPageId - Page id → previous template key.
+ * @param context.previousSizeByKey - Previous template key → member count.
+ * @param context.previousLabels - Previous template key → label.
+ * @param context.claimedPreviousKeys - Previous keys already inherited by a larger new cluster.
+ * @returns The matched previous key and its label, or `null`.
+ */
+function findInheritedLabel(pageIds, context) {
+    const overlapByKey = new Map();
+    for (const pageId of pageIds) {
+        const previousKey = context.previousKeyByPageId.get(pageId);
+        if (previousKey != null) {
+            overlapByKey.set(previousKey, (overlapByKey.get(previousKey) ?? 0) + 1);
+        }
+    }
+    let best = null;
+    for (const [previousKey, overlap] of overlapByKey) {
+        if (!best || overlap > best.overlap) {
+            best = { previousKey, overlap };
+        }
+    }
+    if (!best || context.claimedPreviousKeys.has(best.previousKey)) {
+        return null;
+    }
+    const label = context.previousLabels.get(best.previousKey);
+    if (!label) {
+        return null;
+    }
+    const previousSize = context.previousSizeByKey.get(best.previousKey) ?? 0;
+    if (best.overlap * 2 <= pageIds.length || best.overlap * 2 <= previousSize) {
+        return null;
+    }
+    return { previousKey: best.previousKey, label };
+}
+/**
+ * The first path segment shared by every URL, or `null` when they disagree
+ * or all sit at the site root. A URL that fails to parse counts as
+ * disagreeing — a cluster is not pinned to a section by guesswork.
+ * @param urls - The cluster's member page URLs.
+ * @returns The shared section name, or `null`.
+ */
+function deriveSection(urls) {
+    let section;
+    for (const url of urls) {
+        let segment;
+        try {
+            segment = new URL(url).pathname.split('/')[1] ?? '';
+        }
+        catch {
+            return null;
+        }
+        if (section === undefined) {
+            section = segment;
+        }
+        else if (section !== segment) {
+            return null;
+        }
+    }
+    return section || null;
+}

package/lib/db-ops/templates/replace-page-templates.d.ts

@@ -0,0 +1,44 @@
+import type { ReplacePageTemplatesParams } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Replaces the stored DOM-structure template classification with a freshly
+ * generated set.
+ *
+ * Every classified page is a full-archive, all-or-nothing recomputation
+ * (see `classifyPageTemplates`), so this always deletes
+ * every existing row before inserting the new set — there is no per-page
+ * incremental update path. A page whose URL can't be
+ * resolved back to a `content_items` row is silently skipped rather than
+ * treated as a hard failure: losing one page's template classification
+ * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
+ * and the stored `url_refs.url`) should not discard the rest of a
+ * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
+ *
+ * `page_template_labels` is the one table that is **not** wholesale
+ * replaced: the human-facing label of each new cluster is carried forward
+ * from whichever previous cluster it shares most pages with
+ * (`assignTemplateLabels`), which is what lets "events template B" keep
+ * meaning the same template across re-runs, and the rows of clusters that
+ * disappeared are kept as *retired* labels so their letters are never
+ * re-issued to a different template on a later run (only the rows for the
+ * keys being written, and the old row of a label that moved to a new key,
+ * are removed). Readers look labels up by the keys
+ * currently in `page_templates`, so retired rows are invisible to them.
+ * The previous `page_templates` and `page_template_labels` rows are read
+ * inside the same transaction that writes — there is no window in which
+ * another writer could change what the labels were derived from. An empty
+ * `templateKeysByUrl` therefore clears the classification but leaves every
+ * label row in place.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - See {@link ReplacePageTemplatesParams}.
+ */
+export declare function replacePageTemplates(knex: Knex, params: ReplacePageTemplatesParams): Promise<void>;

package/lib/db-ops/templates/replace-page-templates.js

@@ -0,0 +1,174 @@
+import { eachSplitted } from '../../utils/array/each-splitted.js';
+import { compressPayload } from '../_shared/compress-payload.js';
+import { assignTemplateLabels } from './assign-template-labels.js';
+import { serializeTemplateClusterReason } from './serialize-template-cluster-reason.js';
+/**
+ * Replaces the stored DOM-structure template classification with a freshly
+ * generated set.
+ *
+ * Every classified page is a full-archive, all-or-nothing recomputation
+ * (see `classifyPageTemplates`), so this always deletes
+ * every existing row before inserting the new set — there is no per-page
+ * incremental update path. A page whose URL can't be
+ * resolved back to a `content_items` row is silently skipped rather than
+ * treated as a hard failure: losing one page's template classification
+ * (e.g. a URL-normalization mismatch between the in-memory `Page.url.href`
+ * and the stored `url_refs.url`) should not discard the rest of a
+ * potentially multi-thousand-page classification run.
+ *
+ * `page_template_clusters` is always cleared alongside `page_templates`
+ * regardless of whether `clusterReasonsByTemplateKey` is passed — "no
+ * reason" must mean "not captured for this run", never "carry over the
+ * previous run's reason". Reason rows are inserted for every key in
+ * `clusterReasonsByTemplateKey` even if some have no surviving member page
+ * in `templateKeysByUrl` after URL-resolution skips above — harmless
+ * (nothing joins `page_template_clusters` back to `page_templates` by FK;
+ * see the table's own JSDoc), and simpler than cross-filtering the two maps.
+ *
+ * `page_template_labels` is the one table that is **not** wholesale
+ * replaced: the human-facing label of each new cluster is carried forward
+ * from whichever previous cluster it shares most pages with
+ * (`assignTemplateLabels`), which is what lets "events template B" keep
+ * meaning the same template across re-runs, and the rows of clusters that
+ * disappeared are kept as *retired* labels so their letters are never
+ * re-issued to a different template on a later run (only the rows for the
+ * keys being written, and the old row of a label that moved to a new key,
+ * are removed). Readers look labels up by the keys
+ * currently in `page_templates`, so retired rows are invisible to them.
+ * The previous `page_templates` and `page_template_labels` rows are read
+ * inside the same transaction that writes — there is no window in which
+ * another writer could change what the labels were derived from. An empty
+ * `templateKeysByUrl` therefore clears the classification but leaves every
+ * label row in place.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param params - See {@link ReplacePageTemplatesParams}.
+ */
+export async function replacePageTemplates(knex, params) {
+    const { templateKeysByUrl, clusterReasonsByTemplateKey } = params;
+    // Compressing every reason is pure CPU work independent of the DB — done
+    // before opening the transaction below so it doesn't extend how long the
+    // SQLite write-lock is held for.
+    const reasonRows = clusterReasonsByTemplateKey && clusterReasonsByTemplateKey.size > 0
+        ? [...clusterReasonsByTemplateKey].map(([templateKey, reason]) => {
+            const { body, codec, sizeRaw, sizeStored } = compressPayload(Buffer.from(serializeTemplateClusterReason(reason), 'utf8'));
+            return {
+                template_key: templateKey,
+                member_count: reason.memberCount,
+                reason_json: body,
+                codec,
+                size_raw: sizeRaw,
+                size_stored: sizeStored,
+            };
+        })
+        : [];
+    await knex.transaction(async (trx) => {
+        // Resolve the new membership and read the previous one *before* the
+        // deletes below — label inheritance needs both sides.
+        const urls = [...templateKeysByUrl.keys()];
+        const pageIdByUrl = new Map();
+        await eachSplitted(urls, 500, async (chunk) => {
+            const pageRows = await trx('content_items')
+                .join('url_refs', 'url_refs.id', 'content_items.url_id')
+                .select('content_items.id as id', 'url_refs.url as url')
+                .whereIn('url_refs.url', chunk);
+            for (const row of pageRows) {
+                pageIdByUrl.set(row.url, row.id);
+            }
+        });
+        const rows = [];
+        const clusters = new Map();
+        for (const [url, templateKey] of templateKeysByUrl) {
+            const pageId = pageIdByUrl.get(url);
+            if (pageId == null) {
+                continue;
+            }
+            rows.push({ page_id: pageId, template_key: templateKey });
+            let cluster = clusters.get(templateKey);
+            if (!cluster) {
+                cluster = { pageIds: [], urls: [] };
+                clusters.set(templateKey, cluster);
+            }
+            cluster.pageIds.push(pageId);
+            cluster.urls.push(url);
+        }
+        const previousLabels = await loadPreviousLabels(trx);
+        const labels = assignTemplateLabels({
+            clusters,
+            previousMembership: await loadPreviousMembership(trx),
+            previousLabels,
+        });
+        // Rows to drop: the keys being (re)written, plus previous keys whose
+        // label was inherited by a new key — the label moved, so its old row
+        // would otherwise linger as a duplicate of the same name.
+        const writtenPairs = new Set([...labels.values()].map(labelPair));
+        const staleKeys = [...labels.keys()];
+        for (const [previousKey, label] of previousLabels) {
+            if (!labels.has(previousKey) && writtenPairs.has(labelPair(label))) {
+                staleKeys.push(previousKey);
+            }
+        }
+        await trx('page_templates').delete();
+        await trx('page_template_clusters').delete();
+        await eachSplitted(staleKeys, 500, async (chunk) => {
+            await trx('page_template_labels').whereIn('template_key', chunk).delete();
+        });
+        if (reasonRows.length > 0) {
+            await eachSplitted(reasonRows, 100, async (chunk) => {
+                await trx('page_template_clusters').insert(chunk);
+            });
+        }
+        if (rows.length === 0) {
+            return;
+        }
+        await eachSplitted(rows, 500, async (chunk) => {
+            await trx('page_templates').insert(chunk);
+        });
+        const labelRows = [...labels].map(([templateKey, label]) => ({
+            template_key: templateKey,
+            section: label.section,
+            ordinal: label.ordinal,
+        }));
+        await eachSplitted(labelRows, 500, async (chunk) => {
+            await trx('page_template_labels').insert(chunk);
+        });
+    });
+}
+/**
+ * The identity of a label as a name: its `(section, ordinal)` pair.
+ * @param label - The label.
+ * @returns A string key unique per name.
+ */
+function labelPair(label) {
+    return `${label.section ?? '\0'}:${label.ordinal}`;
+}
+/**
+ * Reads the previous run's `page_templates` rows as template key → page ids.
+ * @param trx - The replacing transaction.
+ * @returns The previous membership; empty on a first run.
+ */
+async function loadPreviousMembership(trx) {
+    const previousRows = (await trx('page_templates').select('page_id as pageId', 'template_key as templateKey'));
+    const membership = new Map();
+    for (const row of previousRows) {
+        const pageIds = membership.get(row.templateKey);
+        if (pageIds) {
+            pageIds.push(row.pageId);
+        }
+        else {
+            membership.set(row.templateKey, [row.pageId]);
+        }
+    }
+    return membership;
+}
+/**
+ * Reads the previous run's `page_template_labels` rows.
+ * @param trx - The replacing transaction.
+ * @returns Template key → label; empty on a first run.
+ */
+async function loadPreviousLabels(trx) {
+    const labelRows = (await trx('page_template_labels').select('template_key as templateKey', 'section', 'ordinal'));
+    return new Map(labelRows.map((row) => [
+        row.templateKey,
+        { section: row.section, ordinal: row.ordinal },
+    ]));
+}

package/lib/db-ops/templates/serialize-template-cluster-reason.d.ts

@@ -0,0 +1,16 @@
+import type { TemplateClusterReason } from './types.js';
+/**
+ * Serializes a cluster reason to the JSON text that `page_template_clusters.reason_json`
+ * stores (before compression). The single definition of that format:
+ * `replacePageTemplates` writes it, and the template-classification golden
+ * pins it, so a change here shows up as a golden difference instead of
+ * silently diverging from what the golden claims to cover.
+ * @param reason - The reason.
+ * @returns The JSON text.
+ * @example
+ * ```ts
+ * const json = serializeTemplateClusterReason(reason);
+ * const { body } = compressPayload(Buffer.from(json, 'utf8'));
+ * ```
+ */
+export declare function serializeTemplateClusterReason(reason: TemplateClusterReason): string;

package/lib/db-ops/templates/serialize-template-cluster-reason.js

@@ -0,0 +1,17 @@
+/**
+ * Serializes a cluster reason to the JSON text that `page_template_clusters.reason_json`
+ * stores (before compression). The single definition of that format:
+ * `replacePageTemplates` writes it, and the template-classification golden
+ * pins it, so a change here shows up as a golden difference instead of
+ * silently diverging from what the golden claims to cover.
+ * @param reason - The reason.
+ * @returns The JSON text.
+ * @example
+ * ```ts
+ * const json = serializeTemplateClusterReason(reason);
+ * const { body } = compressPayload(Buffer.from(json, 'utf8'));
+ * ```
+ */
+export function serializeTemplateClusterReason(reason) {
+    return JSON.stringify(reason);
+}

package/lib/db-ops/templates/types.d.ts

@@ -0,0 +1,103 @@
+/**
+ * The six landmark types a cluster reason's `landmarks` recognizes.
+ */
+export type TemplateClusterLandmarkType = 'header' | 'footer' | 'nav' | 'aside' | 'form' | 'search';
+/**
+ * Why a Pass-0 block (one of possibly several that merged into a final
+ * cluster) was formed — mirrors `@nitpicker/core`'s `TemplateBlockingReason`
+ * discriminated union.
+ */
+export type TemplateClusterBlockingReason = {
+    readonly kind: 'css';
+    readonly distinctiveStylesheetHrefs: readonly string[];
+} | {
+    readonly kind: 'path';
+    readonly pathKey: string;
+} | {
+    readonly kind: 'orphanMerge';
+    readonly pathKey: string;
+};
+/** One block's blocking key and the reason it was formed. */
+export interface TemplateClusterBlockingEvidence {
+    readonly blockKey: string;
+    readonly reason: TemplateClusterBlockingReason;
+}
+/** How common one landmark type is across a cluster's member pages. */
+export interface TemplateClusterLandmarkProfile {
+    readonly presenceRate: number;
+    readonly chromeRate: number;
+    readonly shellTokens: readonly string[];
+    readonly memberCountWithInstance: number;
+}
+/**
+ * A copy of the classifier's reason shape (`@nitpicker/core`'s
+ * `TemplateClusterReason`, built by `createTemplateClassificationRun`). Kept
+ * independent of it (rather than importing its type directly) because this
+ * is the persisted contract of `page_template_clusters.reason` that
+ * `@nitpicker/query` and the browser-side viewer build type values against,
+ * and removing or retyping a field in core's shape surfaces as a compile
+ * error at the assignment in `classifyPageTemplates` instead of silently
+ * changing what older archives are read as. A field core *adds* is not
+ * caught there (structural assignability) and is persisted as-is in the
+ * JSON blob; mirror it here and in `@nitpicker/query`'s
+ * `isTemplateClusterReason` when it should be read back. Core's reason is
+ * structurally assignable to this type.
+ */
+export interface TemplateClusterReason {
+    readonly memberCount: number;
+    readonly blocking: readonly TemplateClusterBlockingEvidence[];
+    readonly structuralCoreTokens: readonly string[];
+    readonly landmarks: Partial<Record<TemplateClusterLandmarkType, TemplateClusterLandmarkProfile>>;
+    readonly siblingClusterKeys: readonly string[];
+}
+/**
+ * The human-facing name of one template cluster, as stored in
+ * `page_template_labels`: rendered as `<section> template <letter>` (or
+ * `template <letter>` when `section` is `null`), where the letter is
+ * `ordinal` in A, B, …, Z, AA, AB, … form. Assigned by
+ * {@link import('./assign-template-labels.js').assignTemplateLabels}.
+ */
+export interface TemplateLabel {
+    /**
+     * The top-level URL directory every member page sits under (`events` for
+     * `/events/...`), or `null` when members span several directories or all
+     * sit at the site root — the label is then numbered site-wide.
+     */
+    readonly section: string | null;
+    /** 1-based position within `section`'s label sequence. */
+    readonly ordinal: number;
+}
+/** One cluster's membership as {@link import('./assign-template-labels.js').assignTemplateLabels} sees it. */
+export interface TemplateLabelClusterInput {
+    /** Resolved `content_items.id` of every member page. */
+    readonly pageIds: readonly number[];
+    /** URL of every member page, for deriving {@link TemplateLabel.section}. */
+    readonly urls: readonly string[];
+}
+/**
+ * Params for {@link import('./assign-template-labels.js').assignTemplateLabels}.
+ */
+export interface AssignTemplateLabelsParams {
+    /** The new classification: template key → members. */
+    readonly clusters: ReadonlyMap<string, TemplateLabelClusterInput>;
+    /**
+     * The previous classification's `page_templates` rows: template key →
+     * member page ids. Empty on a first run.
+     */
+    readonly previousMembership: ReadonlyMap<string, readonly number[]>;
+    /** The previous classification's `page_template_labels` rows. Empty on a first run. */
+    readonly previousLabels: ReadonlyMap<string, TemplateLabel>;
+}
+/**
+ * Params for {@link import('./replace-page-templates.js').replacePageTemplates}.
+ */
+export interface ReplacePageTemplatesParams {
+    /** Page URL → template key, as produced by `classifyPageTemplates`. */
+    readonly templateKeysByUrl: ReadonlyMap<string, string>;
+    /**
+     * Template key → page-cluster's cluster-selection evidence for
+     * that key, as produced by `classifyPageTemplates`. Omitted entirely (not
+     * just empty) when the caller didn't request reasons.
+     */
+    readonly clusterReasonsByTemplateKey?: ReadonlyMap<string, TemplateClusterReason>;
+}

package/lib/db-ops/templates/types.js

@@ -0,0 +1 @@
+export {};

package/lib/debug.d.ts

@@ -0,0 +1,8 @@
+/** Debug logger for the archive package. Namespace: `Nitpicker:Utils:Archive`. */
+export declare const log: import("debug").Debugger;
+/** Debug logger for archive save operations. Namespace: `Nitpicker:Utils:Archive:Save`. */
+export declare const saveLog: import("debug").Debugger;
+/** Debug logger for database operations. Namespace: `Nitpicker:Utils:Archive:DB`. */
+export declare const dbLog: import("debug").Debugger;
+/** Debug logger for archive errors. Namespace: `Nitpicker:Utils:Archive:Error`. */
+export declare const errorLog: import("debug").Debugger;

package/lib/debug.js

@@ -0,0 +1,9 @@
+import { log as globalLog } from './utils/debug.js';
+/** Debug logger for the archive package. Namespace: `Nitpicker:Utils:Archive`. */
+export const log = globalLog.extend('Archive');
+/** Debug logger for archive save operations. Namespace: `Nitpicker:Utils:Archive:Save`. */
+export const saveLog = log.extend('Save');
+/** Debug logger for database operations. Namespace: `Nitpicker:Utils:Archive:DB`. */
+export const dbLog = log.extend('DB');
+/** Debug logger for archive errors. Namespace: `Nitpicker:Utils:Archive:Error`. */
+export const errorLog = log.extend('Error');

package/lib/decode-html-blob.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Decodes a stored HTML body BLOB according to its codec marker. The codec
+ * column on `page_html_blobs` records which encoding was applied at write
+ * time so individual rows can be migrated to a different compression without
+ * a table-wide rewrite.
+ *
+ * Decodes the bytes alone, so an error names no row; a reader that knows the
+ * page (`getHtmlOfPageById`, the 0.13 migration's `populate-image-items.ts`)
+ * decodes through `decodePageHtmlBlobBytes` instead, which names the page and
+ * the blob hash.
+ * @param body - Raw bytes as stored in `page_html_blobs.body`.
+ * @param codec - The `codec` column value (e.g. `'zstd'`, `'none'`).
+ * @returns The decoded HTML string.
+ * @throws {Error} If the codec is not recognised, or a zstd blob decompresses past `MAX_DECODED_SNAPSHOT_BYTES`.
+ * @example
+ * ```ts
+ * const html = decodeStoredBlob(row.body, row.codec);
+ * ```
+ */
+export declare function decodeStoredBlob(body: Uint8Array, codec: string): string;

package/lib/decode-html-blob.js

@@ -0,0 +1,23 @@
+import { decodeStoredBlobBytes } from './decode-stored-blob-bytes.js';
+/**
+ * Decodes a stored HTML body BLOB according to its codec marker. The codec
+ * column on `page_html_blobs` records which encoding was applied at write
+ * time so individual rows can be migrated to a different compression without
+ * a table-wide rewrite.
+ *
+ * Decodes the bytes alone, so an error names no row; a reader that knows the
+ * page (`getHtmlOfPageById`, the 0.13 migration's `populate-image-items.ts`)
+ * decodes through `decodePageHtmlBlobBytes` instead, which names the page and
+ * the blob hash.
+ * @param body - Raw bytes as stored in `page_html_blobs.body`.
+ * @param codec - The `codec` column value (e.g. `'zstd'`, `'none'`).
+ * @returns The decoded HTML string.
+ * @throws {Error} If the codec is not recognised, or a zstd blob decompresses past `MAX_DECODED_SNAPSHOT_BYTES`.
+ * @example
+ * ```ts
+ * const html = decodeStoredBlob(row.body, row.codec);
+ * ```
+ */
+export function decodeStoredBlob(body, codec) {
+    return decodeStoredBlobBytes(body, codec).toString('utf8');
+}

package/lib/decode-page-html-blob-bytes.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Decodes a page's stored HTML snapshot to its UTF-8 bytes, as
+ * {@link decodeStoredBlobBytes} does, naming the page and the blob when it
+ * cannot be decoded (corrupt zstd data, a zstd blob that decompresses past
+ * `MAX_DECODED_SNAPSHOT_BYTES`, or an unknown codec). The decoder sees only
+ * the bytes, so without the page and hash its error points at no row.
+ * @param row - The page and the `page_html_blobs` row it references.
+ * @param row.pageId - The page's id.
+ * @param row.hash - `page_html_blobs.hash`.
+ * @param row.body - `page_html_blobs.body`.
+ * @param row.codec - `page_html_blobs.codec`.
+ * @returns The HTML as UTF-8 bytes.
+ * @throws {Error} `HTML snapshot of page <id> (page_html_blobs hash <hex>) could not be decoded: <reason>`,
+ *   with the decoder's error as its `cause`.
+ * @example
+ * ```ts
+ * const html = decodePageHtmlBlobBytes({ pageId, hash, body, codec }).toString('utf8');
+ * ```
+ */
+export declare function decodePageHtmlBlobBytes(row: {
+    readonly pageId: number;
+    readonly hash: Uint8Array;
+    readonly body: Uint8Array;
+    readonly codec: string;
+}): Buffer;

package/lib/decode-page-html-blob-bytes.js

@@ -0,0 +1,29 @@
+import { decodeStoredBlobBytes } from './decode-stored-blob-bytes.js';
+/**
+ * Decodes a page's stored HTML snapshot to its UTF-8 bytes, as
+ * {@link decodeStoredBlobBytes} does, naming the page and the blob when it
+ * cannot be decoded (corrupt zstd data, a zstd blob that decompresses past
+ * `MAX_DECODED_SNAPSHOT_BYTES`, or an unknown codec). The decoder sees only
+ * the bytes, so without the page and hash its error points at no row.
+ * @param row - The page and the `page_html_blobs` row it references.
+ * @param row.pageId - The page's id.
+ * @param row.hash - `page_html_blobs.hash`.
+ * @param row.body - `page_html_blobs.body`.
+ * @param row.codec - `page_html_blobs.codec`.
+ * @returns The HTML as UTF-8 bytes.
+ * @throws {Error} `HTML snapshot of page <id> (page_html_blobs hash <hex>) could not be decoded: <reason>`,
+ *   with the decoder's error as its `cause`.
+ * @example
+ * ```ts
+ * const html = decodePageHtmlBlobBytes({ pageId, hash, body, codec }).toString('utf8');
+ * ```
+ */
+export function decodePageHtmlBlobBytes(row) {
+    try {
+        return decodeStoredBlobBytes(row.body, row.codec);
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new Error(`HTML snapshot of page ${row.pageId} (page_html_blobs hash ${Buffer.from(row.hash).toString('hex')}) could not be decoded: ${reason}`, { cause: error });
+    }
+}