@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/archive/verify-migration/check-anchor-edges-count.js

@@ -0,0 +1,72 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #3: the `anchor_edges` dedup does not
+ * eliminate rows and does not create phantom rows.
+ *
+ * `populateAnchorEdges` collapses N `anchors` rows per `(pageId, hrefId)`
+ * pair into a single `anchor_edges` row with `count = N`. Two ways it
+ * could go wrong:
+ *
+ * - `count(anchor_edges) == 0 && count(anchors) > 0` — dedup annihilated
+ *   the graph. A populate bug or a group-by mis-key would trip this.
+ * - `count(anchor_edges) > count(anchors)` — dedup produced more rows than
+ *   inputs. Impossible under a correct `GROUP BY` but a phantom-insert bug
+ *   in the populate loop would trip this.
+ *
+ * `count(anchor_edges) == count(anchors)` is **not** a failure. Issue #194
+ * spelled the upper bound as `< count(anchors)` (strict), but a legitimate
+ * small crawl where every `(pageId, hrefId)` pair is unique produces one
+ * edge per anchor (count=1) and satisfies the identity naturally. Enforcing
+ * strict `<` would refuse to migrate such archives even though every other
+ * invariant holds; the true invariant is "no rows added or dropped by
+ * dedup" (`anchor_edges <= anchors`), so the code enforces the weaker,
+ * correct clause. Check #4 (`SUM(count) == count(anchors)`) already
+ * establishes that no rows were silently discarded when the counts happen
+ * to match.
+ *
+ * A trivial archive with zero `anchors` legitimately produces zero
+ * `anchor_edges`, so both counts being zero passes; a non-zero `anchor_edges`
+ * with a zero `anchors` still throws — that would mean rows appeared from
+ * nowhere.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the invariant is violated.
+ */
+export async function checkAnchorEdgesCount(trx) {
+    const anchorEdgesRows = await trx('anchor_edges').count({ n: '*' });
+    const anchorsRows = await trx('anchors').count({ n: '*' });
+    const anchorEdgesCount = Number(anchorEdgesRows[0].n);
+    const anchorsCount = Number(anchorsRows[0].n);
+    if (anchorsCount === 0) {
+        if (anchorEdgesCount !== 0) {
+            throw new MigrationVerificationError({
+                check: '#3 anchor_edges row count',
+                context: {
+                    anchor_edges: anchorEdgesCount,
+                    anchors: anchorsCount,
+                    reason: 'anchor_edges must be empty when anchors is empty',
+                },
+            });
+        }
+        return;
+    }
+    if (anchorEdgesCount === 0) {
+        throw new MigrationVerificationError({
+            check: '#3 anchor_edges row count',
+            context: {
+                anchor_edges: anchorEdgesCount,
+                anchors: anchorsCount,
+                reason: 'anchor_edges must be greater than zero when anchors is non-empty',
+            },
+        });
+    }
+    if (anchorEdgesCount > anchorsCount) {
+        throw new MigrationVerificationError({
+            check: '#3 anchor_edges row count',
+            context: {
+                anchor_edges: anchorEdgesCount,
+                anchors: anchorsCount,
+                reason: 'anchor_edges must not exceed anchors (dedup should not create rows)',
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-anchor-edges-sum.d.ts

@@ -0,0 +1,13 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #4: the sum of `anchor_edges.count` equals the
+ * total number of legacy `anchors` rows.
+ *
+ * `populateAnchorEdges` collapses N `anchors` rows per `(pageId, hrefId)`
+ * pair into a single edge with `count = N`; the invariant re-establishes the identity
+ * `sum(count) = number of source rows`. Together with invariant #3 this
+ * proves the dedup is exact (no rows lost, no rows double-counted).
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the sum diverges from the anchors row count.
+ */
+export declare function checkAnchorEdgesSum(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-anchor-edges-sum.js

@@ -0,0 +1,27 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #4: the sum of `anchor_edges.count` equals the
+ * total number of legacy `anchors` rows.
+ *
+ * `populateAnchorEdges` collapses N `anchors` rows per `(pageId, hrefId)`
+ * pair into a single edge with `count = N`; the invariant re-establishes the identity
+ * `sum(count) = number of source rows`. Together with invariant #3 this
+ * proves the dedup is exact (no rows lost, no rows double-counted).
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the sum diverges from the anchors row count.
+ */
+export async function checkAnchorEdgesSum(trx) {
+    const sumRows = await trx('anchor_edges').sum({ n: 'count' });
+    const anchorsRows = await trx('anchors').count({ n: '*' });
+    const anchorEdgesSum = Number(sumRows[0].n ?? 0);
+    const anchorsCount = Number(anchorsRows[0].n);
+    if (anchorEdgesSum !== anchorsCount) {
+        throw new MigrationVerificationError({
+            check: '#4 anchor_edges count sum',
+            context: {
+                sum_of_anchor_edges_count: anchorEdgesSum,
+                anchors: anchorsCount,
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-content-items-count.d.ts

@@ -0,0 +1,16 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #1: every legacy `pages` row is mirrored by
+ * one row in `content_items`.
+ *
+ * `populateContentItems` fills `content_items` with `id = pages.id` for every page
+ * (see `populate-content-items.ts`); the invariant is broken only if the
+ * populate loop skipped or double-inserted rows. `INSERT OR IGNORE` on the
+ * PK collapses duplicates, so a mismatch always means "populate wrote
+ * fewer rows than expected" (extra rows would be impossible via the natural
+ * PK). We still emit both counts in the error context so operators can see
+ * the direction of the discrepancy at a glance.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export declare function checkContentItemsCount(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-content-items-count.js

@@ -0,0 +1,30 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #1: every legacy `pages` row is mirrored by
+ * one row in `content_items`.
+ *
+ * `populateContentItems` fills `content_items` with `id = pages.id` for every page
+ * (see `populate-content-items.ts`); the invariant is broken only if the
+ * populate loop skipped or double-inserted rows. `INSERT OR IGNORE` on the
+ * PK collapses duplicates, so a mismatch always means "populate wrote
+ * fewer rows than expected" (extra rows would be impossible via the natural
+ * PK). We still emit both counts in the error context so operators can see
+ * the direction of the discrepancy at a glance.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export async function checkContentItemsCount(trx) {
+    const contentItemsRows = await trx('content_items').count({ n: '*' });
+    const pagesRows = await trx('pages').count({ n: '*' });
+    const contentItemsCount = Number(contentItemsRows[0].n);
+    const pagesCount = Number(pagesRows[0].n);
+    if (contentItemsCount !== pagesCount) {
+        throw new MigrationVerificationError({
+            check: '#1 content_items row count',
+            context: {
+                content_items: contentItemsCount,
+                pages: pagesCount,
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-content-type-preservation.d.ts

@@ -0,0 +1,22 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #7: no page loses its content-type across the
+ * `content_items` populate.
+ *
+ * `populate-content-items.ts` sets `content_type_id = null` whenever the
+ * legacy `pages.contentType` is null OR the empty string (the empty string
+ * meaning "no content-type recorded"). Any *other* case where `contentType`
+ * carries a real value but `content_items.content_type_id` ended up null
+ * would indicate a resolver bug — the current populate throws before insert
+ * in that path, so this check is a defence in depth against a future
+ * regression.
+ *
+ * The check runs a single `SELECT id, contentType LIMIT 1` filtered to
+ * offending rows; if any row comes back the invariant is violated and the
+ * error context surfaces that one sample for the operator. A separate
+ * `COUNT(*)` would only add cost without adding information — the presence
+ * of a single row already means the invariant fails.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when any content-type is silently dropped.
+ */
+export declare function checkContentTypePreservation(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-content-type-preservation.js

@@ -0,0 +1,40 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #7: no page loses its content-type across the
+ * `content_items` populate.
+ *
+ * `populate-content-items.ts` sets `content_type_id = null` whenever the
+ * legacy `pages.contentType` is null OR the empty string (the empty string
+ * meaning "no content-type recorded"). Any *other* case where `contentType`
+ * carries a real value but `content_items.content_type_id` ended up null
+ * would indicate a resolver bug — the current populate throws before insert
+ * in that path, so this check is a defence in depth against a future
+ * regression.
+ *
+ * The check runs a single `SELECT id, contentType LIMIT 1` filtered to
+ * offending rows; if any row comes back the invariant is violated and the
+ * error context surfaces that one sample for the operator. A separate
+ * `COUNT(*)` would only add cost without adding information — the presence
+ * of a single row already means the invariant fails.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when any content-type is silently dropped.
+ */
+export async function checkContentTypePreservation(trx) {
+    const sample = await trx('content_items as ci')
+        .join('pages as p', 'ci.id', 'p.id')
+        .whereNull('ci.content_type_id')
+        .whereNotNull('p.contentType')
+        .andWhere('p.contentType', '<>', '')
+        .select('ci.id as id', 'p.contentType as contentType')
+        .limit(1);
+    if (sample.length > 0) {
+        const offending = sample[0];
+        throw new MigrationVerificationError({
+            check: '#7 content_type preservation',
+            context: {
+                sample_page_id: offending.id,
+                sample_content_type: offending.contentType,
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-foreign-key-integrity.d.ts

@@ -0,0 +1,31 @@
+import type { Knex } from 'knex';
+/**
+ * Asserts that `PRAGMA foreign_key_check` reports zero violations across
+ * the whole archive.
+ *
+ * Runs after `retargetLegacyFkTables` + `dropLegacyTables`, i.e. against
+ * the final schema — every FK declaration at that point targets the
+ * entity / ref tables, so a clean result proves the rebuilt adjunct
+ * tables' rows all resolve against `content_items` and no dangling
+ * reference survived the legacy-table removal. Running it earlier would
+ * also scan the legacy tables (`anchors` alone can hold millions of
+ * rows) for declarations that are about to be dropped anyway.
+ *
+ * `PRAGMA foreign_key_check` inspects data regardless of the
+ * `foreign_keys` enforcement setting, so this check stays valid even
+ * though the preceding drop phase runs with enforcement OFF.
+ *
+ * Not part of {@link verifyMigration}'s chain: that orchestrator runs
+ * inside the entity-populate transaction while the legacy tables still
+ * exist, a different point in the migration lifecycle than this
+ * final-schema assertion.
+ * @param trx - Knex instance or transaction on the migrated archive.
+ * @throws {MigrationVerificationError} when any row violates an FK
+ *   declaration; the context carries the violation count and the first
+ *   offending child/parent table pair.
+ * @example
+ * await retargetLegacyFkTables(trx);
+ * await dropLegacyTables(trx);
+ * await checkForeignKeyIntegrity(trx); // throws unless zero violations
+ */
+export declare function checkForeignKeyIntegrity(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-foreign-key-integrity.js

@@ -0,0 +1,47 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Asserts that `PRAGMA foreign_key_check` reports zero violations across
+ * the whole archive.
+ *
+ * Runs after `retargetLegacyFkTables` + `dropLegacyTables`, i.e. against
+ * the final schema — every FK declaration at that point targets the
+ * entity / ref tables, so a clean result proves the rebuilt adjunct
+ * tables' rows all resolve against `content_items` and no dangling
+ * reference survived the legacy-table removal. Running it earlier would
+ * also scan the legacy tables (`anchors` alone can hold millions of
+ * rows) for declarations that are about to be dropped anyway.
+ *
+ * `PRAGMA foreign_key_check` inspects data regardless of the
+ * `foreign_keys` enforcement setting, so this check stays valid even
+ * though the preceding drop phase runs with enforcement OFF.
+ *
+ * Not part of {@link verifyMigration}'s chain: that orchestrator runs
+ * inside the entity-populate transaction while the legacy tables still
+ * exist, a different point in the migration lifecycle than this
+ * final-schema assertion.
+ * @param trx - Knex instance or transaction on the migrated archive.
+ * @throws {MigrationVerificationError} when any row violates an FK
+ *   declaration; the context carries the violation count and the first
+ *   offending child/parent table pair.
+ * @example
+ * await retargetLegacyFkTables(trx);
+ * await dropLegacyTables(trx);
+ * await checkForeignKeyIntegrity(trx); // throws unless zero violations
+ */
+export async function checkForeignKeyIntegrity(trx) {
+    const rows = await trx
+        .select('*')
+        .from(trx.raw('pragma_foreign_key_check'));
+    if (rows.length === 0) {
+        return;
+    }
+    throw new MigrationVerificationError({
+        check: 'foreign_key_check',
+        context: {
+            violation_count: rows.length,
+            first_offending_table: rows[0]?.table ?? '(unknown)',
+            first_offending_parent: rows[0]?.parent ?? '(unknown)',
+            first_offending_rowid: rows[0]?.rowid ?? null,
+        },
+    });
+}

package/lib/archive/verify-migration/check-image-items-count.d.ts

@@ -0,0 +1,12 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #5: every legacy `images` row is mirrored by
+ * one row in `image_items`.
+ *
+ * `populateImageItems` fills `image_items` with the same PK as `images.id` (see
+ * `populate-image-items.ts`); the invariant is broken only if the populate
+ * loop skipped rows during URL/blob routing or dom-path derivation.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export declare function checkImageItemsCount(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-image-items-count.js

@@ -0,0 +1,26 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #5: every legacy `images` row is mirrored by
+ * one row in `image_items`.
+ *
+ * `populateImageItems` fills `image_items` with the same PK as `images.id` (see
+ * `populate-image-items.ts`); the invariant is broken only if the populate
+ * loop skipped rows during URL/blob routing or dom-path derivation.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export async function checkImageItemsCount(trx) {
+    const imageItemsRows = await trx('image_items').count({ n: '*' });
+    const imagesRows = await trx('images').count({ n: '*' });
+    const imageItemsCount = Number(imageItemsRows[0].n);
+    const imagesCount = Number(imagesRows[0].n);
+    if (imageItemsCount !== imagesCount) {
+        throw new MigrationVerificationError({
+            check: '#5 image_items row count',
+            context: {
+                image_items: imageItemsCount,
+                images: imagesCount,
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-page-meta-count.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #2: every scraped page has one `page_meta`
+ * row.
+ *
+ * `page_meta` mirrors the meta columns of `pages` for `scraped = 1` rows
+ * only (see `populate-page-meta.ts`); un-scraped pages have no meta to
+ * carry. A mismatch usually indicates that the populate loop encountered a
+ * scraped page whose meta resolution failed silently (e.g. a text ref that
+ * did not deduplicate), in which case the archive would be missing meta for
+ * an otherwise valid page.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export declare function checkPageMetaCount(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-page-meta-count.js

@@ -0,0 +1,31 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Verifies 0.13 invariant #2: every scraped page has one `page_meta`
+ * row.
+ *
+ * `page_meta` mirrors the meta columns of `pages` for `scraped = 1` rows
+ * only (see `populate-page-meta.ts`); un-scraped pages have no meta to
+ * carry. A mismatch usually indicates that the populate loop encountered a
+ * scraped page whose meta resolution failed silently (e.g. a text ref that
+ * did not deduplicate), in which case the archive would be missing meta for
+ * an otherwise valid page.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export async function checkPageMetaCount(trx) {
+    const pageMetaRows = await trx('page_meta').count({ n: '*' });
+    const scrapedPagesRows = await trx('pages')
+        .where('scraped', true)
+        .count({ n: '*' });
+    const pageMetaCount = Number(pageMetaRows[0].n);
+    const scrapedPagesCount = Number(scrapedPagesRows[0].n);
+    if (pageMetaCount !== scrapedPagesCount) {
+        throw new MigrationVerificationError({
+            check: '#2 page_meta row count',
+            context: {
+                page_meta: pageMetaCount,
+                scraped_pages: scrapedPagesCount,
+            },
+        });
+    }
+}

package/lib/archive/verify-migration/check-reader-parity.d.ts

@@ -0,0 +1,23 @@
+import type { Knex } from 'knex';
+/**
+ * Compares totals reported by 0.13 readers (which query the 0.13
+ * entity tables exclusively) against equivalent inline SQL against the
+ * still-present legacy write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`). A mismatch here means either the
+ * 0.13 populates lost rows or one of the new reader implementations
+ * has a filter/predicate off-by-one relative to its legacy counterpart.
+ *
+ * This is deliberately lighter than the "byte-identical items array"
+ * verification issue #195 acceptance references — running every reader on
+ * every archive and diffing full result sets would balloon the migrator's
+ * runtime and require importing `@nitpicker/query` back into the crawler
+ * package (a dependency inversion). The eight totals below capture every
+ * reader whose scope was flagged in issue #195 and catch the classes of
+ * regression the 0.13 row-count invariants cannot: predicate drift
+ * inside a reader function (e.g. a `WHERE contentType='text/html'` filter
+ * silently swapping meaning between `pages.contentType` and
+ * `content_type_refs.raw`).
+ * @param trx - The open migration transaction.
+ * @throws {MigrationVerificationError} If any pair of totals disagrees.
+ */
+export declare function checkReaderParity(trx: Knex): Promise<void>;

package/lib/archive/verify-migration/check-reader-parity.js

@@ -0,0 +1,211 @@
+import { MigrationVerificationError } from './types.js';
+/**
+ * Compares totals reported by 0.13 readers (which query the 0.13
+ * entity tables exclusively) against equivalent inline SQL against the
+ * still-present legacy write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`). A mismatch here means either the
+ * 0.13 populates lost rows or one of the new reader implementations
+ * has a filter/predicate off-by-one relative to its legacy counterpart.
+ *
+ * This is deliberately lighter than the "byte-identical items array"
+ * verification issue #195 acceptance references — running every reader on
+ * every archive and diffing full result sets would balloon the migrator's
+ * runtime and require importing `@nitpicker/query` back into the crawler
+ * package (a dependency inversion). The eight totals below capture every
+ * reader whose scope was flagged in issue #195 and catch the classes of
+ * regression the 0.13 row-count invariants cannot: predicate drift
+ * inside a reader function (e.g. a `WHERE contentType='text/html'` filter
+ * silently swapping meaning between `pages.contentType` and
+ * `content_type_refs.raw`).
+ * @param trx - The open migration transaction.
+ * @throws {MigrationVerificationError} If any pair of totals disagrees.
+ */
+export async function checkReaderParity(trx) {
+    // `analysis_violations` postdates issue #116 — an archive from before
+    // that feature landed (and that never ran `nitpicker analyze` after
+    // upgrading) has no such table at all. Unlike the other legacy
+    // tables here (`pages` / `anchors` / `images` / `resources`), which
+    // `init-schema.ts` has always created, this one is genuinely absent
+    // on some real archives, so the check below is skipped rather than
+    // querying a table that may not exist.
+    const hasAnalysisViolations = await trx.schema.hasTable('analysis_violations');
+    const checks = [
+        {
+            label: 'listPages default (scraped=1, redirect null, html-or-null)',
+            legacy: async () => scalar(trx('pages')
+                .where('scraped', 1)
+                .whereNull('redirectDestId')
+                .where((qb) => {
+                qb.where('isSkipped', 0).orWhereNull('isSkipped');
+            })
+                .where((qb) => {
+                qb.whereNull('contentType').orWhere('contentType', 'text/html');
+            })
+                .count({ count: '*' })),
+            current: async () => scalar(trx('content_items as ci')
+                .leftJoin('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
+                .where('ci.scraped', 1)
+                .whereNull('ci.redirect_dest_id')
+                .where((qb) => {
+                qb.where('ci.is_skipped', 0).orWhereNull('ci.is_skipped');
+            })
+                .where((qb) => {
+                qb.whereNull('ctr.raw').orWhere('ctr.raw', 'text/html');
+            })
+                .count({ count: '*' })),
+        },
+        {
+            label: 'listImages total',
+            legacy: async () => scalar(trx('images').count({ count: '*' })),
+            current: async () => scalar(trx('image_items').count({ count: '*' })),
+        },
+        {
+            label: 'listResources total',
+            legacy: async () => scalar(trx('resources').count({ count: '*' })),
+            current: async () => scalar(trx('resource_items').count({ count: '*' })),
+        },
+        {
+            label: 'listLinks broken (status=404 through redirect)',
+            legacy: async () => scalar(trx('anchors')
+                .join('pages as dest', 'anchors.hrefId', 'dest.id')
+                .leftJoin('pages as canonical', 'dest.redirectDestId', 'canonical.id')
+                .whereRaw('COALESCE("canonical"."status", "dest"."status") = 404')
+                .count({ count: '*' })),
+            current: async () => scalar(trx('anchor_edges as ae')
+                .join('content_items as dest', 'ae.href_page_id', 'dest.id')
+                .leftJoin('content_items as canonical', 'dest.redirect_dest_id', 'canonical.id')
+                .whereRaw('COALESCE("canonical"."status", "dest"."status") = 404')
+                .sum({ count: 'ae.count' })),
+        },
+        {
+            label: 'checkHeaders scope (internal html scraped, no redirect)',
+            legacy: async () => scalar(trx('pages')
+                .where({ scraped: 1, isExternal: 0, contentType: 'text/html' })
+                .whereNull('redirectDestId')
+                .count({ count: '*' })),
+            current: async () => scalar(trx('content_items as ci')
+                .join('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
+                .where({
+                'ci.scraped': 1,
+                'ci.is_external': 0,
+                'ctr.raw': 'text/html',
+            })
+                .whereNull('ci.redirect_dest_id')
+                .count({ count: '*' })),
+        },
+        {
+            label: 'findDuplicates(title) group count',
+            legacy: async () => scalar(trx
+                .count({ count: '*' })
+                .from(trx('pages')
+                .select('title')
+                .where({ scraped: 1, isExternal: 0, contentType: 'text/html' })
+                .whereNull('redirectDestId')
+                .whereNotNull('title')
+                .whereNot('title', '')
+                .groupBy('title')
+                .having(trx.raw('count(*) > 1'))
+                .as('g'))),
+            current: async () => scalar(trx.count({ count: '*' }).from(trx('content_items as ci')
+                .join('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
+                .join('page_meta as pm', 'pm.page_id', 'ci.id')
+                .join('text_refs as tr', 'tr.id', 'pm.title_text_id')
+                .select('pm.title_text_id')
+                .where({
+                'ci.scraped': 1,
+                'ci.is_external': 0,
+                'ctr.raw': 'text/html',
+            })
+                .whereNull('ci.redirect_dest_id')
+                .whereNotNull('pm.title_text_id')
+                .whereNot('tr.text', '')
+                .groupBy('pm.title_text_id')
+                .having(trx.raw('count(*) > 1'))
+                .as('g'))),
+        },
+        {
+            label: 'findMismatches(canonical) count',
+            legacy: async () => scalar(trx('pages')
+                .where({ scraped: 1, isExternal: 0, contentType: 'text/html' })
+                .whereNull('redirectDestId')
+                .whereNotNull('canonical')
+                .whereNot('canonical', '')
+                .whereRaw('canonical != url')
+                .count({ count: '*' })),
+            current: async () => scalar(trx('content_items as ci')
+                .join('content_type_refs as ctr', 'ctr.id', 'ci.content_type_id')
+                .join('page_meta as pm', 'pm.page_id', 'ci.id')
+                .join('url_refs as canonical_ur', 'canonical_ur.id', 'pm.canonical_url_id')
+                .where({
+                'ci.scraped': 1,
+                'ci.is_external': 0,
+                'ctr.raw': 'text/html',
+            })
+                .whereNull('ci.redirect_dest_id')
+                .whereNotNull('pm.canonical_url_id')
+                .whereNot('canonical_ur.url', '')
+                .whereRaw('"pm"."canonical_url_id" != "ci"."url_id"')
+                .count({ count: '*' })),
+        },
+        ...(hasAnalysisViolations
+            ? [
+                {
+                    // `getViolations` joins `analysis_violations` → `content_items`
+                    // → `url_refs` to project the page URL for each violation. The
+                    // current-path parity check exercises that JOIN chain so a
+                    // broken FK / missing `content_items` row surfaces here as a
+                    // row-count discrepancy (the legacy side counts `pages` rows
+                    // via `analysis_violations.page_id`, so a lost `content_items`
+                    // row makes the current-side INNER JOIN drop the violation).
+                    label: 'getViolations page-URL join (analysis_violations → content_items → url_refs)',
+                    legacy: async () => scalar(trx('analysis_violations as v')
+                        .join('pages as p', 'p.id', 'v.page_id')
+                        .count({ count: '*' })),
+                    current: async () => scalar(trx('analysis_violations as v')
+                        .join('content_items as p', 'p.id', 'v.page_id')
+                        .join('url_refs as ur', 'ur.id', 'p.url_id')
+                        .count({ count: '*' })),
+                },
+            ]
+            : []),
+    ];
+    const failures = [];
+    // Zero-vs-zero cannot detect a "populate silently produced no rows"
+    // bug, so a check whose LEGACY side returns 0 is skipped rather than
+    // counted as pass — the parity check exists to catch predicate drift
+    // once both sides have rows to compare. `checkContentItemsCount` etc.
+    // already guard the "current is empty when legacy is not" direction
+    // upstream in {@link verifyMigration}.
+    let comparedChecks = 0;
+    for (const { label, legacy, current } of checks) {
+        const [legacyValue, currentValue] = await Promise.all([legacy(), current()]);
+        if (legacyValue === 0 && currentValue === 0) {
+            continue;
+        }
+        comparedChecks++;
+        if (legacyValue !== currentValue) {
+            failures.push(`${label}: legacy=${legacyValue}, current=${currentValue}`);
+        }
+    }
+    if (failures.length > 0) {
+        throw new MigrationVerificationError({
+            check: '#9 reader parity',
+            context: {
+                failures: failures.join('; '),
+                compared_checks: comparedChecks,
+            },
+        });
+    }
+}
+/**
+ * Extracts the single numeric value from a Knex `count()` / `sum()` result.
+ * SQLite returns strings for aggregate expressions; we coerce and treat
+ * `null` as `0` (an empty aggregate).
+ * @param queryBuilder - The pending Knex query that resolves to a single
+ *   aggregate row.
+ * @returns The aggregate value as a JS number.
+ */
+async function scalar(queryBuilder) {
+    const rows = (await queryBuilder);
+    return Number(rows[0]?.count ?? 0);
+}

package/lib/archive/verify-migration/check-resource-items-count.d.ts

@@ -0,0 +1,17 @@
+import type { Knex } from 'knex';
+/**
+ * Verifies 0.13 invariant #6: every legacy `resources` row is mirrored
+ * by one row in `resource_items`.
+ *
+ * `populateResourceItems` fills `resource_items` with the same PK as
+ * `resources.id`
+ * (see `populate-resource-items.ts`); the invariant is broken only if the
+ * populate loop skipped rows during URL / header-set / content-type
+ * resolution. The paired invariant `resource_ref_edges = "resources-referrers"`
+ * is left unchecked here because issue #194's spec list stops at the
+ * six enumerated checks, and `resource_ref_edges` populate is a straight
+ * `INSERT … SELECT` (no dedup, no resolution) with negligible failure surface.
+ * @param trx - Knex instance or transaction connected to the populated archive.
+ * @throws {MigrationVerificationError} when the row counts diverge.
+ */
+export declare function checkResourceItemsCount(trx: Knex): Promise<void>;