@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/archive/populate-entity-tables/populate-content-items.js

@@ -0,0 +1,141 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { loadContentTypeRefs } from './resolve-content-type-refs.js';
+import { resolveHeaderSets } from './resolve-header-sets.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against `pages`. The
+ * migration reads every page column that maps to `content_items` in one
+ * SELECT per chunk, so scaling this too high would blow up the result-set
+ * size for long meta text columns. 500 keeps peak memory bounded at
+ * ≈ 30 MB for a chunk of `pages` with populated meta and header sets.
+ */
+const READ_CHUNK_SIZE = 500;
+/**
+ * Rows sent per `INSERT INTO content_items ... VALUES (...)` statement.
+ * Each row binds 16 params (id + 15 core columns), so 300 rows =
+ * 4 800 params — safely under SQLite's default variable limit of 32 766.
+ */
+const INSERT_CHUNK_SIZE = 300;
+/**
+ * Populates `content_items` from `pages` (issue #193).
+ *
+ * Strategy per chunk:
+ *
+ * 1. **Keyset-paginate** `pages` in id order so partial-failure re-runs
+ *    resume from the last committed row without reading the full table.
+ *    Same pattern as {@link ../populate-ref-tables/populate-url-refs.ts}.
+ * 2. **Batch-resolve** `url_refs.id`, `content_type_refs.id`, and
+ *    `header_sets.id` for every distinct value in the chunk — one round
+ *    trip per ref table instead of N round trips per row.
+ * 3. **Bulk INSERT** with explicit `id = pages.id`. Reusing the legacy
+ *    PK preserves every FK reference in `page_errors` / `page_tags` /
+ *    `page_jsonld` / `page_html_ref` without a per-row UPDATE.
+ *
+ * `redirect_dest_id` is copied verbatim from `redirectDestId` — the FK
+ * self-reference is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}) so a redirect source
+ * inserted before its destination is validated only at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent — a
+ * re-run after a partial failure only inserts the rows that are still
+ * missing.
+ *
+ * `content_type_refs` is preloaded once at the start because its
+ * cardinality is small (see {@link ./resolve-content-type-refs.ts}); the
+ * URL and header-set resolvers are called per chunk because their
+ * per-archive cardinality can be very large (one row per distinct URL /
+ * response JSON).
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ * });
+ */
+export async function populateContentItems(trx, onProgress) {
+    const contentTypeIds = await loadContentTypeRefs(trx);
+    const countRows = await trx('pages').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('content_items (pages)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('pages')
+            .select('id', 'url', 'redirectDestId', 'scraped', 'isTarget', 'isExternal', 'status', 'statusText', 'contentType', 'contentLength', 'responseHeaders', 'firstCrawledAt', 'lastCrawledAt', 'order', 'isSkipped', 'skipReason', 'source')
+            .where('id', '>', cursor)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        const urls = new Set();
+        const headerJsonStrings = new Set();
+        for (const row of rows) {
+            urls.add(row.url);
+            if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') {
+                headerJsonStrings.add(row.responseHeaders);
+            }
+        }
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const headerSetIds = await resolveHeaderSets(trx, headerJsonStrings);
+        const inserts = rows.map((row) => {
+            const urlId = urlIds.get(row.url) ?? null;
+            if (urlId === null) {
+                throw new Error(`populateContentItems: url_refs.id not resolved for page id=${row.id} url=${row.url} — populateUrlRefs must run first`);
+            }
+            let contentTypeId = null;
+            if (row.contentType != null && row.contentType !== '') {
+                const resolved = contentTypeIds.get(row.contentType);
+                if (resolved === undefined) {
+                    throw new Error(`populateContentItems: content_type_refs.id not resolved for page id=${row.id} contentType=${row.contentType} — populateContentTypeRefs must run first`);
+                }
+                contentTypeId = resolved;
+            }
+            let headerSetId = null;
+            if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') {
+                const resolved = headerSetIds.get(row.responseHeaders);
+                if (resolved === undefined) {
+                    // `resolveHeaderSets` skips sentinel values ('{}' / 'null')
+                    // AND payloads that `decomposeHeaderSet` cannot parse into
+                    // any entries — in both cases the source row genuinely had
+                    // no header set to point at, so `header_set_id = null` is
+                    // the correct final state. `resolveHeaderSets` already ran
+                    // the raw_hash fallback, so this branch only fires when the
+                    // row has no persistable header set (parse error, or every
+                    // entry stripped for non-string values).
+                    headerSetId = null;
+                }
+                else {
+                    headerSetId = resolved;
+                }
+            }
+            return {
+                id: row.id,
+                url_id: urlId,
+                is_external: row.isExternal == null ? 0 : row.isExternal ? 1 : 0,
+                scraped: row.scraped ? 1 : 0,
+                is_target: row.isTarget ? 1 : 0,
+                status: row.status ?? null,
+                status_text: row.statusText ?? null,
+                content_type_id: contentTypeId,
+                content_length: row.contentLength ?? null,
+                header_set_id: headerSetId,
+                redirect_dest_id: row.redirectDestId ?? null,
+                source: row.source,
+                first_crawled_at: row.firstCrawledAt ?? null,
+                last_crawled_at: row.lastCrawledAt ?? null,
+                crawl_order: row.order ?? null,
+                is_skipped: row.isSkipped == null ? null : row.isSkipped ? 1 : 0,
+                skip_reason: row.skipReason ?? null,
+            };
+        });
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('content_items').insert(chunk).onConflict('id').ignore();
+        }
+    }
+}

package/lib/archive/populate-entity-tables/populate-entities.d.ts

@@ -0,0 +1,81 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { PageDomPathResolver } from './populate-image-items.js';
+import type { Knex } from 'knex';
+/**
+ * Runs the six 0.13 entity/edge populates (issue #193) in dependency
+ * order against an already-connected archive.
+ *
+ * Order rationale:
+ *
+ * 1. **`content_items`** — every downstream populate's FKs reference
+ *    `content_items(id)`. Must land first so the FKs are valid at
+ *    COMMIT time. `redirect_dest_id` is `DEFERRABLE INITIALLY DEFERRED`
+ *    (see {@link ../create-entity-tables.ts}) so a redirect
+ *    source inserted before its destination is validated only at
+ *    COMMIT — the insert order within this populate is not
+ *    load-bearing.
+ * 2. **`page_meta`** — FK to `content_items(id)`.
+ * 3. **`resource_items`** — required before `resource_ref_edges`.
+ * 4. **`anchor_edges`** — FKs to `content_items(id)` on both
+ *    sides.
+ * 5. **`resource_ref_edges`** — FKs to `resource_items(id)` and
+ *    `content_items(id)`.
+ * 6. **`image_items`** — FK to `content_items(id)`; the
+ *    dom-path text_refs upsert is scoped to this populate so re-runs are
+ *    self-contained.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE` on
+ * its natural key or PK. On a **re-crawl** (`crawl --append` /
+ * `--retry-failed` / `--inventory` all UPDATE existing `pages` /
+ * `resources` rows in place, and `#insertPage` deletes + re-inserts
+ * `anchors` / `images` per page), running populate again against the
+ * previously-populated archive would leave `content_items` / `page_meta`
+ * / `resource_items` at their FIRST-crawl values (source-priority
+ * upgrades, refreshed status, changed metadata never propagate) and
+ * `anchor_edges` / `image_items` / `resource_ref_edges` would keep stale
+ * rows keyed by the now-deleted legacy ids. To keep the reader-side
+ * view of the archive faithful to `pages` / `resources` on re-crawl,
+ * every entity + edge table is TRUNCATEd (in child-first order so no FK
+ * check trips) at the top of this function before the six sub-populates
+ * re-insert from the current legacy state. Ref tables (`url_refs`,
+ * `text_refs`, `content_type_refs`, `header_*`) are NOT truncated —
+ * they are content-addressable and additive; re-inserting the same
+ * value hits `INSERT OR IGNORE` and is a no-op.
+ *
+ * The whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level (a single WAL transaction
+ * with `.bak` rollback on failure);
+ * this function does not open its own transaction so the
+ * caller controls the boundary — see {@link
+ * ../populate-ref-tables/populate-refs.ts} for the same convention.
+ *
+ * **`PRAGMA foreign_keys = ON` is REQUIRED** on the underlying
+ * connection before this function is called. `content_items.redirect_dest_id`
+ * is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}); without foreign-key
+ * enforcement libsql commits `content_items` rows whose `redirect_dest_id`
+ * points at a non-existent page and the deferrable-FK-enforcement-at-COMMIT
+ * invariant is silently broken. The
+ * migration script sets the pragma explicitly; any other caller must do
+ * the same.
+ * HTML BLOB reads happen inline inside {@link populateImageItems} against
+ * the same `trx` — no `getPageHtml` callback here — because a callback
+ * that routed through `Database.getHtmlOfPageById` would re-enter the
+ * connection pool from inside this writer transaction and deadlock on
+ * libsql's single writer connection. See `./populate-image-items.ts` for
+ * the detailed rationale.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images. Injected rather than hard-coded so
+ *   `@nitpicker/crawler` does not become a jsdom consumer at runtime.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateEntityTables(trx, jsdomResolver);
+ * });
+ * await archive.write();
+ */
+export declare function populateEntityTables(trx: Knex, resolvePageDomPaths: PageDomPathResolver, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-entity-tables/populate-entities.js

@@ -0,0 +1,111 @@
+import { populateAnchorEdges } from './populate-anchor-edges.js';
+import { populateContentItems } from './populate-content-items.js';
+import { populateImageItems } from './populate-image-items.js';
+import { populatePageMeta } from './populate-page-meta.js';
+import { populateResourceItems } from './populate-resource-items.js';
+import { populateResourceRefEdges } from './populate-resource-ref-edges.js';
+/**
+ * Runs the six 0.13 entity/edge populates (issue #193) in dependency
+ * order against an already-connected archive.
+ *
+ * Order rationale:
+ *
+ * 1. **`content_items`** — every downstream populate's FKs reference
+ *    `content_items(id)`. Must land first so the FKs are valid at
+ *    COMMIT time. `redirect_dest_id` is `DEFERRABLE INITIALLY DEFERRED`
+ *    (see {@link ../create-entity-tables.ts}) so a redirect
+ *    source inserted before its destination is validated only at
+ *    COMMIT — the insert order within this populate is not
+ *    load-bearing.
+ * 2. **`page_meta`** — FK to `content_items(id)`.
+ * 3. **`resource_items`** — required before `resource_ref_edges`.
+ * 4. **`anchor_edges`** — FKs to `content_items(id)` on both
+ *    sides.
+ * 5. **`resource_ref_edges`** — FKs to `resource_items(id)` and
+ *    `content_items(id)`.
+ * 6. **`image_items`** — FK to `content_items(id)`; the
+ *    dom-path text_refs upsert is scoped to this populate so re-runs are
+ *    self-contained.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE` on
+ * its natural key or PK. On a **re-crawl** (`crawl --append` /
+ * `--retry-failed` / `--inventory` all UPDATE existing `pages` /
+ * `resources` rows in place, and `#insertPage` deletes + re-inserts
+ * `anchors` / `images` per page), running populate again against the
+ * previously-populated archive would leave `content_items` / `page_meta`
+ * / `resource_items` at their FIRST-crawl values (source-priority
+ * upgrades, refreshed status, changed metadata never propagate) and
+ * `anchor_edges` / `image_items` / `resource_ref_edges` would keep stale
+ * rows keyed by the now-deleted legacy ids. To keep the reader-side
+ * view of the archive faithful to `pages` / `resources` on re-crawl,
+ * every entity + edge table is TRUNCATEd (in child-first order so no FK
+ * check trips) at the top of this function before the six sub-populates
+ * re-insert from the current legacy state. Ref tables (`url_refs`,
+ * `text_refs`, `content_type_refs`, `header_*`) are NOT truncated —
+ * they are content-addressable and additive; re-inserting the same
+ * value hits `INSERT OR IGNORE` and is a no-op.
+ *
+ * The whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level (a single WAL transaction
+ * with `.bak` rollback on failure);
+ * this function does not open its own transaction so the
+ * caller controls the boundary — see {@link
+ * ../populate-ref-tables/populate-refs.ts} for the same convention.
+ *
+ * **`PRAGMA foreign_keys = ON` is REQUIRED** on the underlying
+ * connection before this function is called. `content_items.redirect_dest_id`
+ * is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}); without foreign-key
+ * enforcement libsql commits `content_items` rows whose `redirect_dest_id`
+ * points at a non-existent page and the deferrable-FK-enforcement-at-COMMIT
+ * invariant is silently broken. The
+ * migration script sets the pragma explicitly; any other caller must do
+ * the same.
+ * HTML BLOB reads happen inline inside {@link populateImageItems} against
+ * the same `trx` — no `getPageHtml` callback here — because a callback
+ * that routed through `Database.getHtmlOfPageById` would re-enter the
+ * connection pool from inside this writer transaction and deadlock on
+ * libsql's single writer connection. See `./populate-image-items.ts` for
+ * the detailed rationale.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images. Injected rather than hard-coded so
+ *   `@nitpicker/crawler` does not become a jsdom consumer at runtime.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateEntityTables(trx, jsdomResolver);
+ * });
+ * await archive.write();
+ */
+export async function populateEntityTables(trx, resolvePageDomPaths, onProgress) {
+    // Truncate child-first so no outgoing FK check ever sees a broken
+    // reference mid-delete. Order is: leaf edge/entity tables that have
+    // no incoming FKs (`image_items`, `anchor_edges`, `resource_ref_edges`),
+    // then the parent entities they referenced (`page_meta` and
+    // `resource_items`; `page_meta` references `content_items` so it
+    // must be dropped before `content_items`; `resource_items` was
+    // pointed at by `resource_ref_edges` which is now empty), and
+    // finally `content_items` itself — the only remaining incoming
+    // reference is its own DEFERRABLE INITIALLY DEFERRED
+    // `redirect_dest_id`, which is validated at COMMIT so a mid-trx
+    // wipe-and-refill is legal. No external table has an enforced FK
+    // into `content_items` (viewer read-model tables and
+    // `analysis_violations` reference `pages(id)` or hold logical-only
+    // pointers), so the truncation stops here.
+    await trx('image_items').delete();
+    await trx('anchor_edges').delete();
+    await trx('resource_ref_edges').delete();
+    await trx('page_meta').delete();
+    await trx('resource_items').delete();
+    await trx('content_items').delete();
+    await populateContentItems(trx, onProgress);
+    await populatePageMeta(trx, onProgress);
+    await populateResourceItems(trx, onProgress);
+    await populateAnchorEdges(trx, onProgress);
+    await populateResourceRefEdges(trx);
+    await populateImageItems(trx, resolvePageDomPaths, onProgress);
+}

package/lib/archive/populate-entity-tables/populate-image-items.d.ts

@@ -0,0 +1,91 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { DomPathResult } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Callback that resolves DOM-path strings for one page's images. The
+ * callback receives the page's HTML string (or `null` when no snapshot
+ * is stored) plus the page's `images` rows, and must return one
+ * {@link DomPathResult} per row.
+ *
+ * Injected rather than hard-coded so `@nitpicker/crawler` does not
+ * become a jsdom consumer at runtime — the migration script
+ * (`scripts/migrate-to-0.13.mjs`) wires a jsdom-backed
+ * implementation, while unit tests inject stubs. A future crawler-time
+ * DOM-path capture can use the same signature with
+ * puppeteer-backed elements.
+ */
+export type PageDomPathResolver = (pageId: number, htmlString: string | null, images: readonly ImageRowForResolver[]) => Promise<ReadonlyMap<number, DomPathResult>>;
+/**
+ * The projection of `images` rows passed to a
+ * {@link PageDomPathResolver}. Kept as a narrow shape (only the fields
+ * the resolver needs) so callers of the resolver do not depend on the
+ * full row width.
+ */
+export interface ImageRowForResolver {
+    /** Legacy `images.id`. */
+    id: number;
+    /** Legacy `images.sourceCode`. */
+    sourceCode: string | null;
+}
+/**
+ * Populates `image_items` from `images` (issue #193).
+ *
+ * The outer loop iterates **pages**, not images. Each page's images are
+ * processed as one whole unit — dom-path derivation, HTML BLOB fetch,
+ * and ref lookups all happen once per page. This design closes three
+ * separate correctness / efficiency traps that a purely image-chunked
+ * loop would expose:
+ *
+ * 1. **Ordinal-cursor bleed across chunks** — `matchImagesToDomPaths`
+ *    tracks per-outerHTML cursor state, so a page with 1 000 identical
+ *    `<img>` tags spanning two image-chunks would reset the cursor at
+ *    the boundary and duplicate dom_paths already assigned in the
+ *    previous chunk. Iterating by page keeps every page's cursor state
+ *    self-contained.
+ * 2. **Repeated `getPageHtml` + jsdom parse** — a page whose images
+ *    straddle N image-chunks would fetch and re-parse the same multi-MB
+ *    HTML snapshot N times. Iterating by page makes this exactly once
+ *    per page.
+ * 3. **Data-URI routing gap** — legacy `images.src` can be either a
+ *    plain URL (goes to `url_refs`) or a large `data:` URI (goes to
+ *    `blob_refs`, per the 512-byte threshold). The lookups partition
+ *    by that rule, so a data URI is never bound into
+ *    `resolveUrlRefs`' `WHERE url IN (?)` — which could otherwise
+ *    exceed SQLite's `SQLITE_MAX_SQL_LENGTH` for multi-KB URIs.
+ *
+ * Per outer iteration:
+ *
+ * 1. **Keyset-paginate** distinct `pageId` values from `images`. Peak
+ *    memory ≈ 500 integers.
+ * 2. **Fetch all images** for those page ids in one query, ordered by
+ *    `(pageId, id)` so JS-side grouping is a linear pass.
+ * 3. **Resolve dom paths per page** — one `getPageHtml` + one resolver
+ *    call each. `dom_path` texts are collected across the whole batch
+ *    so `upsertTextRefs` runs once per batch.
+ * 4. **Partition src / currentSrc** by the data-URI routing rule and
+ *    batch-resolve url refs, blob refs, and alt text refs across the
+ *    whole batch.
+ * 5. **Bulk INSERT** with explicit `id = images.id`.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent.
+ *
+ * HTML BLOBs are read inline via the same `trx` handle rather than
+ * through a caller-supplied `getPageHtml` callback: a callback that
+ * queried `Database.getHtmlOfPageById` (which uses the non-transactional
+ * Knex instance) would re-enter the pool from inside this writer
+ * transaction and deadlock on libsql's single writer connection. Reading
+ * `page_html_ref` + `page_html_blobs` via `trx` keeps every read on the
+ * same connection the outer trx already owns.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images (see {@link PageDomPathResolver}).
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of distinct `images.pageId` values scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * const jsdomResolver = createJsdomResolver();
+ * await knex.transaction(async (trx) => {
+ *   await populateImageItems(trx, jsdomResolver);
+ * });
+ */
+export declare function populateImageItems(trx: Knex, resolvePageDomPaths: PageDomPathResolver, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-entity-tables/populate-image-items.js

@@ -0,0 +1,223 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { decodeStoredBlob } from '../decode-html-blob.js';
+import { isBlobRefValue } from './is-blob-ref-value.js';
+import { resolveBlobRefs } from './resolve-blob-refs.js';
+import { resolveTextRefs } from './resolve-text-refs.js';
+import { resolveUrlOrBlobFromMaps } from './resolve-url-or-blob-from-maps.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+import { upsertTextRefs } from './upsert-text-refs.js';
+/**
+ * Page ids scanned per keyset-paginated `SELECT DISTINCT pageId` chunk.
+ * Batches the outer loop so 470 K pages worth of `images` rows are
+ * traversed in ~1 K round trips instead of one round-trip per page,
+ * without holding the entire distinct-pageId list in memory at once.
+ */
+const PAGE_ID_CHUNK_SIZE = 500;
+/**
+ * Rows sent per `INSERT INTO image_items ... VALUES (...)` statement.
+ * Each row binds 13 params, so 300 rows = 3 900 params — safely under
+ * the SQLite variable limit.
+ */
+const INSERT_CHUNK_SIZE = 300;
+/**
+ * Populates `image_items` from `images` (issue #193).
+ *
+ * The outer loop iterates **pages**, not images. Each page's images are
+ * processed as one whole unit — dom-path derivation, HTML BLOB fetch,
+ * and ref lookups all happen once per page. This design closes three
+ * separate correctness / efficiency traps that a purely image-chunked
+ * loop would expose:
+ *
+ * 1. **Ordinal-cursor bleed across chunks** — `matchImagesToDomPaths`
+ *    tracks per-outerHTML cursor state, so a page with 1 000 identical
+ *    `<img>` tags spanning two image-chunks would reset the cursor at
+ *    the boundary and duplicate dom_paths already assigned in the
+ *    previous chunk. Iterating by page keeps every page's cursor state
+ *    self-contained.
+ * 2. **Repeated `getPageHtml` + jsdom parse** — a page whose images
+ *    straddle N image-chunks would fetch and re-parse the same multi-MB
+ *    HTML snapshot N times. Iterating by page makes this exactly once
+ *    per page.
+ * 3. **Data-URI routing gap** — legacy `images.src` can be either a
+ *    plain URL (goes to `url_refs`) or a large `data:` URI (goes to
+ *    `blob_refs`, per the 512-byte threshold). The lookups partition
+ *    by that rule, so a data URI is never bound into
+ *    `resolveUrlRefs`' `WHERE url IN (?)` — which could otherwise
+ *    exceed SQLite's `SQLITE_MAX_SQL_LENGTH` for multi-KB URIs.
+ *
+ * Per outer iteration:
+ *
+ * 1. **Keyset-paginate** distinct `pageId` values from `images`. Peak
+ *    memory ≈ 500 integers.
+ * 2. **Fetch all images** for those page ids in one query, ordered by
+ *    `(pageId, id)` so JS-side grouping is a linear pass.
+ * 3. **Resolve dom paths per page** — one `getPageHtml` + one resolver
+ *    call each. `dom_path` texts are collected across the whole batch
+ *    so `upsertTextRefs` runs once per batch.
+ * 4. **Partition src / currentSrc** by the data-URI routing rule and
+ *    batch-resolve url refs, blob refs, and alt text refs across the
+ *    whole batch.
+ * 5. **Bulk INSERT** with explicit `id = images.id`.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent.
+ *
+ * HTML BLOBs are read inline via the same `trx` handle rather than
+ * through a caller-supplied `getPageHtml` callback: a callback that
+ * queried `Database.getHtmlOfPageById` (which uses the non-transactional
+ * Knex instance) would re-enter the pool from inside this writer
+ * transaction and deadlock on libsql's single writer connection. Reading
+ * `page_html_ref` + `page_html_blobs` via `trx` keeps every read on the
+ * same connection the outer trx already owns.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images (see {@link PageDomPathResolver}).
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of distinct `images.pageId` values scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * const jsdomResolver = createJsdomResolver();
+ * await knex.transaction(async (trx) => {
+ *   await populateImageItems(trx, jsdomResolver);
+ * });
+ */
+export async function populateImageItems(trx, resolvePageDomPaths, onProgress) {
+    const countRows = await trx('images').countDistinct({ n: 'pageId' });
+    const totalRow = countRows[0];
+    const total = Number(totalRow?.n ?? 0);
+    const report = createProgressReporter('image_items (distinct pages)', total, onProgress);
+    let processedPages = 0;
+    let cursorPageId = 0;
+    while (true) {
+        const pageIdRows = await trx('images')
+            .distinct('pageId')
+            .where('pageId', '>', cursorPageId)
+            .orderBy('pageId', 'asc')
+            .limit(PAGE_ID_CHUNK_SIZE);
+        if (pageIdRows.length === 0) {
+            break;
+        }
+        const pageIds = pageIdRows.map((r) => r.pageId);
+        cursorPageId = pageIds.at(-1);
+        processedPages += pageIds.length;
+        report(processedPages);
+        const rows = await trx('images')
+            .select('id', 'pageId', 'src', 'currentSrc', 'alt', 'width', 'height', 'naturalWidth', 'naturalHeight', 'isLazy', 'viewportWidth', 'sourceCode')
+            .whereIn('pageId', pageIds)
+            .orderBy([
+            { column: 'pageId', order: 'asc' },
+            { column: 'id', order: 'asc' },
+        ]);
+        if (rows.length === 0) {
+            continue;
+        }
+        const byPage = new Map();
+        for (const row of rows) {
+            const bucket = byPage.get(row.pageId);
+            if (bucket === undefined) {
+                byPage.set(row.pageId, [row]);
+            }
+            else {
+                bucket.push(row);
+            }
+        }
+        const domPaths = new Map();
+        for (const [pageId, pageImages] of byPage) {
+            const html = await readPageHtmlInTrx(trx, pageId);
+            const resolved = await resolvePageDomPaths(pageId, html, pageImages);
+            for (const [imageId, entry] of resolved) {
+                domPaths.set(imageId, entry);
+            }
+        }
+        const urls = new Set();
+        const dataUris = new Set();
+        const alts = new Set();
+        const domPathTexts = new Set();
+        for (const row of rows) {
+            for (const value of [row.src, row.currentSrc]) {
+                if (typeof value !== 'string' || value === '') {
+                    continue;
+                }
+                if (isBlobRefValue(value)) {
+                    dataUris.add(value);
+                }
+                else {
+                    urls.add(value);
+                }
+            }
+            if (typeof row.alt === 'string' && row.alt !== '') {
+                alts.add(row.alt);
+            }
+            const domPath = domPaths.get(row.id);
+            if (domPath !== undefined) {
+                domPathTexts.add(domPath.path);
+            }
+        }
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const blobIds = await resolveBlobRefs(trx, dataUris);
+        const altIds = await resolveTextRefs(trx, alts);
+        // `dom_path` strings are synthesised right here and were not
+        // among the columns the `text_refs` populate scanned
+        // (`populate-ref-tables/populate-text-refs.ts`) — insert every distinct
+        // derived path (including `unknown/<id>` fallbacks) before
+        // resolving its id. The upsert is idempotent so re-runs across
+        // partial failures do not duplicate rows.
+        const domPathIds = await upsertTextRefs(trx, domPathTexts);
+        const inserts = [];
+        for (const row of rows) {
+            const domPath = domPaths.get(row.id);
+            if (domPath === undefined) {
+                throw new Error(`populateImageItems: dom_path not resolved for image id=${row.id}`);
+            }
+            const domPathId = domPathIds.get(domPath.path);
+            if (domPathId === undefined) {
+                throw new Error(`populateImageItems: text_refs.id not resolved for dom_path=${domPath.path} — the upsertTextRefs above must have inserted it`);
+            }
+            const srcSlot = resolveUrlOrBlobFromMaps(row.src, urlIds, blobIds);
+            const currentSrcSlot = resolveUrlOrBlobFromMaps(row.currentSrc, urlIds, blobIds);
+            inserts.push({
+                id: row.id,
+                page_id: row.pageId,
+                src_url_id: srcSlot.url,
+                current_src_url_id: currentSrcSlot.url,
+                src_blob_id: srcSlot.blob,
+                current_src_blob_id: currentSrcSlot.blob,
+                alt_text_id: typeof row.alt === 'string' && row.alt !== ''
+                    ? (altIds.get(row.alt) ?? null)
+                    : null,
+                width: row.width,
+                height: row.height,
+                natural_width: row.naturalWidth,
+                natural_height: row.naturalHeight,
+                is_lazy: row.isLazy == null ? null : row.isLazy ? 1 : 0,
+                viewport_width: row.viewportWidth,
+                dom_path_text_id: domPathId,
+            });
+        }
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('image_items').insert(chunk).onConflict('id').ignore();
+        }
+    }
+}
+/**
+ * Reads the stored HTML snapshot for one legacy page id, via the same
+ * `trx` handle that the outer populate transaction owns. Mirrors
+ * `Database.getHtmlOfPageById` but keeps every read on the trx
+ * connection so the populate does not re-enter the pool from
+ * inside its own writer transaction (see {@link populateImageItems}'s
+ * doc for the deadlock this avoids).
+ * @param trx - The open migration transaction.
+ * @param pageId - `pages.id` (== `content_items.id`) to read HTML for.
+ * @returns Decoded HTML string, or `null` when no snapshot is stored.
+ */
+async function readPageHtmlInTrx(trx, pageId) {
+    const row = (await trx('page_html_ref')
+        .join('page_html_blobs', 'page_html_ref.hash', '=', 'page_html_blobs.hash')
+        .select('page_html_blobs.body as body', 'page_html_blobs.codec as codec')
+        .where('page_html_ref.page_id', pageId)
+        .first());
+    if (!row) {
+        return null;
+    }
+    return decodeStoredBlob(row.body, row.codec);
+}