@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/populate-entity-tables/collapse-anchor-rows.js

@@ -0,0 +1,87 @@
+/**
+ * Collapses consecutive `anchors`-shaped rows into `anchor_edges` rows by
+ * `(pageId, hrefId)` (issue #193).
+ *
+ * **Input contract**: `rows` MUST be sorted by `(pageId, hrefId, id)`
+ * ascending. The caller (`populate-anchor-edges.ts`) achieves this via a
+ * single keyset scan `ORDER BY pageId, hrefId, id` over `anchors`.
+ *
+ * The `id`-then-key sort matters because the dedup rule is
+ * "first instance wins": the smallest `anchors.id` for a given
+ * `(pageId, hrefId)` pair contributes its `hash` and `textContent` to
+ * the edge; every subsequent duplicate is counted but its body is
+ * discarded. A naive `min(hash) GROUP BY pageId, hrefId` would pick the
+ * lexicographically smallest hex string (uniformly distributed) rather
+ * than the earliest occurrence, so the pass has to happen in JS.
+ *
+ * The generator yields one {@link AnchorEdgeRowInProgress} per distinct
+ * pair as soon as the next pair boundary is observed. Emitting eagerly
+ * keeps peak memory bounded to O(1) — the collapser holds at most one
+ * open pair state at a time.
+ *
+ * `first_text_id` is intentionally left `null` here; the caller resolves
+ * it in a second pass after all edges are known (see
+ * `populate-anchor-edges.ts`). The `first_hash` field is set to the
+ * first row's `hash` verbatim; a first row with `hash === null` results
+ * in `first_hash = null` on the edge (rare but possible on legacy
+ * archives that failed to compute a hash).
+ * @param rows - Anchor input rows sorted by `(pageId, hrefId, id)`.
+ * @yields {AnchorEdgeRowInProgress} One entry per distinct
+ *   `(pageId, hrefId)` pair in the input.
+ * @example
+ * const edges = [...collapseAnchorRows([
+ *   { id: 1, pageId: 10, hrefId: 20, hash: 'a', textContent: 'first' },
+ *   { id: 2, pageId: 10, hrefId: 20, hash: 'b', textContent: 'second' },
+ *   { id: 3, pageId: 10, hrefId: 30, hash: 'c', textContent: 'x' },
+ * ])];
+ * // edges[0] = { page_id: 10, href_page_id: 20, count: 2, first_hash: 'a', ... }
+ * // edges[1] = { page_id: 10, href_page_id: 30, count: 1, first_hash: 'c', ... }
+ */
+export function* collapseAnchorRows(rows) {
+    let openPageId = null;
+    let openHrefId = null;
+    let openCount = 0;
+    let openFirstHash = null;
+    let openFirstTextContent = null;
+    let openLastId = null;
+    for (const row of rows) {
+        if (openPageId === row.pageId && openHrefId === row.hrefId) {
+            // Same pair — assert monotonically-increasing `id` so callers
+            // that regress the `ORDER BY id` fail loudly instead of
+            // silently producing the wrong "first instance". The count-
+            // based acceptance check does not catch this (it only sums
+            // counts, not first_hash values), so the collapser defends
+            // its own precondition.
+            if (openLastId !== null && row.id <= openLastId) {
+                throw new Error(`collapseAnchorRows: input not sorted by id within (pageId=${row.pageId}, hrefId=${row.hrefId}) — saw id=${row.id} after id=${openLastId}. Caller must ORDER BY pageId, hrefId, id ASC.`);
+            }
+            openLastId = row.id;
+            openCount += 1;
+            continue;
+        }
+        if (openPageId !== null && openHrefId !== null) {
+            yield {
+                page_id: openPageId,
+                href_page_id: openHrefId,
+                count: openCount,
+                first_hash: openFirstHash,
+                first_textContent: openFirstTextContent,
+            };
+        }
+        openPageId = row.pageId;
+        openHrefId = row.hrefId;
+        openCount = 1;
+        openFirstHash = row.hash;
+        openFirstTextContent = row.textContent;
+        openLastId = row.id;
+    }
+    if (openPageId !== null && openHrefId !== null) {
+        yield {
+            page_id: openPageId,
+            href_page_id: openHrefId,
+            count: openCount,
+            first_hash: openFirstHash,
+            first_textContent: openFirstTextContent,
+        };
+    }
+}

package/lib/archive/populate-entity-tables/derive-dom-path.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Computes the `dom_path` string for one DOM element as required by
+ * `image_items.dom_path_text_id` (issue #193).
+ *
+ * Format: slash-joined ancestor tags starting at `<html>`, each tag
+ * followed by a 1-based sibling ordinal counting **same-tag** siblings
+ * only (e.g.
+ * `html/body[1]/main[1]/section[2]/picture[1]/img[1]`). The `<html>`
+ * root has no bracketed ordinal because there is only one document
+ * root per page.
+ *
+ * The walk uses `parentElement` + `previousElementSibling` — no
+ * document mutation, no reliance on `id` / `class` — so a re-render
+ * that only reshuffles class names or ids still produces the same path.
+ * The chain of tag names alone is stable for a given DOM shape.
+ *
+ * The function operates on the generic `Element` interface from DOM lib
+ * (available in the crawler's TS lib config) so it does not force
+ * jsdom into the crawler runtime — the migration script and unit tests
+ * inject jsdom-backed elements, while a future crawler-side wrapper
+ * can pass puppeteer's DOM handles through the same code.
+ *
+ * Detached elements (`element.parentElement === null` before reaching
+ * `<html>`) return the tag chain from wherever the walk terminates —
+ * a legally detached image (rare, but
+ * technically possible during scripted mutations) yields e.g.
+ * `img[1]` alone instead of throwing.
+ * @param element - The DOM element to derive the path for. Typically an
+ *   `<img>` in the archived HTML, but the function is not tag-specific.
+ * @returns The dom_path string.
+ * @example
+ * // In an HTML snapshot `<html><body><main><img></main></body></html>`
+ * deriveDomPath(imgElement); // 'html/body[1]/main[1]/img[1]'
+ */
+export declare function deriveDomPath(element: Element): string;

package/lib/archive/populate-entity-tables/derive-dom-path.js

@@ -0,0 +1,72 @@
+/**
+ * Computes the `dom_path` string for one DOM element as required by
+ * `image_items.dom_path_text_id` (issue #193).
+ *
+ * Format: slash-joined ancestor tags starting at `<html>`, each tag
+ * followed by a 1-based sibling ordinal counting **same-tag** siblings
+ * only (e.g.
+ * `html/body[1]/main[1]/section[2]/picture[1]/img[1]`). The `<html>`
+ * root has no bracketed ordinal because there is only one document
+ * root per page.
+ *
+ * The walk uses `parentElement` + `previousElementSibling` — no
+ * document mutation, no reliance on `id` / `class` — so a re-render
+ * that only reshuffles class names or ids still produces the same path.
+ * The chain of tag names alone is stable for a given DOM shape.
+ *
+ * The function operates on the generic `Element` interface from DOM lib
+ * (available in the crawler's TS lib config) so it does not force
+ * jsdom into the crawler runtime — the migration script and unit tests
+ * inject jsdom-backed elements, while a future crawler-side wrapper
+ * can pass puppeteer's DOM handles through the same code.
+ *
+ * Detached elements (`element.parentElement === null` before reaching
+ * `<html>`) return the tag chain from wherever the walk terminates —
+ * a legally detached image (rare, but
+ * technically possible during scripted mutations) yields e.g.
+ * `img[1]` alone instead of throwing.
+ * @param element - The DOM element to derive the path for. Typically an
+ *   `<img>` in the archived HTML, but the function is not tag-specific.
+ * @returns The dom_path string.
+ * @example
+ * // In an HTML snapshot `<html><body><main><img></main></body></html>`
+ * deriveDomPath(imgElement); // 'html/body[1]/main[1]/img[1]'
+ */
+export function deriveDomPath(element) {
+    const segments = [];
+    let current = element;
+    while (current !== null) {
+        const tag = current.tagName.toLowerCase();
+        if (tag === 'html') {
+            segments.unshift('html');
+            break;
+        }
+        segments.unshift(`${tag}[${computeSiblingOrdinal(current, tag)}]`);
+        current = current.parentElement;
+    }
+    return segments.join('/');
+}
+/**
+ * Counts how many earlier siblings share the same tag name as `element`,
+ * returning `count + 1` as the 1-based ordinal.
+ *
+ * Iterating `previousElementSibling` is O(k) where k is the sibling
+ * position; for typical DOM depth and sibling counts this is negligible
+ * next to the jsdom parse cost of the enclosing page.
+ * @param element - The element whose ordinal is being computed.
+ * @param tag - The lower-cased tag name to match on. Passed in by the
+ *   caller so `element.tagName.toLowerCase()` is not recomputed inside
+ *   the loop for every sibling comparison.
+ * @returns 1-based ordinal among same-tag siblings.
+ */
+function computeSiblingOrdinal(element, tag) {
+    let count = 1;
+    let sibling = element.previousElementSibling;
+    while (sibling !== null) {
+        if (sibling.tagName.toLowerCase() === tag) {
+            count += 1;
+        }
+        sibling = sibling.previousElementSibling;
+    }
+    return count;
+}

package/lib/archive/populate-entity-tables/is-blob-ref-value.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Predicate matching the ref-tables populate's routing rule
+ * (`populate-ref-tables/populate-url-refs.ts` /
+ * `populate-ref-tables/populate-blob-refs.ts`): a `data:` URI whose
+ * length exceeds {@link DATA_URI_URL_REFS_LIMIT} lands in `blob_refs`;
+ * anything else (regular URL or short data URI) lands in `url_refs`.
+ * Shared by every entity populate that partitions a URL-shaped column
+ * into the two dictionaries (`populate-image-items.ts`,
+ * `populate-resource-items.ts`).
+ * @param value - Raw URL-shaped column value.
+ * @returns `true` when the value belongs in `blob_refs`.
+ * @example
+ * isBlobRefValue('data:image/png;base64,' + 'x'.repeat(600)); // true
+ * isBlobRefValue('https://example.com/a.png'); // false
+ */
+export declare function isBlobRefValue(value: string): boolean;

package/lib/archive/populate-entity-tables/is-blob-ref-value.js

@@ -0,0 +1,19 @@
+import { DATA_URI_URL_REFS_LIMIT } from '../populate-ref-tables/data-uri-url-refs-limit.js';
+/**
+ * Predicate matching the ref-tables populate's routing rule
+ * (`populate-ref-tables/populate-url-refs.ts` /
+ * `populate-ref-tables/populate-blob-refs.ts`): a `data:` URI whose
+ * length exceeds {@link DATA_URI_URL_REFS_LIMIT} lands in `blob_refs`;
+ * anything else (regular URL or short data URI) lands in `url_refs`.
+ * Shared by every entity populate that partitions a URL-shaped column
+ * into the two dictionaries (`populate-image-items.ts`,
+ * `populate-resource-items.ts`).
+ * @param value - Raw URL-shaped column value.
+ * @returns `true` when the value belongs in `blob_refs`.
+ * @example
+ * isBlobRefValue('data:image/png;base64,' + 'x'.repeat(600)); // true
+ * isBlobRefValue('https://example.com/a.png'); // false
+ */
+export function isBlobRefValue(value) {
+    return value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT;
+}

package/lib/archive/populate-entity-tables/match-images-to-dom-paths.d.ts

@@ -0,0 +1,66 @@
+import type { DomPathCandidate, DomPathResult } from './types.js';
+/**
+ * One image record projected onto the fields the matcher needs:
+ * `id` (used to build the returned Map) and `sourceCode` (the outerHTML
+ * string to match against document candidates). `id` may be a real
+ * `images.id` (migration) or a synthetic per-page index (crawler write
+ * path) — the matcher only requires ids to be unique within one call and
+ * ordered in crawler insertion order.
+ */
+interface ImageRowForMatching {
+    /** Unique-within-call id, in crawler insertion order. */
+    id: number;
+    /**
+     * The `<img>` element's `outerHTML` as captured by the crawler.
+     * `null` when the crawler failed to capture the source (a null blob
+     * or a puppeteer serialisation edge case).
+     */
+    sourceCode: string | null;
+}
+/**
+ * Matches image records to their DOM-path strings using a 3-case
+ * algorithm (issue #193).
+ *
+ * The three cases:
+ *
+ * 1. **Single match** — `image.sourceCode` matches exactly one `<img>`
+ *    outerHTML in the document. Assign that candidate's `dom_path`.
+ * 2. **Ordinal match** — multiple `<img>` elements share the same
+ *    outerHTML (identical `src` / `alt` / attributes on the same page).
+ *    Assign paths in `id` order, matching the crawler's insertion order
+ *    which is expected to correspond to DOM order.
+ * 3. **Unknown** — `sourceCode` is null OR no matching `<img>` exists
+ *    in the document (crawler-side rewriting drift). Fall back to the
+ *    synthetic `unknown/<id>` marker so the `dom_path_text_id NOT NULL`
+ *    constraint is still satisfied.
+ *
+ * The function is pure — no DB, no DOM. Callers produce the candidate
+ * list from whatever DOM they have: jsdom over an archived HTML snapshot
+ * (migration script) or an in-browser walk over the live page (crawler
+ * write path). See {@link ./types.ts}'s `DomPathCandidate`.
+ *
+ * `candidatesInDocumentOrder` MUST already be in document order — the
+ * matcher does not sort. `document.querySelectorAll('img')` satisfies
+ * this contract; ordering is the caller's responsibility.
+ *
+ * `images` MUST already be sorted by `id` ascending for the ordinal-
+ * match case to reproduce the crawler's insertion order deterministically.
+ * A minor deviation from this ordering only affects the ordinal-match
+ * branch — single-match and unknown branches are order-independent.
+ * @param images - The page's image records in `id` order.
+ * @param candidatesInDocumentOrder - `<img>` outerHTML + dom_path pairs
+ *   from the document, in document order.
+ * @returns Map keyed by `images[].id`; every input row gets one entry.
+ * @example
+ * const jsdom = new JSDOM(html);
+ * const candidates = [...jsdom.window.document.querySelectorAll('img')].map(
+ *   (img) => ({ outerHTML: img.outerHTML, path: deriveDomPath(img) }),
+ * );
+ * const result = matchImagesToDomPaths(
+ *   [{ id: 1, sourceCode: '<img src="a.png">' }],
+ *   candidates,
+ * );
+ * result.get(1); // { path: 'html/body[1]/img[1]', case: 'single-match' }
+ */
+export declare function matchImagesToDomPaths(images: readonly ImageRowForMatching[], candidatesInDocumentOrder: readonly DomPathCandidate[]): ReadonlyMap<number, DomPathResult>;
+export {};

package/lib/archive/populate-entity-tables/match-images-to-dom-paths.js

@@ -0,0 +1,96 @@
+/**
+ * Matches image records to their DOM-path strings using a 3-case
+ * algorithm (issue #193).
+ *
+ * The three cases:
+ *
+ * 1. **Single match** — `image.sourceCode` matches exactly one `<img>`
+ *    outerHTML in the document. Assign that candidate's `dom_path`.
+ * 2. **Ordinal match** — multiple `<img>` elements share the same
+ *    outerHTML (identical `src` / `alt` / attributes on the same page).
+ *    Assign paths in `id` order, matching the crawler's insertion order
+ *    which is expected to correspond to DOM order.
+ * 3. **Unknown** — `sourceCode` is null OR no matching `<img>` exists
+ *    in the document (crawler-side rewriting drift). Fall back to the
+ *    synthetic `unknown/<id>` marker so the `dom_path_text_id NOT NULL`
+ *    constraint is still satisfied.
+ *
+ * The function is pure — no DB, no DOM. Callers produce the candidate
+ * list from whatever DOM they have: jsdom over an archived HTML snapshot
+ * (migration script) or an in-browser walk over the live page (crawler
+ * write path). See {@link ./types.ts}'s `DomPathCandidate`.
+ *
+ * `candidatesInDocumentOrder` MUST already be in document order — the
+ * matcher does not sort. `document.querySelectorAll('img')` satisfies
+ * this contract; ordering is the caller's responsibility.
+ *
+ * `images` MUST already be sorted by `id` ascending for the ordinal-
+ * match case to reproduce the crawler's insertion order deterministically.
+ * A minor deviation from this ordering only affects the ordinal-match
+ * branch — single-match and unknown branches are order-independent.
+ * @param images - The page's image records in `id` order.
+ * @param candidatesInDocumentOrder - `<img>` outerHTML + dom_path pairs
+ *   from the document, in document order.
+ * @returns Map keyed by `images[].id`; every input row gets one entry.
+ * @example
+ * const jsdom = new JSDOM(html);
+ * const candidates = [...jsdom.window.document.querySelectorAll('img')].map(
+ *   (img) => ({ outerHTML: img.outerHTML, path: deriveDomPath(img) }),
+ * );
+ * const result = matchImagesToDomPaths(
+ *   [{ id: 1, sourceCode: '<img src="a.png">' }],
+ *   candidates,
+ * );
+ * result.get(1); // { path: 'html/body[1]/img[1]', case: 'single-match' }
+ */
+export function matchImagesToDomPaths(images, candidatesInDocumentOrder) {
+    const byOuterHtml = new Map();
+    for (const candidate of candidatesInDocumentOrder) {
+        const bucket = byOuterHtml.get(candidate.outerHTML);
+        if (bucket === undefined) {
+            byOuterHtml.set(candidate.outerHTML, [candidate]);
+        }
+        else {
+            bucket.push(candidate);
+        }
+    }
+    const cursorByOuterHtml = new Map();
+    const result = new Map();
+    for (const image of images) {
+        if (image.sourceCode === null || image.sourceCode === '') {
+            result.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' });
+            continue;
+        }
+        const candidates = byOuterHtml.get(image.sourceCode);
+        if (candidates === undefined || candidates.length === 0) {
+            result.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' });
+            continue;
+        }
+        if (candidates.length === 1) {
+            result.set(image.id, {
+                path: candidates[0].path,
+                case: 'single-match',
+            });
+            continue;
+        }
+        const cursor = cursorByOuterHtml.get(image.sourceCode) ?? 0;
+        cursorByOuterHtml.set(image.sourceCode, cursor + 1);
+        const chosen = candidates[cursor];
+        if (chosen === undefined) {
+            // More image records share this outerHTML than the document
+            // has matching `<img>` elements. Reusing the last candidate
+            // (`candidates.at(-1)`) would silently map every overflow row
+            // to the same element, producing multiple `image_items` rows
+            // with the same `dom_path_text_id`. Falling back to
+            // `unknown/<id>` keeps overflow rows individually
+            // distinguishable in the archive.
+            result.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' });
+            continue;
+        }
+        result.set(image.id, {
+            path: chosen.path,
+            case: 'ordinal-match',
+        });
+    }
+    return result;
+}

package/lib/archive/populate-entity-tables/populate-anchor-edges.d.ts

@@ -0,0 +1,33 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `anchor_edges` from `anchors` (issue #193).
+ *
+ * The algorithm is a **single keyset-paginated scan** over `anchors`
+ * ordered by `(pageId, hrefId, id)`. Each chunk feeds
+ * {@link ./collapse-anchor-rows.ts}, which yields one edge per distinct
+ * `(pageId, hrefId)` pair as soon as the pair boundary is observed.
+ *
+ * A boundary can straddle two chunks (the last row of chunk N shares a
+ * pair with the first row of chunk N+1), so the outer loop keeps an
+ * "open edge" state that flushes only when the *next* pair actually
+ * starts. On end-of-stream the last open edge is emitted.
+ *
+ * `first_text_id` is left unresolved during the streaming pass — the
+ * text is buffered on each pending edge and resolved in bulk by
+ * {@link ./resolve-text-refs.ts} at INSERT time. Every INSERT batch
+ * therefore issues one text-refs lookup + one INSERT, keeping DB
+ * round-trips per edge low.
+ *
+ * `INSERT OR IGNORE` on the `(page_id, href_page_id)` UNIQUE composite
+ * makes the step idempotent: a re-run after partial failure re-emits the
+ * same edges but the writes no-op.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `anchors` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateAnchorEdges(trx);
+ * });
+ */
+export declare function populateAnchorEdges(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-entity-tables/populate-anchor-edges.js

@@ -0,0 +1,153 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { collapseAnchorRows } from './collapse-anchor-rows.js';
+import { resolveTextRefs } from './resolve-text-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against `anchors`.
+ * `anchors` is the largest table in the archive (≈ 13 M rows on the
+ * reference archive) so the chunk size trades off memory against
+ * round-trip overhead. 5 000 rows per SELECT keeps peak chunk memory
+ * ≈ 5 MB (per-row ≈ 1 KB with URL / textContent stored elsewhere) and
+ * amortises the per-query round-trip cost across many collapses.
+ */
+const READ_CHUNK_SIZE = 5000;
+/**
+ * Edges buffered before a bulk `.insert(rows).onConflict(...).ignore()`.
+ * The real constraint here is NOT the SQLite bound-parameter limit —
+ * knex's sqlite3-family dialect compiles any multi-row `.insert()` (with
+ * or without `onConflict`) into `INSERT INTO ... SELECT ... UNION ALL
+ * SELECT ...`, which is capped by SQLite's `SQLITE_LIMIT_COMPOUND_SELECT`
+ * (default 500). A chunk size above 500 fails with "too many terms in
+ * compound SELECT" — confirmed against a real archive whose anchor count
+ * routinely produces 500+ distinct `(page_id, href_page_id)` pairs per
+ * page. Kept at exactly 500 (not lower) to match the established
+ * convention already used by `upsert-one-header-set.ts` and the
+ * ref-tables populate steps.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Populates `anchor_edges` from `anchors` (issue #193).
+ *
+ * The algorithm is a **single keyset-paginated scan** over `anchors`
+ * ordered by `(pageId, hrefId, id)`. Each chunk feeds
+ * {@link ./collapse-anchor-rows.ts}, which yields one edge per distinct
+ * `(pageId, hrefId)` pair as soon as the pair boundary is observed.
+ *
+ * A boundary can straddle two chunks (the last row of chunk N shares a
+ * pair with the first row of chunk N+1), so the outer loop keeps an
+ * "open edge" state that flushes only when the *next* pair actually
+ * starts. On end-of-stream the last open edge is emitted.
+ *
+ * `first_text_id` is left unresolved during the streaming pass — the
+ * text is buffered on each pending edge and resolved in bulk by
+ * {@link ./resolve-text-refs.ts} at INSERT time. Every INSERT batch
+ * therefore issues one text-refs lookup + one INSERT, keeping DB
+ * round-trips per edge low.
+ *
+ * `INSERT OR IGNORE` on the `(page_id, href_page_id)` UNIQUE composite
+ * makes the step idempotent: a re-run after partial failure re-emits the
+ * same edges but the writes no-op.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `anchors` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateAnchorEdges(trx);
+ * });
+ */
+export async function populateAnchorEdges(trx, onProgress) {
+    let cursorPageId = 0;
+    let cursorHrefId = 0;
+    let cursorId = 0;
+    let carryOver = null;
+    const pending = [];
+    const countRows = await trx('anchors').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('anchor_edges (anchors)', total, onProgress);
+    let processed = 0;
+    while (true) {
+        const rows = await trx('anchors')
+            .select('id', 'pageId', 'hrefId', 'hash', 'textContent')
+            .where(function () {
+            this.where('pageId', '>', cursorPageId)
+                .orWhere(function () {
+                this.where('pageId', cursorPageId).andWhere('hrefId', '>', cursorHrefId);
+            })
+                .orWhere(function () {
+                this.where('pageId', cursorPageId)
+                    .andWhere('hrefId', cursorHrefId)
+                    .andWhere('id', '>', cursorId);
+            });
+        })
+            .orderBy([
+            { column: 'pageId', order: 'asc' },
+            { column: 'hrefId', order: 'asc' },
+            { column: 'id', order: 'asc' },
+        ])
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        const lastRow = rows.at(-1);
+        cursorPageId = lastRow.pageId;
+        cursorHrefId = lastRow.hrefId;
+        cursorId = lastRow.id;
+        processed += rows.length;
+        report(processed);
+        const chunkEdges = [...collapseAnchorRows(rows)];
+        for (const edge of chunkEdges) {
+            if (carryOver !== null &&
+                carryOver.page_id === edge.page_id &&
+                carryOver.href_page_id === edge.href_page_id) {
+                carryOver.count += edge.count;
+                continue;
+            }
+            if (carryOver !== null) {
+                pending.push(carryOver);
+            }
+            carryOver = edge;
+            if (pending.length >= INSERT_CHUNK_SIZE) {
+                await flush(trx, pending);
+            }
+        }
+    }
+    if (carryOver !== null) {
+        pending.push(carryOver);
+    }
+    if (pending.length > 0) {
+        await flush(trx, pending);
+    }
+}
+/**
+ * Resolves `first_text_id` for every buffered edge in `pending`, then
+ * bulk-inserts the batch into `anchor_edges` and clears the buffer.
+ *
+ * Extracted from the main loop so the "resolve + insert" pair happens in
+ * exactly one place — the streaming loop, the carry-over flush at
+ * end-of-stream, and any future re-order boundary all land here.
+ * @param trx - Knex instance or transaction.
+ * @param pending - Buffered edges awaiting `first_text_id` resolution
+ *   and INSERT; mutated in place (cleared on return).
+ */
+async function flush(trx, pending) {
+    const texts = new Set();
+    for (const edge of pending) {
+        if (edge.first_textContent != null && edge.first_textContent !== '') {
+            texts.add(edge.first_textContent);
+        }
+    }
+    const textIds = await resolveTextRefs(trx, texts);
+    const inserts = pending.map((edge) => ({
+        page_id: edge.page_id,
+        href_page_id: edge.href_page_id,
+        count: edge.count,
+        first_hash: edge.first_hash,
+        first_text_id: edge.first_textContent != null && edge.first_textContent !== ''
+            ? (textIds.get(edge.first_textContent) ?? null)
+            : null,
+    }));
+    await trx('anchor_edges')
+        .insert(inserts)
+        .onConflict(['page_id', 'href_page_id'])
+        .ignore();
+    pending.length = 0;
+}

package/lib/archive/populate-entity-tables/populate-content-items.d.ts

@@ -0,0 +1,40 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `content_items` from `pages` (issue #193).
+ *
+ * Strategy per chunk:
+ *
+ * 1. **Keyset-paginate** `pages` in id order so partial-failure re-runs
+ *    resume from the last committed row without reading the full table.
+ *    Same pattern as {@link ../populate-ref-tables/populate-url-refs.ts}.
+ * 2. **Batch-resolve** `url_refs.id`, `content_type_refs.id`, and
+ *    `header_sets.id` for every distinct value in the chunk — one round
+ *    trip per ref table instead of N round trips per row.
+ * 3. **Bulk INSERT** with explicit `id = pages.id`. Reusing the legacy
+ *    PK preserves every FK reference in `page_errors` / `page_tags` /
+ *    `page_jsonld` / `page_html_ref` without a per-row UPDATE.
+ *
+ * `redirect_dest_id` is copied verbatim from `redirectDestId` — the FK
+ * self-reference is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}) so a redirect source
+ * inserted before its destination is validated only at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent — a
+ * re-run after a partial failure only inserts the rows that are still
+ * missing.
+ *
+ * `content_type_refs` is preloaded once at the start because its
+ * cardinality is small (see {@link ./resolve-content-type-refs.ts}); the
+ * URL and header-set resolvers are called per chunk because their
+ * per-archive cardinality can be very large (one row per distinct URL /
+ * response JSON).
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ * });
+ */
+export declare function populateContentItems(trx: Knex, onProgress?: ProgressCallback): Promise<void>;