@nitpicker/archive 0.0.0 → 0.25.0

package/lib/populate-entity-tables/populate-anchor-edges.js

@@ -0,0 +1,153 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { collapseAnchorRows } from './collapse-anchor-rows.js';
+import { resolveTextRefs } from './resolve-text-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against `anchors`.
+ * `anchors` is the largest table in the archive (≈ 13 M rows on the
+ * reference archive) so the chunk size trades off memory against
+ * round-trip overhead. 5 000 rows per SELECT keeps peak chunk memory
+ * ≈ 5 MB (per-row ≈ 1 KB with URL / textContent stored elsewhere) and
+ * amortises the per-query round-trip cost across many collapses.
+ */
+const READ_CHUNK_SIZE = 5000;
+/**
+ * Edges buffered before a bulk `.insert(rows).onConflict(...).ignore()`.
+ * The real constraint here is NOT the SQLite bound-parameter limit —
+ * knex's sqlite3-family dialect compiles any multi-row `.insert()` (with
+ * or without `onConflict`) into `INSERT INTO ... SELECT ... UNION ALL
+ * SELECT ...`, which is capped by SQLite's `SQLITE_LIMIT_COMPOUND_SELECT`
+ * (default 500). A chunk size above 500 fails with "too many terms in
+ * compound SELECT" — confirmed against a real archive whose anchor count
+ * routinely produces 500+ distinct `(page_id, href_page_id)` pairs per
+ * page. Kept at exactly 500 (not lower) to match the established
+ * convention already used by `upsert-one-header-set.ts` and the
+ * ref-tables populate steps.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Populates `anchor_edges` from `anchors` (issue #193).
+ *
+ * The algorithm is a **single keyset-paginated scan** over `anchors`
+ * ordered by `(pageId, hrefId, id)`. Each chunk feeds
+ * {@link ./collapse-anchor-rows.ts}, which yields one edge per distinct
+ * `(pageId, hrefId)` pair as soon as the pair boundary is observed.
+ *
+ * A boundary can straddle two chunks (the last row of chunk N shares a
+ * pair with the first row of chunk N+1), so the outer loop keeps an
+ * "open edge" state that flushes only when the *next* pair actually
+ * starts. On end-of-stream the last open edge is emitted.
+ *
+ * `first_text_id` is left unresolved during the streaming pass — the
+ * text is buffered on each pending edge and resolved in bulk by
+ * {@link ./resolve-text-refs.ts} at INSERT time. Every INSERT batch
+ * therefore issues one text-refs lookup + one INSERT, keeping DB
+ * round-trips per edge low.
+ *
+ * `INSERT OR IGNORE` on the `(page_id, href_page_id)` UNIQUE composite
+ * makes the step idempotent: a re-run after partial failure re-emits the
+ * same edges but the writes no-op.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `anchors` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateAnchorEdges(trx);
+ * });
+ */
+export async function populateAnchorEdges(trx, onProgress) {
+    let cursorPageId = 0;
+    let cursorHrefId = 0;
+    let cursorId = 0;
+    let carryOver = null;
+    const pending = [];
+    const countRows = await trx('anchors').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('anchor_edges (anchors)', total, onProgress);
+    let processed = 0;
+    while (true) {
+        const rows = await trx('anchors')
+            .select('id', 'pageId', 'hrefId', 'hash', 'textContent')
+            .where(function () {
+            this.where('pageId', '>', cursorPageId)
+                .orWhere(function () {
+                this.where('pageId', cursorPageId).andWhere('hrefId', '>', cursorHrefId);
+            })
+                .orWhere(function () {
+                this.where('pageId', cursorPageId)
+                    .andWhere('hrefId', cursorHrefId)
+                    .andWhere('id', '>', cursorId);
+            });
+        })
+            .orderBy([
+            { column: 'pageId', order: 'asc' },
+            { column: 'hrefId', order: 'asc' },
+            { column: 'id', order: 'asc' },
+        ])
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        const lastRow = rows.at(-1);
+        cursorPageId = lastRow.pageId;
+        cursorHrefId = lastRow.hrefId;
+        cursorId = lastRow.id;
+        processed += rows.length;
+        report(processed);
+        const chunkEdges = [...collapseAnchorRows(rows)];
+        for (const edge of chunkEdges) {
+            if (carryOver !== null &&
+                carryOver.page_id === edge.page_id &&
+                carryOver.href_page_id === edge.href_page_id) {
+                carryOver.count += edge.count;
+                continue;
+            }
+            if (carryOver !== null) {
+                pending.push(carryOver);
+            }
+            carryOver = edge;
+            if (pending.length >= INSERT_CHUNK_SIZE) {
+                await flush(trx, pending);
+            }
+        }
+    }
+    if (carryOver !== null) {
+        pending.push(carryOver);
+    }
+    if (pending.length > 0) {
+        await flush(trx, pending);
+    }
+}
+/**
+ * Resolves `first_text_id` for every buffered edge in `pending`, then
+ * bulk-inserts the batch into `anchor_edges` and clears the buffer.
+ *
+ * Extracted from the main loop so the "resolve + insert" pair happens in
+ * exactly one place — the streaming loop, the carry-over flush at
+ * end-of-stream, and any future re-order boundary all land here.
+ * @param trx - Knex instance or transaction.
+ * @param pending - Buffered edges awaiting `first_text_id` resolution
+ *   and INSERT; mutated in place (cleared on return).
+ */
+async function flush(trx, pending) {
+    const texts = new Set();
+    for (const edge of pending) {
+        if (edge.first_textContent != null && edge.first_textContent !== '') {
+            texts.add(edge.first_textContent);
+        }
+    }
+    const textIds = await resolveTextRefs(trx, texts);
+    const inserts = pending.map((edge) => ({
+        page_id: edge.page_id,
+        href_page_id: edge.href_page_id,
+        count: edge.count,
+        first_hash: edge.first_hash,
+        first_text_id: edge.first_textContent != null && edge.first_textContent !== ''
+            ? (textIds.get(edge.first_textContent) ?? null)
+            : null,
+    }));
+    await trx('anchor_edges')
+        .insert(inserts)
+        .onConflict(['page_id', 'href_page_id'])
+        .ignore();
+    pending.length = 0;
+}

package/lib/populate-entity-tables/populate-content-items.d.ts

@@ -0,0 +1,40 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `content_items` from `pages` (issue #193).
+ *
+ * Strategy per chunk:
+ *
+ * 1. **Keyset-paginate** `pages` in id order so partial-failure re-runs
+ *    resume from the last committed row without reading the full table.
+ *    Same pattern as {@link ../populate-ref-tables/populate-url-refs.ts}.
+ * 2. **Batch-resolve** `url_refs.id`, `content_type_refs.id`, and
+ *    `header_sets.id` for every distinct value in the chunk — one round
+ *    trip per ref table instead of N round trips per row.
+ * 3. **Bulk INSERT** with explicit `id = pages.id`. Reusing the legacy
+ *    PK preserves every FK reference in `page_errors` / `page_tags` /
+ *    `page_jsonld` / `page_html_ref` without a per-row UPDATE.
+ *
+ * `redirect_dest_id` is copied verbatim from `redirectDestId` — the FK
+ * self-reference is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}) so a redirect source
+ * inserted before its destination is validated only at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent — a
+ * re-run after a partial failure only inserts the rows that are still
+ * missing.
+ *
+ * `content_type_refs` is preloaded once at the start because its
+ * cardinality is small (see {@link ./resolve-content-type-refs.ts}); the
+ * URL and header-set resolvers are called per chunk because their
+ * per-archive cardinality can be very large (one row per distinct URL /
+ * response JSON).
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ * });
+ */
+export declare function populateContentItems(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-entity-tables/populate-content-items.js

@@ -0,0 +1,141 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { loadContentTypeRefs } from './resolve-content-type-refs.js';
+import { resolveHeaderSets } from './resolve-header-sets.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against `pages`. The
+ * migration reads every page column that maps to `content_items` in one
+ * SELECT per chunk, so scaling this too high would blow up the result-set
+ * size for long meta text columns. 500 keeps peak memory bounded at
+ * ≈ 30 MB for a chunk of `pages` with populated meta and header sets.
+ */
+const READ_CHUNK_SIZE = 500;
+/**
+ * Rows sent per `INSERT INTO content_items ... VALUES (...)` statement.
+ * Each row binds 16 params (id + 15 core columns), so 300 rows =
+ * 4 800 params — safely under SQLite's default variable limit of 32 766.
+ */
+const INSERT_CHUNK_SIZE = 300;
+/**
+ * Populates `content_items` from `pages` (issue #193).
+ *
+ * Strategy per chunk:
+ *
+ * 1. **Keyset-paginate** `pages` in id order so partial-failure re-runs
+ *    resume from the last committed row without reading the full table.
+ *    Same pattern as {@link ../populate-ref-tables/populate-url-refs.ts}.
+ * 2. **Batch-resolve** `url_refs.id`, `content_type_refs.id`, and
+ *    `header_sets.id` for every distinct value in the chunk — one round
+ *    trip per ref table instead of N round trips per row.
+ * 3. **Bulk INSERT** with explicit `id = pages.id`. Reusing the legacy
+ *    PK preserves every FK reference in `page_errors` / `page_tags` /
+ *    `page_jsonld` / `page_html_ref` without a per-row UPDATE.
+ *
+ * `redirect_dest_id` is copied verbatim from `redirectDestId` — the FK
+ * self-reference is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}) so a redirect source
+ * inserted before its destination is validated only at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent — a
+ * re-run after a partial failure only inserts the rows that are still
+ * missing.
+ *
+ * `content_type_refs` is preloaded once at the start because its
+ * cardinality is small (see {@link ./resolve-content-type-refs.ts}); the
+ * URL and header-set resolvers are called per chunk because their
+ * per-archive cardinality can be very large (one row per distinct URL /
+ * response JSON).
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ * });
+ */
+export async function populateContentItems(trx, onProgress) {
+    const contentTypeIds = await loadContentTypeRefs(trx);
+    const countRows = await trx('pages').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('content_items (pages)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('pages')
+            .select('id', 'url', 'redirectDestId', 'scraped', 'isTarget', 'isExternal', 'status', 'statusText', 'contentType', 'contentLength', 'responseHeaders', 'firstCrawledAt', 'lastCrawledAt', 'order', 'isSkipped', 'skipReason', 'source')
+            .where('id', '>', cursor)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        const urls = new Set();
+        const headerJsonStrings = new Set();
+        for (const row of rows) {
+            urls.add(row.url);
+            if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') {
+                headerJsonStrings.add(row.responseHeaders);
+            }
+        }
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const headerSetIds = await resolveHeaderSets(trx, headerJsonStrings);
+        const inserts = rows.map((row) => {
+            const urlId = urlIds.get(row.url) ?? null;
+            if (urlId === null) {
+                throw new Error(`populateContentItems: url_refs.id not resolved for page id=${row.id} url=${row.url} — populateUrlRefs must run first`);
+            }
+            let contentTypeId = null;
+            if (row.contentType != null && row.contentType !== '') {
+                const resolved = contentTypeIds.get(row.contentType);
+                if (resolved === undefined) {
+                    throw new Error(`populateContentItems: content_type_refs.id not resolved for page id=${row.id} contentType=${row.contentType} — populateContentTypeRefs must run first`);
+                }
+                contentTypeId = resolved;
+            }
+            let headerSetId = null;
+            if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') {
+                const resolved = headerSetIds.get(row.responseHeaders);
+                if (resolved === undefined) {
+                    // `resolveHeaderSets` skips sentinel values ('{}' / 'null')
+                    // AND payloads that `decomposeHeaderSet` cannot parse into
+                    // any entries — in both cases the source row genuinely had
+                    // no header set to point at, so `header_set_id = null` is
+                    // the correct final state. `resolveHeaderSets` already ran
+                    // the raw_hash fallback, so this branch only fires when the
+                    // row has no persistable header set (parse error, or every
+                    // entry stripped for non-string values).
+                    headerSetId = null;
+                }
+                else {
+                    headerSetId = resolved;
+                }
+            }
+            return {
+                id: row.id,
+                url_id: urlId,
+                is_external: row.isExternal == null ? 0 : row.isExternal ? 1 : 0,
+                scraped: row.scraped ? 1 : 0,
+                is_target: row.isTarget ? 1 : 0,
+                status: row.status ?? null,
+                status_text: row.statusText ?? null,
+                content_type_id: contentTypeId,
+                content_length: row.contentLength ?? null,
+                header_set_id: headerSetId,
+                redirect_dest_id: row.redirectDestId ?? null,
+                source: row.source,
+                first_crawled_at: row.firstCrawledAt ?? null,
+                last_crawled_at: row.lastCrawledAt ?? null,
+                crawl_order: row.order ?? null,
+                is_skipped: row.isSkipped == null ? null : row.isSkipped ? 1 : 0,
+                skip_reason: row.skipReason ?? null,
+            };
+        });
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('content_items').insert(chunk).onConflict('id').ignore();
+        }
+    }
+}

package/lib/populate-entity-tables/populate-entities.d.ts

@@ -0,0 +1,81 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { PageDomPathResolver } from './populate-image-items.js';
+import type { Knex } from 'knex';
+/**
+ * Runs the six 0.13 entity/edge populates (issue #193) in dependency
+ * order against an already-connected archive.
+ *
+ * Order rationale:
+ *
+ * 1. **`content_items`** — every downstream populate's FKs reference
+ *    `content_items(id)`. Must land first so the FKs are valid at
+ *    COMMIT time. `redirect_dest_id` is `DEFERRABLE INITIALLY DEFERRED`
+ *    (see {@link ../create-entity-tables.ts}) so a redirect
+ *    source inserted before its destination is validated only at
+ *    COMMIT — the insert order within this populate is not
+ *    load-bearing.
+ * 2. **`page_meta`** — FK to `content_items(id)`.
+ * 3. **`resource_items`** — required before `resource_ref_edges`.
+ * 4. **`anchor_edges`** — FKs to `content_items(id)` on both
+ *    sides.
+ * 5. **`resource_ref_edges`** — FKs to `resource_items(id)` and
+ *    `content_items(id)`.
+ * 6. **`image_items`** — FK to `content_items(id)`; the
+ *    dom-path text_refs upsert is scoped to this populate so re-runs are
+ *    self-contained.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE` on
+ * its natural key or PK. On a **re-crawl** (`crawl --append` /
+ * `--retry-failed` / `--inventory` all UPDATE existing `pages` /
+ * `resources` rows in place, and `#insertPage` deletes + re-inserts
+ * `anchors` / `images` per page), running populate again against the
+ * previously-populated archive would leave `content_items` / `page_meta`
+ * / `resource_items` at their FIRST-crawl values (source-priority
+ * upgrades, refreshed status, changed metadata never propagate) and
+ * `anchor_edges` / `image_items` / `resource_ref_edges` would keep stale
+ * rows keyed by the now-deleted legacy ids. To keep the reader-side
+ * view of the archive faithful to `pages` / `resources` on re-crawl,
+ * every entity + edge table is TRUNCATEd (in child-first order so no FK
+ * check trips) at the top of this function before the six sub-populates
+ * re-insert from the current legacy state. Ref tables (`url_refs`,
+ * `text_refs`, `content_type_refs`, `header_*`) are NOT truncated —
+ * they are content-addressable and additive; re-inserting the same
+ * value hits `INSERT OR IGNORE` and is a no-op.
+ *
+ * The whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level (a single WAL transaction
+ * with `.bak` rollback on failure);
+ * this function does not open its own transaction so the
+ * caller controls the boundary — see {@link
+ * ../populate-ref-tables/populate-refs.ts} for the same convention.
+ *
+ * **`PRAGMA foreign_keys = ON` is REQUIRED** on the underlying
+ * connection before this function is called. `content_items.redirect_dest_id`
+ * is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}); without foreign-key
+ * enforcement libsql commits `content_items` rows whose `redirect_dest_id`
+ * points at a non-existent page and the deferrable-FK-enforcement-at-COMMIT
+ * invariant is silently broken. The
+ * migration script sets the pragma explicitly; any other caller must do
+ * the same.
+ * HTML BLOB reads happen inline inside {@link populateImageItems} against
+ * the same `trx` — no `getPageHtml` callback here — because a callback
+ * that routed through `Database.getHtmlOfPageById` would re-enter the
+ * connection pool from inside this writer transaction and deadlock on
+ * libsql's single writer connection. See `./populate-image-items.ts` for
+ * the detailed rationale.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images. Injected rather than hard-coded so
+ *   `@nitpicker/archive` does not become a jsdom consumer at runtime.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateEntityTables(trx, jsdomResolver);
+ * });
+ * await archive.write();
+ */
+export declare function populateEntityTables(trx: Knex, resolvePageDomPaths: PageDomPathResolver, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-entity-tables/populate-entities.js

@@ -0,0 +1,110 @@
+import { populateAnchorEdges } from './populate-anchor-edges.js';
+import { populateContentItems } from './populate-content-items.js';
+import { populateImageItems } from './populate-image-items.js';
+import { populatePageMeta } from './populate-page-meta.js';
+import { populateResourceItems } from './populate-resource-items.js';
+import { populateResourceRefEdges } from './populate-resource-ref-edges.js';
+/**
+ * Runs the six 0.13 entity/edge populates (issue #193) in dependency
+ * order against an already-connected archive.
+ *
+ * Order rationale:
+ *
+ * 1. **`content_items`** — every downstream populate's FKs reference
+ *    `content_items(id)`. Must land first so the FKs are valid at
+ *    COMMIT time. `redirect_dest_id` is `DEFERRABLE INITIALLY DEFERRED`
+ *    (see {@link ../create-entity-tables.ts}) so a redirect
+ *    source inserted before its destination is validated only at
+ *    COMMIT — the insert order within this populate is not
+ *    load-bearing.
+ * 2. **`page_meta`** — FK to `content_items(id)`.
+ * 3. **`resource_items`** — required before `resource_ref_edges`.
+ * 4. **`anchor_edges`** — FKs to `content_items(id)` on both
+ *    sides.
+ * 5. **`resource_ref_edges`** — FKs to `resource_items(id)` and
+ *    `content_items(id)`.
+ * 6. **`image_items`** — FK to `content_items(id)`; the
+ *    dom-path text_refs upsert is scoped to this populate so re-runs are
+ *    self-contained.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE` on
+ * its natural key or PK. On a **re-crawl** (`crawl --append` /
+ * `--retry-failed` / `--inventory` all UPDATE existing `pages` /
+ * `resources` rows in place, and `#insertPage` deletes + re-inserts
+ * `anchors` / `images` per page), running populate again against the
+ * previously-populated archive would leave `content_items` / `page_meta`
+ * / `resource_items` at their FIRST-crawl values (source-priority
+ * upgrades, refreshed status, changed metadata never propagate) and
+ * `anchor_edges` / `image_items` / `resource_ref_edges` would keep stale
+ * rows keyed by the now-deleted legacy ids. To keep the reader-side
+ * view of the archive faithful to `pages` / `resources` on re-crawl,
+ * every entity + edge table is TRUNCATEd (in child-first order so no FK
+ * check trips) at the top of this function before the six sub-populates
+ * re-insert from the current legacy state. Ref tables (`url_refs`,
+ * `text_refs`, `content_type_refs`, `header_*`) are NOT truncated —
+ * they are content-addressable and additive; re-inserting the same
+ * value hits `INSERT OR IGNORE` and is a no-op.
+ *
+ * The whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level (a single WAL transaction
+ * with `.bak` rollback on failure);
+ * this function does not open its own transaction so the
+ * caller controls the boundary — see {@link
+ * ../populate-ref-tables/populate-refs.ts} for the same convention.
+ *
+ * **`PRAGMA foreign_keys = ON` is REQUIRED** on the underlying
+ * connection before this function is called. `content_items.redirect_dest_id`
+ * is `DEFERRABLE INITIALLY DEFERRED` (see
+ * {@link ../create-entity-tables.ts}); without foreign-key
+ * enforcement libsql commits `content_items` rows whose `redirect_dest_id`
+ * points at a non-existent page and the deferrable-FK-enforcement-at-COMMIT
+ * invariant is silently broken. The
+ * migration script sets the pragma explicitly; any other caller must do
+ * the same.
+ * HTML BLOB reads happen inline inside {@link populateImageItems} against
+ * the same `trx` — no `getPageHtml` callback here — because a callback
+ * that routed through `Database.getHtmlOfPageById` would re-enter the
+ * connection pool from inside this writer transaction and deadlock on
+ * libsql's single writer connection. See `./populate-image-items.ts` for
+ * the detailed rationale.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images. Injected rather than hard-coded so
+ *   `@nitpicker/archive` does not become a jsdom consumer at runtime.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateEntityTables(trx, jsdomResolver);
+ * });
+ * await archive.write();
+ */
+export async function populateEntityTables(trx, resolvePageDomPaths, onProgress) {
+    // Truncate child-first so no outgoing FK check ever sees a broken
+    // reference mid-delete. Order is: leaf edge/entity tables that have
+    // no incoming FKs (`image_items`, `anchor_edges`, `resource_ref_edges`),
+    // then the parent entities they referenced (`page_meta` and
+    // `resource_items`; `page_meta` references `content_items` so it
+    // must be dropped before `content_items`; `resource_items` was
+    // pointed at by `resource_ref_edges` which is now empty), and
+    // finally `content_items` itself — the only remaining incoming
+    // reference is its own DEFERRABLE INITIALLY DEFERRED
+    // `redirect_dest_id`, which is validated at COMMIT so a mid-trx
+    // wipe-and-refill is legal. No external table has an enforced FK
+    // into `content_items` (viewer read-model tables
+    // hold logical-only pointers), so the truncation stops here.
+    await trx('image_items').delete();
+    await trx('anchor_edges').delete();
+    await trx('resource_ref_edges').delete();
+    await trx('page_meta').delete();
+    await trx('resource_items').delete();
+    await trx('content_items').delete();
+    await populateContentItems(trx, onProgress);
+    await populatePageMeta(trx, onProgress);
+    await populateResourceItems(trx, onProgress);
+    await populateAnchorEdges(trx, onProgress);
+    await populateResourceRefEdges(trx);
+    await populateImageItems(trx, resolvePageDomPaths, onProgress);
+}

package/lib/populate-entity-tables/populate-image-items.d.ts

@@ -0,0 +1,91 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { DomPathResult } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Callback that resolves DOM-path strings for one page's images. The
+ * callback receives the page's HTML string (or `null` when no snapshot
+ * is stored) plus the page's `images` rows, and must return one
+ * {@link DomPathResult} per row.
+ *
+ * Injected rather than hard-coded so `@nitpicker/archive` does not
+ * become a jsdom consumer at runtime — the migration script
+ * (`scripts/migrate-to-0.13.mjs`) wires a jsdom-backed
+ * implementation, while unit tests inject stubs. A future crawler-time
+ * DOM-path capture can use the same signature with
+ * puppeteer-backed elements.
+ */
+export type PageDomPathResolver = (pageId: number, htmlString: string | null, images: readonly ImageRowForResolver[]) => Promise<ReadonlyMap<number, DomPathResult>>;
+/**
+ * The projection of `images` rows passed to a
+ * {@link PageDomPathResolver}. Kept as a narrow shape (only the fields
+ * the resolver needs) so callers of the resolver do not depend on the
+ * full row width.
+ */
+export interface ImageRowForResolver {
+    /** Legacy `images.id`. */
+    id: number;
+    /** Legacy `images.sourceCode`. */
+    sourceCode: string | null;
+}
+/**
+ * Populates `image_items` from `images` (issue #193).
+ *
+ * The outer loop iterates **pages**, not images. Each page's images are
+ * processed as one whole unit — dom-path derivation, HTML BLOB fetch,
+ * and ref lookups all happen once per page. This design closes three
+ * separate correctness / efficiency traps that a purely image-chunked
+ * loop would expose:
+ *
+ * 1. **Ordinal-cursor bleed across chunks** — `matchImagesToDomPaths`
+ *    tracks per-outerHTML cursor state, so a page with 1 000 identical
+ *    `<img>` tags spanning two image-chunks would reset the cursor at
+ *    the boundary and duplicate dom_paths already assigned in the
+ *    previous chunk. Iterating by page keeps every page's cursor state
+ *    self-contained.
+ * 2. **Repeated `getPageHtml` + jsdom parse** — a page whose images
+ *    straddle N image-chunks would fetch and re-parse the same multi-MB
+ *    HTML snapshot N times. Iterating by page makes this exactly once
+ *    per page.
+ * 3. **Data-URI routing gap** — legacy `images.src` can be either a
+ *    plain URL (goes to `url_refs`) or a large `data:` URI (goes to
+ *    `blob_refs`, per the 512-byte threshold). The lookups partition
+ *    by that rule, so a data URI is never bound into
+ *    `resolveUrlRefs`' `WHERE url IN (?)` — which could otherwise
+ *    exceed SQLite's `SQLITE_MAX_SQL_LENGTH` for multi-KB URIs.
+ *
+ * Per outer iteration:
+ *
+ * 1. **Keyset-paginate** distinct `pageId` values from `images`. Peak
+ *    memory ≈ 500 integers.
+ * 2. **Fetch all images** for those page ids in one query, ordered by
+ *    `(pageId, id)` so JS-side grouping is a linear pass.
+ * 3. **Resolve dom paths per page** — one `getPageHtml` + one resolver
+ *    call each. `dom_path` texts are collected across the whole batch
+ *    so `upsertTextRefs` runs once per batch.
+ * 4. **Partition src / currentSrc** by the data-URI routing rule and
+ *    batch-resolve url refs, blob refs, and alt text refs across the
+ *    whole batch.
+ * 5. **Bulk INSERT** with explicit `id = images.id`.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent.
+ *
+ * HTML BLOBs are read inline via the same `trx` handle rather than
+ * through a caller-supplied `getPageHtml` callback: a callback that
+ * queried `Database.getHtmlOfPageById` (which uses the non-transactional
+ * Knex instance) would re-enter the pool from inside this writer
+ * transaction and deadlock on libsql's single writer connection. Reading
+ * `page_html_ref` + `page_html_blobs` via `trx` keeps every read on the
+ * same connection the outer trx already owns.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images (see {@link PageDomPathResolver}).
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of distinct `images.pageId` values scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * const jsdomResolver = createJsdomResolver();
+ * await knex.transaction(async (trx) => {
+ *   await populateImageItems(trx, jsdomResolver);
+ * });
+ */
+export declare function populateImageItems(trx: Knex, resolvePageDomPaths: PageDomPathResolver, onProgress?: ProgressCallback): Promise<void>;