@nitpicker/archive 0.0.0 → 0.25.1

package/lib/populate-entity-tables/populate-image-items.js

@@ -0,0 +1,226 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { decodePageHtmlBlobBytes } from '../decode-page-html-blob-bytes.js';
+import { isBlobRefValue } from './is-blob-ref-value.js';
+import { resolveBlobRefs } from './resolve-blob-refs.js';
+import { resolveTextRefs } from './resolve-text-refs.js';
+import { resolveUrlOrBlobFromMaps } from './resolve-url-or-blob-from-maps.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+import { upsertTextRefs } from './upsert-text-refs.js';
+/**
+ * Page ids scanned per keyset-paginated `SELECT DISTINCT pageId` chunk.
+ * Batches the outer loop so 470 K pages worth of `images` rows are
+ * traversed in ~1 K round trips instead of one round-trip per page,
+ * without holding the entire distinct-pageId list in memory at once.
+ */
+const PAGE_ID_CHUNK_SIZE = 500;
+/**
+ * Rows sent per `INSERT INTO image_items ... VALUES (...)` statement.
+ * Each row binds 13 params, so 300 rows = 3 900 params — safely under
+ * the SQLite variable limit.
+ */
+const INSERT_CHUNK_SIZE = 300;
+/**
+ * Populates `image_items` from `images` (issue #193).
+ *
+ * The outer loop iterates **pages**, not images. Each page's images are
+ * processed as one whole unit — dom-path derivation, HTML BLOB fetch,
+ * and ref lookups all happen once per page. This design closes three
+ * separate correctness / efficiency traps that a purely image-chunked
+ * loop would expose:
+ *
+ * 1. **Ordinal-cursor bleed across chunks** — `matchImagesToDomPaths`
+ *    tracks per-outerHTML cursor state, so a page with 1 000 identical
+ *    `<img>` tags spanning two image-chunks would reset the cursor at
+ *    the boundary and duplicate dom_paths already assigned in the
+ *    previous chunk. Iterating by page keeps every page's cursor state
+ *    self-contained.
+ * 2. **Repeated `getPageHtml` + jsdom parse** — a page whose images
+ *    straddle N image-chunks would fetch and re-parse the same multi-MB
+ *    HTML snapshot N times. Iterating by page makes this exactly once
+ *    per page.
+ * 3. **Data-URI routing gap** — legacy `images.src` can be either a
+ *    plain URL (goes to `url_refs`) or a large `data:` URI (goes to
+ *    `blob_refs`, per the 512-byte threshold). The lookups partition
+ *    by that rule, so a data URI is never bound into
+ *    `resolveUrlRefs`' `WHERE url IN (?)` — which could otherwise
+ *    exceed SQLite's `SQLITE_MAX_SQL_LENGTH` for multi-KB URIs.
+ *
+ * Per outer iteration:
+ *
+ * 1. **Keyset-paginate** distinct `pageId` values from `images`. Peak
+ *    memory ≈ 500 integers.
+ * 2. **Fetch all images** for those page ids in one query, ordered by
+ *    `(pageId, id)` so JS-side grouping is a linear pass.
+ * 3. **Resolve dom paths per page** — one `getPageHtml` + one resolver
+ *    call each. `dom_path` texts are collected across the whole batch
+ *    so `upsertTextRefs` runs once per batch.
+ * 4. **Partition src / currentSrc** by the data-URI routing rule and
+ *    batch-resolve url refs, blob refs, and alt text refs across the
+ *    whole batch.
+ * 5. **Bulk INSERT** with explicit `id = images.id`.
+ *
+ * `INSERT OR IGNORE` on the natural PK makes the step idempotent.
+ *
+ * HTML BLOBs are read inline via the same `trx` handle rather than
+ * through a caller-supplied `getPageHtml` callback: a callback that
+ * queried `Database.getHtmlOfPageById` (which uses the non-transactional
+ * Knex instance) would re-enter the pool from inside this writer
+ * transaction and deadlock on libsql's single writer connection. Reading
+ * `page_html_ref` + `page_html_blobs` via `trx` keeps every read on the
+ * same connection the outer trx already owns.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param resolvePageDomPaths - Callback that returns dom_path strings
+ *   for one page's images (see {@link PageDomPathResolver}).
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of distinct `images.pageId` values scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * const jsdomResolver = createJsdomResolver();
+ * await knex.transaction(async (trx) => {
+ *   await populateImageItems(trx, jsdomResolver);
+ * });
+ */
+export async function populateImageItems(trx, resolvePageDomPaths, onProgress) {
+    const countRows = await trx('images').countDistinct({ n: 'pageId' });
+    const totalRow = countRows[0];
+    const total = Number(totalRow?.n ?? 0);
+    const report = createProgressReporter('image_items (distinct pages)', total, onProgress);
+    let processedPages = 0;
+    let cursorPageId = 0;
+    while (true) {
+        const pageIdRows = await trx('images')
+            .distinct('pageId')
+            .where('pageId', '>', cursorPageId)
+            .orderBy('pageId', 'asc')
+            .limit(PAGE_ID_CHUNK_SIZE);
+        if (pageIdRows.length === 0) {
+            break;
+        }
+        const pageIds = pageIdRows.map((r) => r.pageId);
+        cursorPageId = pageIds.at(-1);
+        processedPages += pageIds.length;
+        report(processedPages);
+        const rows = await trx('images')
+            .select('id', 'pageId', 'src', 'currentSrc', 'alt', 'width', 'height', 'naturalWidth', 'naturalHeight', 'isLazy', 'viewportWidth', 'sourceCode')
+            .whereIn('pageId', pageIds)
+            .orderBy([
+            { column: 'pageId', order: 'asc' },
+            { column: 'id', order: 'asc' },
+        ]);
+        if (rows.length === 0) {
+            continue;
+        }
+        const byPage = new Map();
+        for (const row of rows) {
+            const bucket = byPage.get(row.pageId);
+            if (bucket === undefined) {
+                byPage.set(row.pageId, [row]);
+            }
+            else {
+                bucket.push(row);
+            }
+        }
+        const domPaths = new Map();
+        for (const [pageId, pageImages] of byPage) {
+            const html = await readPageHtmlInTrx(trx, pageId);
+            const resolved = await resolvePageDomPaths(pageId, html, pageImages);
+            for (const [imageId, entry] of resolved) {
+                domPaths.set(imageId, entry);
+            }
+        }
+        const urls = new Set();
+        const dataUris = new Set();
+        const alts = new Set();
+        const domPathTexts = new Set();
+        for (const row of rows) {
+            for (const value of [row.src, row.currentSrc]) {
+                if (typeof value !== 'string' || value === '') {
+                    continue;
+                }
+                if (isBlobRefValue(value)) {
+                    dataUris.add(value);
+                }
+                else {
+                    urls.add(value);
+                }
+            }
+            if (typeof row.alt === 'string' && row.alt !== '') {
+                alts.add(row.alt);
+            }
+            const domPath = domPaths.get(row.id);
+            if (domPath !== undefined) {
+                domPathTexts.add(domPath.path);
+            }
+        }
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const blobIds = await resolveBlobRefs(trx, dataUris);
+        const altIds = await resolveTextRefs(trx, alts);
+        // `dom_path` strings are synthesised right here and were not
+        // among the columns the `text_refs` populate scanned
+        // (`populate-ref-tables/populate-text-refs.ts`) — insert every distinct
+        // derived path (including `unknown/<id>` fallbacks) before
+        // resolving its id. The upsert is idempotent so re-runs across
+        // partial failures do not duplicate rows.
+        const domPathIds = await upsertTextRefs(trx, domPathTexts);
+        const inserts = [];
+        for (const row of rows) {
+            const domPath = domPaths.get(row.id);
+            if (domPath === undefined) {
+                throw new Error(`populateImageItems: dom_path not resolved for image id=${row.id}`);
+            }
+            const domPathId = domPathIds.get(domPath.path);
+            if (domPathId === undefined) {
+                throw new Error(`populateImageItems: text_refs.id not resolved for dom_path=${domPath.path} — the upsertTextRefs above must have inserted it`);
+            }
+            const srcSlot = resolveUrlOrBlobFromMaps(row.src, urlIds, blobIds);
+            const currentSrcSlot = resolveUrlOrBlobFromMaps(row.currentSrc, urlIds, blobIds);
+            inserts.push({
+                id: row.id,
+                page_id: row.pageId,
+                src_url_id: srcSlot.url,
+                current_src_url_id: currentSrcSlot.url,
+                src_blob_id: srcSlot.blob,
+                current_src_blob_id: currentSrcSlot.blob,
+                alt_text_id: typeof row.alt === 'string' && row.alt !== ''
+                    ? (altIds.get(row.alt) ?? null)
+                    : null,
+                width: row.width,
+                height: row.height,
+                natural_width: row.naturalWidth,
+                natural_height: row.naturalHeight,
+                is_lazy: row.isLazy == null ? null : row.isLazy ? 1 : 0,
+                viewport_width: row.viewportWidth,
+                dom_path_text_id: domPathId,
+            });
+        }
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('image_items').insert(chunk).onConflict('id').ignore();
+        }
+    }
+}
+/**
+ * Reads the stored HTML snapshot for one legacy page id, via the same
+ * `trx` handle that the outer populate transaction owns. Mirrors
+ * `Database.getHtmlOfPageById` but keeps every read on the trx
+ * connection so the populate does not re-enter the pool from
+ * inside its own writer transaction (see {@link populateImageItems}'s
+ * doc for the deadlock this avoids).
+ * @param trx - The open migration transaction.
+ * @param pageId - `pages.id` (== `content_items.id`) to read HTML for.
+ * @returns Decoded HTML string, or `null` when no snapshot is stored.
+ * @throws {Error} Naming the page and the blob hash when the snapshot is
+ *   corrupt or decompresses past `MAX_DECODED_SNAPSHOT_BYTES`; the migration
+ *   fails rather than resolving the page's images without its HTML.
+ */
+async function readPageHtmlInTrx(trx, pageId) {
+    const row = (await trx('page_html_ref')
+        .join('page_html_blobs', 'page_html_ref.hash', '=', 'page_html_blobs.hash')
+        .select('page_html_blobs.hash as hash', 'page_html_blobs.body as body', 'page_html_blobs.codec as codec')
+        .where('page_html_ref.page_id', pageId)
+        .first());
+    if (!row) {
+        return null;
+    }
+    return decodePageHtmlBlobBytes({ pageId, ...row }).toString('utf8');
+}

package/lib/populate-entity-tables/populate-page-meta.d.ts

@@ -0,0 +1,33 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `page_meta` from `pages WHERE scraped = 1` (issue #193).
+ *
+ * Pages that were never scraped contribute no meta rows — they have no rendered title,
+ * description, or JSON-LD to preserve, so filtering at the SELECT keeps
+ * the reader working set narrow and matches the migration acceptance check
+ * `count(*) FROM page_meta = count(*) FROM pages WHERE scraped = 1`.
+ *
+ * Per chunk:
+ *
+ * 1. **Keyset-paginate** `pages.id > cursor AND scraped = 1`.
+ * 2. **Batch-resolve** all text / URL / JSON ref ids for every value in
+ *    the chunk — three resolvers, one round-trip per ref table.
+ * 3. **Bulk INSERT** with `page_id = pages.id`. `page_id` is both PK and
+ *    FK on `content_items(id)`; `populateContentItems` must have already
+ *    inserted the corresponding `content_items` row for the FK to be
+ *    valid at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the `page_id` PK makes the step idempotent so
+ * partial-failure re-runs are safe.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages WHERE scraped = 1` scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ *   await populatePageMeta(trx);
+ * });
+ */
+export declare function populatePageMeta(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-entity-tables/populate-page-meta.js

@@ -0,0 +1,267 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { PAGE_META_COLUMN_MAPS } from '../page-meta-column-maps.js';
+import { DATA_URI_URL_REFS_LIMIT } from '../populate-ref-tables/data-uri-url-refs-limit.js';
+import { resolveJsonRefs } from './resolve-json-refs.js';
+import { resolveTextRefs } from './resolve-text-refs.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against
+ * `pages WHERE scraped = 1`. The row width here is dominated by long
+ * text columns (`meta_extras` JSON, `description`, `twitter_description`)
+ * so 400 rows caps peak chunk memory at ≈ 40 MB on the reference
+ * archive.
+ */
+const READ_CHUNK_SIZE = 400;
+/**
+ * Rows sent per `INSERT INTO page_meta ... VALUES (...)` statement. Each
+ * row binds 47 params (see the column list below), so 200 rows =
+ * 9 400 params — safely under SQLite's default variable limit.
+ */
+const INSERT_CHUNK_SIZE = 200;
+/**
+ * Populates `page_meta` from `pages WHERE scraped = 1` (issue #193).
+ *
+ * Pages that were never scraped contribute no meta rows — they have no rendered title,
+ * description, or JSON-LD to preserve, so filtering at the SELECT keeps
+ * the reader working set narrow and matches the migration acceptance check
+ * `count(*) FROM page_meta = count(*) FROM pages WHERE scraped = 1`.
+ *
+ * Per chunk:
+ *
+ * 1. **Keyset-paginate** `pages.id > cursor AND scraped = 1`.
+ * 2. **Batch-resolve** all text / URL / JSON ref ids for every value in
+ *    the chunk — three resolvers, one round-trip per ref table.
+ * 3. **Bulk INSERT** with `page_id = pages.id`. `page_id` is both PK and
+ *    FK on `content_items(id)`; `populateContentItems` must have already
+ *    inserted the corresponding `content_items` row for the FK to be
+ *    valid at COMMIT time.
+ *
+ * `INSERT OR IGNORE` on the `page_id` PK makes the step idempotent so
+ * partial-failure re-runs are safe.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages WHERE scraped = 1` scanned); see
+ *   {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentItems(trx);
+ *   await populatePageMeta(trx);
+ * });
+ */
+export async function populatePageMeta(trx, onProgress) {
+    const textColumns = PAGE_META_COLUMN_MAPS.text.map((entry) => entry.source);
+    const urlColumns = PAGE_META_COLUMN_MAPS.url.map((entry) => entry.source);
+    const plainColumns = [
+        'id',
+        'lang',
+        'dir',
+        'charset',
+        'baseHref',
+        'viewport_raw',
+        'themeColor',
+        'applicationName',
+        'author',
+        'generator',
+        'publisher',
+        'robots_noindex',
+        'robots_nofollow',
+        'robots_noarchive',
+        'robots_noimageindex',
+        'googlebot',
+        'og_type',
+        'og_site_name',
+        'og_image_alt',
+        'og_image_width',
+        'og_image_height',
+        'og_locale',
+        'og_article_published_time',
+        'og_article_modified_time',
+        'twitter_card',
+        'twitter_site',
+        'twitter_creator',
+        'fb_app_id',
+        'verification_google',
+        'formatDetection_telephone',
+        'tag_count',
+        'jsonld_count',
+        'tags_providers_csv',
+        'meta_extras',
+    ];
+    const countRows = await trx('pages').where('scraped', true).count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('page_meta (pages)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('pages')
+            .select(...plainColumns, ...textColumns, ...urlColumns)
+            .where('id', '>', cursor)
+            .andWhere('scraped', true)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        const texts = new Set();
+        const urls = new Set();
+        const jsonStrings = new Set();
+        for (const row of rows) {
+            for (const column of textColumns) {
+                const value = row[column];
+                if (typeof value === 'string' && value !== '') {
+                    texts.add(value);
+                }
+            }
+            for (const column of urlColumns) {
+                const value = row[column];
+                if (typeof value === 'string' && value !== '' && !isLargeDataUri(value)) {
+                    urls.add(value);
+                }
+            }
+            const metaExtras = row.meta_extras;
+            if (typeof metaExtras === 'string' && metaExtras !== '') {
+                jsonStrings.add(metaExtras);
+            }
+        }
+        const textIds = await resolveTextRefs(trx, texts);
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const jsonIds = await resolveJsonRefs(trx, jsonStrings);
+        const inserts = rows.map((row) => {
+            const rowId = row.id;
+            const insert = {
+                page_id: rowId,
+                lang: row.lang ?? null,
+                dir: row.dir ?? null,
+                charset: row.charset ?? null,
+                base_href: row.baseHref ?? null,
+                viewport_raw: row.viewport_raw ?? null,
+                theme_color: row.themeColor ?? null,
+                application_name: row.applicationName ?? null,
+                author: row.author ?? null,
+                generator: row.generator ?? null,
+                publisher: row.publisher ?? null,
+                robots_noindex: row.robots_noindex ?? null,
+                robots_nofollow: row.robots_nofollow ?? null,
+                robots_noarchive: row.robots_noarchive ?? null,
+                robots_noimageindex: row.robots_noimageindex ?? null,
+                googlebot: row.googlebot ?? null,
+                og_type: row.og_type ?? null,
+                og_site_name: row.og_site_name ?? null,
+                og_image_alt: row.og_image_alt ?? null,
+                og_image_width: row.og_image_width ?? null,
+                og_image_height: row.og_image_height ?? null,
+                og_locale: row.og_locale ?? null,
+                og_article_published_time: row.og_article_published_time ?? null,
+                og_article_modified_time: row.og_article_modified_time ?? null,
+                twitter_card: row.twitter_card ?? null,
+                twitter_site: row.twitter_site ?? null,
+                twitter_creator: row.twitter_creator ?? null,
+                fb_app_id: row.fb_app_id ?? null,
+                verification_google: row.verification_google ?? null,
+                format_detection_telephone: row.formatDetection_telephone ?? null,
+                tag_count: row.tag_count ?? null,
+                jsonld_count: row.jsonld_count ?? null,
+                tags_providers_csv: row.tags_providers_csv ?? null,
+                meta_extras_json_id: resolveOptionalJsonId(row.meta_extras, jsonIds, rowId),
+            };
+            for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
+                insert[target] = resolveOptionalTextId(row[source], textIds, rowId, source);
+            }
+            for (const { source, target } of PAGE_META_COLUMN_MAPS.url) {
+                insert[target] = resolveOptionalUrlId(row[source], urlIds, rowId, source);
+            }
+            return insert;
+        });
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('page_meta').insert(chunk).onConflict('page_id').ignore();
+        }
+    }
+}
+/**
+ * Returns `true` when `value` is a `data:` URI larger than the routing
+ * threshold — i.e. one that the `url_refs` populate
+ * (`populate-ref-tables/populate-url-refs.ts`) sent to `blob_refs` instead
+ * of `url_refs`. `page_meta` deliberately has no `*_blob_id` companion
+ * columns for its URL slots, so we skip these values from
+ * the `url_refs` lookup rather than issue a doomed `WHERE url IN (?)`
+ * that would burn a chunk slot per multi-KB URI. The resulting
+ * `*_url_id` is null with a `console.warn` breadcrumb; the operator
+ * can then decide whether the archive needs a schema extension or the
+ * raw data warrants normalisation.
+ * @param value - Raw URL column value.
+ */
+function isLargeDataUri(value) {
+    return value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT;
+}
+/**
+ * Resolves `text_refs.id` for a `page_meta.<name>_text_id` slot or
+ * throws when a non-empty source value has no matching `text_refs` row
+ * (parity with `populateContentItems`' `url_id` throw — silent NULL
+ * corruption would still let the count-based acceptance invariant pass
+ * while individual `page_meta` rows lose their meta text).
+ * @param value - Raw column value from the source `pages` row.
+ * @param textIds - `text → text_refs.id` map batch-resolved for the
+ *   current chunk (see {@link ./resolve-text-refs.ts}).
+ * @param pageId - Owning page id (for the error message).
+ * @param column - Source column name (for the error message).
+ */
+function resolveOptionalTextId(value, textIds, pageId, column) {
+    if (typeof value !== 'string' || value === '') {
+        return null;
+    }
+    const id = textIds.get(value);
+    if (id === undefined) {
+        throw new Error(`populatePageMeta: text_refs.id not resolved for page id=${pageId} column=${column} — populateTextRefs must run first`);
+    }
+    return id;
+}
+/**
+ * Resolves `url_refs.id` for a `page_meta.<name>_url_id` slot, throws
+ * on a missing lookup for a regular URL, and returns `null` (with a
+ * warning) for large data URIs that the `url_refs` populate routed to
+ * `blob_refs` — those cannot land in `page_meta` because the schema has
+ * no `*_blob_id` companion columns.
+ * @param value - Raw column value from the source `pages` row.
+ * @param urlIds - `url → url_refs.id` map batch-resolved for the
+ *   current chunk (see {@link ./resolve-url-refs.ts}).
+ * @param pageId - Owning page id (for the error message).
+ * @param column - Source column name (for the error message).
+ */
+function resolveOptionalUrlId(value, urlIds, pageId, column) {
+    if (typeof value !== 'string' || value === '') {
+        return null;
+    }
+    if (isLargeDataUri(value)) {
+        // oxlint-disable-next-line no-console
+        console.warn(`populatePageMeta: dropping large data URI in page id=${pageId} column=${column} (page_meta has no *_blob_id companion for URL columns)`);
+        return null;
+    }
+    const id = urlIds.get(value);
+    if (id === undefined) {
+        throw new Error(`populatePageMeta: url_refs.id not resolved for page id=${pageId} column=${column} — populateUrlRefs must run first`);
+    }
+    return id;
+}
+/**
+ * Resolves `json_refs.id` for `page_meta.meta_extras_json_id` or throws
+ * when the source `meta_extras` string has no matching `json_refs` row
+ * (parity with the text/url branches above).
+ * @param value - Raw `meta_extras` value from the source `pages` row.
+ * @param jsonIds - `JSON string → json_refs.id` map batch-resolved for
+ *   the current chunk (see {@link ./resolve-json-refs.ts}).
+ * @param pageId - Owning page id (for the error message).
+ */
+function resolveOptionalJsonId(value, jsonIds, pageId) {
+    if (typeof value !== 'string' || value === '') {
+        return null;
+    }
+    const id = jsonIds.get(value);
+    if (id === undefined) {
+        throw new Error(`populatePageMeta: json_refs.id not resolved for page id=${pageId} column=meta_extras — populateJsonRefs must run first`);
+    }
+    return id;
+}

package/lib/populate-entity-tables/populate-resource-items.d.ts

@@ -0,0 +1,22 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `resource_items` from `resources` (issue #193).
+ *
+ * Structurally analogous to {@link ./populate-content-items.ts}: for each
+ * `resources` chunk, batch-resolve `url_refs.id`, `content_type_refs.id`,
+ * and `header_sets.id`, then bulk-INSERT with explicit `id = resources.id`.
+ * Reusing the legacy PK preserves the `resources-referrers.resourceId`
+ * FK reference (which `populateResourceRefEdges` later carries over as
+ * `resource_ref_edges.resource_id`) without any per-row UPDATE.
+ *
+ * `INSERT OR IGNORE` on `id` makes the step idempotent.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `resources` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ * });
+ */
+export declare function populateResourceItems(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-entity-tables/populate-resource-items.js

@@ -0,0 +1,114 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { isBlobRefValue } from './is-blob-ref-value.js';
+import { resolveBlobRefs } from './resolve-blob-refs.js';
+import { loadContentTypeRefs } from './resolve-content-type-refs.js';
+import { resolveHeaderSets } from './resolve-header-sets.js';
+import { resolveUrlOrBlobFromMaps } from './resolve-url-or-blob-from-maps.js';
+import { resolveUrlRefs } from './resolve-url-refs.js';
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk against `resources`.
+ * `resources` rows are narrow (no meta columns) so 800 rows fits
+ * comfortably in a chunk without stressing memory. Bumping this higher
+ * would help throughput but risks pushing large `responseHeaders` JSON
+ * blobs past a healthy per-result-set size on archives with per-row
+ * multi-KB header payloads.
+ */
+const READ_CHUNK_SIZE = 800;
+/**
+ * Rows sent per `INSERT INTO resource_items ... VALUES (...)` statement.
+ * Each row binds 11 params (id + 10 columns), so 500 rows = 5 500 params
+ * — well under SQLite's default variable limit.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Populates `resource_items` from `resources` (issue #193).
+ *
+ * Structurally analogous to {@link ./populate-content-items.ts}: for each
+ * `resources` chunk, batch-resolve `url_refs.id`, `content_type_refs.id`,
+ * and `header_sets.id`, then bulk-INSERT with explicit `id = resources.id`.
+ * Reusing the legacy PK preserves the `resources-referrers.resourceId`
+ * FK reference (which `populateResourceRefEdges` later carries over as
+ * `resource_ref_edges.resource_id`) without any per-row UPDATE.
+ *
+ * `INSERT OR IGNORE` on `id` makes the step idempotent.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `resources` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateResourceItems(trx);
+ * });
+ */
+export async function populateResourceItems(trx, onProgress) {
+    const contentTypeIds = await loadContentTypeRefs(trx);
+    const countRows = await trx('resources').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('resource_items (resources)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('resources')
+            .select('id', 'url', 'isExternal', 'status', 'statusText', 'contentType', 'contentLength', 'responseHeaders', 'compress', 'cdn', 'source')
+            .where('id', '>', cursor)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        const urls = new Set();
+        const dataUris = new Set();
+        const headerJsonStrings = new Set();
+        for (const row of rows) {
+            if (isBlobRefValue(row.url)) {
+                dataUris.add(row.url);
+            }
+            else {
+                urls.add(row.url);
+            }
+            if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') {
+                headerJsonStrings.add(row.responseHeaders);
+            }
+        }
+        const urlIds = await resolveUrlRefs(trx, urls);
+        const blobIds = await resolveBlobRefs(trx, dataUris);
+        const headerSetIds = await resolveHeaderSets(trx, headerJsonStrings);
+        const inserts = rows.map((row) => {
+            const slot = resolveUrlOrBlobFromMaps(row.url, urlIds, blobIds);
+            if (slot.url === null && slot.blob === null) {
+                throw new Error(`populateResourceItems: neither url_refs.id nor blob_refs.id resolved for resource id=${row.id} url=${row.url.slice(0, 80)} — populateUrlRefs/populateBlobRefs must run first`);
+            }
+            let contentTypeId = null;
+            if (row.contentType != null && row.contentType !== '') {
+                const resolved = contentTypeIds.get(row.contentType);
+                if (resolved === undefined) {
+                    throw new Error(`populateResourceItems: content_type_refs.id not resolved for resource id=${row.id} contentType=${row.contentType} — populateContentTypeRefs must run first`);
+                }
+                contentTypeId = resolved;
+            }
+            const headerSetId = typeof row.responseHeaders === 'string' && row.responseHeaders !== ''
+                ? (headerSetIds.get(row.responseHeaders) ?? null)
+                : null;
+            return {
+                id: row.id,
+                url_id: slot.url,
+                url_blob_id: slot.blob,
+                is_external: row.isExternal == null ? 0 : row.isExternal ? 1 : 0,
+                status: row.status ?? null,
+                status_text: row.statusText ?? null,
+                content_type_id: contentTypeId,
+                content_length: row.contentLength ?? null,
+                header_set_id: headerSetId,
+                compress: row.compress ?? null,
+                cdn: row.cdn ?? null,
+                source: row.source,
+            };
+        });
+        for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+            const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+            await trx('resource_items').insert(chunk).onConflict('id').ignore();
+        }
+    }
+}