@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/populate-ref-tables/populate-blob-refs.js

@@ -0,0 +1,134 @@
+import { zstdCompressSync } from 'node:zlib';
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { computeContentHash } from './compute-content-hash.js';
+import { DATA_URI_URL_REFS_LIMIT } from './data-uri-url-refs-limit.js';
+import { decodeDataUri } from './decode-data-uri.js';
+/** Rows scanned per `SELECT`. See `populateTextRefs` for chunking rationale. */
+const READ_CHUNK_SIZE = 2000;
+/**
+ * Rows written per `INSERT`. blob_refs rows carry a compressed body BLOB
+ * so 100 rows keeps the total payload of a single INSERT statement well
+ * under WAL frame limits even for large SVGs.
+ */
+const INSERT_CHUNK_SIZE = 100;
+/**
+ * Source table + its URL-shaped column(s) that may hold a data URI large
+ * enough to route to `blob_refs`. `resources.url` is the resource's own
+ * identity (unlike `pages`/`content_items`, whose own URL is always a
+ * real http(s) address in practice) — a `<link>`/CSS sub-resource can
+ * legally be captured as an inline `data:` URI.
+ */
+const URL_SOURCES = [
+    { table: 'images', columns: ['src', 'currentSrc'] },
+    { table: 'resources', columns: ['url'] },
+];
+/**
+ * Populates `blob_refs` from every `images.src` / `images.currentSrc` /
+ * `resources.url` value that is a data URI longer than
+ * {@link DATA_URI_URL_REFS_LIMIT} bytes (issue #191).
+ *
+ * For each such value:
+ *
+ * 1. `decodeDataUri` strips the `data:...;base64,` (or `data:...,`) prefix
+ *    and returns the raw payload bytes.
+ * 2. `computeContentHash` hashes the payload bytes (32-byte SHA-256).
+ * 3. The payload is zstd-compressed (`codec='zstd'`) — matching
+ *    `page_html_blobs` / `json_refs`.
+ * 4. `INSERT OR IGNORE` on `hash` deduplicates: an `<img>` element and a
+ *    resource that share the same underlying base64 payload share one
+ *    `blob_refs` row.
+ *
+ * Malformed data URIs that `decodeDataUri` cannot decode are logged and
+ * skipped — the raw URI string still exists in the source row, and the
+ * migration script's operator can hunt it down from the warning log. The
+ * alternative — routing malformed large URIs back to `url_refs` — would
+ * break the "data URIs > threshold live in blob_refs" contract that 0.13
+ * lookups depend on.
+ *
+ * On the reference archive only ~429 images use data URIs, so the total
+ * dictionary is tiny — this step is I/O-cheap even without full
+ * chunking, but the same batching shape is used as elsewhere for
+ * consistency.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateBlobRefs(trx);
+ * });
+ */
+export async function populateBlobRefs(trx, onProgress) {
+    const seen = new Set();
+    const pending = [];
+    for (const source of URL_SOURCES) {
+        const hasTable = await trx.schema.hasTable(source.table);
+        if (!hasTable) {
+            continue;
+        }
+        const presentColumns = [];
+        for (const column of source.columns) {
+            if (await trx.schema.hasColumn(source.table, column)) {
+                presentColumns.push(column);
+            }
+        }
+        if (presentColumns.length === 0) {
+            continue;
+        }
+        const countRows = await trx(source.table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`blob_refs (${source.table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(source.table)
+                .select('id', ...presentColumns)
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                for (const column of presentColumns) {
+                    const value = row[column];
+                    if (value == null || value.length <= DATA_URI_URL_REFS_LIMIT) {
+                        continue;
+                    }
+                    if (!value.startsWith('data:')) {
+                        continue;
+                    }
+                    const decoded = decodeDataUri(value);
+                    if (decoded === null) {
+                        // eslint-disable-next-line no-console
+                        console.warn(`populateBlobRefs: skipping malformed data URI (length=${value.length}) — raw string still exists in ${source.table} row`);
+                        continue;
+                    }
+                    const hash = computeContentHash(decoded.bytes);
+                    const hex = hash.toString('hex');
+                    if (seen.has(hex)) {
+                        continue;
+                    }
+                    seen.add(hex);
+                    const compressed = zstdCompressSync(decoded.bytes);
+                    pending.push({
+                        hash,
+                        body: compressed,
+                        codec: 'zstd',
+                        size_raw: decoded.bytes.byteLength,
+                        size_stored: compressed.byteLength,
+                    });
+                    if (pending.length >= INSERT_CHUNK_SIZE) {
+                        await trx('blob_refs').insert(pending).onConflict('hash').ignore();
+                        pending.length = 0;
+                    }
+                }
+            }
+        }
+    }
+    if (pending.length > 0) {
+        await trx('blob_refs').insert(pending).onConflict('hash').ignore();
+    }
+}

package/lib/populate-ref-tables/populate-content-type-refs.d.ts

@@ -0,0 +1,27 @@
+import type { Knex } from 'knex';
+/**
+ * Populates `content_type_refs` from every distinct `contentType` value
+ * currently stored in `pages` and `resources` (issue #191).
+ *
+ * Two independent DISTINCT SELECTs (one per table) are merged in JS
+ * rather than via SQL `UNION` — the cardinality is small in practice and
+ * doing it in JS avoids the knex-`union().select()` column-aliasing
+ * quirk (bare `.select()` on a union wraps in `SELECT *` and can lose
+ * the column name depending on driver version). Two per-table SELECTs
+ * are also fast because `contentType` is indexed on both tables via
+ * `idx_pages_listfilter` / natural column index (see `init-schema.ts`).
+ *
+ * `normalized` and `category` are derived in JS via {@link classifyContentType}
+ * so the rule table stays in one place; SQLite has no equivalent
+ * expression.
+ *
+ * `INSERT OR IGNORE` on the natural key `raw` makes this idempotent —
+ * re-running this populate after a partial failure never duplicates
+ * rows, only appends the new ones.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentTypeRefs(trx);
+ * });
+ */
+export declare function populateContentTypeRefs(trx: Knex): Promise<void>;

package/lib/populate-ref-tables/populate-content-type-refs.js

@@ -0,0 +1,70 @@
+import { classifyContentType } from './classify-content-type.js';
+import { normalizeMime } from './normalize-mime.js';
+/**
+ * Distinct-`contentType` rows are collected from `pages` + `resources` in
+ * chunks of this size before being written into `content_type_refs`. The
+ * DISTINCT list is expected to be tiny (< 1000 rows even on the largest
+ * archives — the wire content-type space is small) so the chunk size only
+ * bounds worst-case parameter counts on `INSERT ... VALUES ...`.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Populates `content_type_refs` from every distinct `contentType` value
+ * currently stored in `pages` and `resources` (issue #191).
+ *
+ * Two independent DISTINCT SELECTs (one per table) are merged in JS
+ * rather than via SQL `UNION` — the cardinality is small in practice and
+ * doing it in JS avoids the knex-`union().select()` column-aliasing
+ * quirk (bare `.select()` on a union wraps in `SELECT *` and can lose
+ * the column name depending on driver version). Two per-table SELECTs
+ * are also fast because `contentType` is indexed on both tables via
+ * `idx_pages_listfilter` / natural column index (see `init-schema.ts`).
+ *
+ * `normalized` and `category` are derived in JS via {@link classifyContentType}
+ * so the rule table stays in one place; SQLite has no equivalent
+ * expression.
+ *
+ * `INSERT OR IGNORE` on the natural key `raw` makes this idempotent —
+ * re-running this populate after a partial failure never duplicates
+ * rows, only appends the new ones.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateContentTypeRefs(trx);
+ * });
+ */
+export async function populateContentTypeRefs(trx) {
+    const distinctRaw = new Set();
+    if (await trx.schema.hasTable('pages')) {
+        const pagesRows = await trx('pages')
+            .distinct('contentType')
+            .whereNotNull('contentType');
+        for (const { contentType } of pagesRows) {
+            if (contentType != null && contentType !== '') {
+                distinctRaw.add(contentType);
+            }
+        }
+    }
+    if (await trx.schema.hasTable('resources')) {
+        const resourcesRows = await trx('resources')
+            .distinct('contentType')
+            .whereNotNull('contentType');
+        for (const { contentType } of resourcesRows) {
+            if (contentType != null && contentType !== '') {
+                distinctRaw.add(contentType);
+            }
+        }
+    }
+    if (distinctRaw.size === 0) {
+        return;
+    }
+    const inserts = [...distinctRaw].map((raw) => ({
+        raw,
+        normalized: normalizeMime(raw),
+        category: classifyContentType(raw),
+    }));
+    for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+        const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+        await trx('content_type_refs').insert(chunk).onConflict('raw').ignore();
+    }
+}

package/lib/populate-ref-tables/populate-header-tables.d.ts

@@ -0,0 +1,35 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates the five header decomposition tables (`header_name_refs`,
+ * `header_value_refs`, `header_sets`, `header_set_entries`,
+ * `header_flags`) from every `pages.responseHeaders` and
+ * `resources.responseHeaders` JSON blob (issue #191).
+ *
+ * Strategy:
+ *
+ * 1. **Warm caches** via {@link ./create-header-table-caches.ts} — load
+ *    existing `header_name_refs`, `header_value_refs`, and `header_sets`
+ *    rows into id maps. Idempotent re-runs then reuse ids instead of
+ *    re-issuing an upsert per entry, and new ids are appended as they
+ *    are inserted.
+ * 2. **Stream `pages.responseHeaders` + `resources.responseHeaders`** in
+ *    id-keyset chunks. Each non-null value is decomposed via
+ *    {@link decomposeHeaderSet} and written via
+ *    {@link ./upsert-one-header-set.ts} — the same per-set primitive the
+ *    crawler's live write path uses, so migrated and live-crawled
+ *    archives produce identical header rows.
+ *
+ * Every write is bulk-batched so the migration stays O(chunks × per-
+ * chunk-distinct-decomposed-sets) round-trips against the DB. Idempotency
+ * across full re-runs is guaranteed by the `INSERT OR IGNORE`s and the
+ * `setIdsProcessedThisRun` guard inside the per-set upsert.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateHeaderTables(trx);
+ * });
+ */
+export declare function populateHeaderTables(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-ref-tables/populate-header-tables.js

@@ -0,0 +1,80 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { createHeaderTableCaches } from './create-header-table-caches.js';
+import { decomposeHeaderSet } from './decompose-header-set.js';
+import { upsertOneHeaderSet } from './upsert-one-header-set.js';
+/**
+ * Rows scanned per SELECT chunk against `pages.responseHeaders` /
+ * `resources.responseHeaders`. Keyset-paginated on `id` to avoid loading
+ * a 470k-row `responseHeaders` column into a single query result.
+ */
+const READ_CHUNK_SIZE = 500;
+/**
+ * Populates the five header decomposition tables (`header_name_refs`,
+ * `header_value_refs`, `header_sets`, `header_set_entries`,
+ * `header_flags`) from every `pages.responseHeaders` and
+ * `resources.responseHeaders` JSON blob (issue #191).
+ *
+ * Strategy:
+ *
+ * 1. **Warm caches** via {@link ./create-header-table-caches.ts} — load
+ *    existing `header_name_refs`, `header_value_refs`, and `header_sets`
+ *    rows into id maps. Idempotent re-runs then reuse ids instead of
+ *    re-issuing an upsert per entry, and new ids are appended as they
+ *    are inserted.
+ * 2. **Stream `pages.responseHeaders` + `resources.responseHeaders`** in
+ *    id-keyset chunks. Each non-null value is decomposed via
+ *    {@link decomposeHeaderSet} and written via
+ *    {@link ./upsert-one-header-set.ts} — the same per-set primitive the
+ *    crawler's live write path uses, so migrated and live-crawled
+ *    archives produce identical header rows.
+ *
+ * Every write is bulk-batched so the migration stays O(chunks × per-
+ * chunk-distinct-decomposed-sets) round-trips against the DB. Idempotency
+ * across full re-runs is guaranteed by the `INSERT OR IGNORE`s and the
+ * `setIdsProcessedThisRun` guard inside the per-set upsert.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateHeaderTables(trx);
+ * });
+ */
+export async function populateHeaderTables(trx, onProgress) {
+    const caches = await createHeaderTableCaches(trx);
+    for (const table of ['pages', 'resources']) {
+        const hasTable = await trx.schema.hasTable(table);
+        if (!hasTable) {
+            continue;
+        }
+        const hasColumn = await trx.schema.hasColumn(table, 'responseHeaders');
+        if (!hasColumn) {
+            continue;
+        }
+        const countRows = await trx(table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`header_tables (${table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(table)
+                .select('id', 'responseHeaders')
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                const decomposed = decomposeHeaderSet(row.responseHeaders);
+                if (decomposed === null) {
+                    continue;
+                }
+                await upsertOneHeaderSet(trx, decomposed, caches);
+            }
+        }
+    }
+}

package/lib/populate-ref-tables/populate-json-refs.d.ts

@@ -0,0 +1,29 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `json_refs` from every non-null `pages.meta_extras` value
+ * (issue #191).
+ *
+ * Each value is:
+ *
+ * 1. Hashed with {@link computeContentHash} (32-byte SHA-256; see that
+ *    function's docs for the algorithm-choice rationale).
+ * 2. Compressed with `zstdCompressSync` (`codec='zstd'`) — same encoder
+ *    used by `page_html_blobs`.
+ * 3. Deduplicated by hash — identical raw JSON strings produce one row.
+ *
+ * `size_raw` records the uncompressed byte length (UTF-8) and
+ * `size_stored` records the compressed byte length; consumers can
+ * estimate compression savings without decompressing.
+ *
+ * `INSERT OR IGNORE` on `hash` makes the step idempotent across partial-
+ * failure restarts.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateJsonRefs(trx);
+ * });
+ */
+export declare function populateJsonRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-ref-tables/populate-json-refs.js

@@ -0,0 +1,101 @@
+import { zstdCompressSync } from 'node:zlib';
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { computeContentHash } from './compute-content-hash.js';
+/**
+ * Number of `json_refs` rows sent per INSERT. Each row binds 5 params
+ * (hash + json_text + codec + size_raw + size_stored). 200 rows = 1000
+ * params — comfortable margin under SQLite's default variable limit and
+ * bounds any one INSERT's compressed-blob payload total to a size that
+ * fits in one WAL frame.
+ */
+const INSERT_CHUNK_SIZE = 200;
+/**
+ * Rows scanned per source query. Same rationale as the other populators:
+ * streaming with keyset pagination on `id` avoids loading a 470k-row
+ * `meta_extras` column into a single result set.
+ */
+const READ_CHUNK_SIZE = 1000;
+/**
+ * Populates `json_refs` from every non-null `pages.meta_extras` value
+ * (issue #191).
+ *
+ * Each value is:
+ *
+ * 1. Hashed with {@link computeContentHash} (32-byte SHA-256; see that
+ *    function's docs for the algorithm-choice rationale).
+ * 2. Compressed with `zstdCompressSync` (`codec='zstd'`) — same encoder
+ *    used by `page_html_blobs`.
+ * 3. Deduplicated by hash — identical raw JSON strings produce one row.
+ *
+ * `size_raw` records the uncompressed byte length (UTF-8) and
+ * `size_stored` records the compressed byte length; consumers can
+ * estimate compression savings without decompressing.
+ *
+ * `INSERT OR IGNORE` on `hash` makes the step idempotent across partial-
+ * failure restarts.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateJsonRefs(trx);
+ * });
+ */
+export async function populateJsonRefs(trx, onProgress) {
+    const hasPages = await trx.schema.hasTable('pages');
+    if (!hasPages) {
+        return;
+    }
+    const hasMetaExtras = await trx.schema.hasColumn('pages', 'meta_extras');
+    if (!hasMetaExtras) {
+        return;
+    }
+    const seen = new Set();
+    const pending = [];
+    const countRows = await trx('pages').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('json_refs (pages)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('pages')
+            .select('id', 'meta_extras as value')
+            .where('id', '>', cursor)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        for (const row of rows) {
+            const raw = row.value;
+            if (raw == null || raw === '') {
+                continue;
+            }
+            const rawBytes = Buffer.from(raw, 'utf8');
+            const hash = computeContentHash(rawBytes);
+            const hex = hash.toString('hex');
+            if (seen.has(hex)) {
+                continue;
+            }
+            seen.add(hex);
+            const compressed = zstdCompressSync(rawBytes);
+            pending.push({
+                hash,
+                json_text: compressed,
+                codec: 'zstd',
+                size_raw: rawBytes.byteLength,
+                size_stored: compressed.byteLength,
+            });
+            if (pending.length >= INSERT_CHUNK_SIZE) {
+                await trx('json_refs').insert(pending).onConflict('hash').ignore();
+                pending.length = 0;
+            }
+        }
+    }
+    if (pending.length > 0) {
+        await trx('json_refs').insert(pending).onConflict('hash').ignore();
+    }
+}

package/lib/populate-ref-tables/populate-refs.d.ts

@@ -0,0 +1,51 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Runs the six 0.13 ref-table populates (issue #191) in this fixed
+ * order against an already-connected archive:
+ *
+ * 1. `content_type_refs`
+ * 2. `url_refs`
+ * 3. `text_refs`
+ * 4. `json_refs`
+ * 5. `blob_refs`
+ * 6. Header tables (name / value / set / entries / flags).
+ *
+ * The order is an execution-order invariant, not arbitrary: the entity
+ * populates that run afterwards (`populateEntityTables`) JOIN against
+ * `content_type_refs` when filling `content_items` / `resource_items`,
+ * and look up `url_refs.id` for every URL column, so both dictionaries
+ * must be complete first. Within this function the ordering is:
+ * dictionary tables (1–5) first, header tables (6) last — the header
+ * tables' entries reference name / value refs, but those are inserted
+ * by `populateHeaderTables` itself, so no barrier between sub-populates
+ * is required.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE`
+ * on its natural key. Running this orchestrator twice on the same archive
+ * produces the same rows — no phase marker table is used. This function
+ * does not open its own transaction so the caller controls the boundary;
+ * the whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level.
+ *
+ * `scripts/migrate-to-0.13.mjs` does NOT call this orchestrator directly
+ * — it calls the same six sub-populates itself, each in its own
+ * transaction, so a killed multi-hour migration only re-does the one
+ * table in flight on resume rather than the whole six-table batch. Use
+ * this function directly when that per-table checkpointing is not
+ * needed and a single all-or-nothing transaction is preferred.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * // Single-transaction use: connect via Archive, wrap in a
+ * // transaction, run every sub-step, then rely on the caller's `.bak`
+ * // safety net if any step throws.
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateRefTables(trx);
+ * });
+ * await archive.write();
+ */
+export declare function populateRefTables(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/populate-ref-tables/populate-refs.js

@@ -0,0 +1,62 @@
+import { populateBlobRefs } from './populate-blob-refs.js';
+import { populateContentTypeRefs } from './populate-content-type-refs.js';
+import { populateHeaderTables } from './populate-header-tables.js';
+import { populateJsonRefs } from './populate-json-refs.js';
+import { populateTextRefs } from './populate-text-refs.js';
+import { populateUrlRefs } from './populate-url-refs.js';
+/**
+ * Runs the six 0.13 ref-table populates (issue #191) in this fixed
+ * order against an already-connected archive:
+ *
+ * 1. `content_type_refs`
+ * 2. `url_refs`
+ * 3. `text_refs`
+ * 4. `json_refs`
+ * 5. `blob_refs`
+ * 6. Header tables (name / value / set / entries / flags).
+ *
+ * The order is an execution-order invariant, not arbitrary: the entity
+ * populates that run afterwards (`populateEntityTables`) JOIN against
+ * `content_type_refs` when filling `content_items` / `resource_items`,
+ * and look up `url_refs.id` for every URL column, so both dictionaries
+ * must be complete first. Within this function the ordering is:
+ * dictionary tables (1–5) first, header tables (6) last — the header
+ * tables' entries reference name / value refs, but those are inserted
+ * by `populateHeaderTables` itself, so no barrier between sub-populates
+ * is required.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE`
+ * on its natural key. Running this orchestrator twice on the same archive
+ * produces the same rows — no phase marker table is used. This function
+ * does not open its own transaction so the caller controls the boundary;
+ * the whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level.
+ *
+ * `scripts/migrate-to-0.13.mjs` does NOT call this orchestrator directly
+ * — it calls the same six sub-populates itself, each in its own
+ * transaction, so a killed multi-hour migration only re-does the one
+ * table in flight on resume rather than the whole six-table batch. Use
+ * this function directly when that per-table checkpointing is not
+ * needed and a single all-or-nothing transaction is preferred.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * // Single-transaction use: connect via Archive, wrap in a
+ * // transaction, run every sub-step, then rely on the caller's `.bak`
+ * // safety net if any step throws.
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateRefTables(trx);
+ * });
+ * await archive.write();
+ */
+export async function populateRefTables(trx, onProgress) {
+    await populateContentTypeRefs(trx);
+    await populateUrlRefs(trx, onProgress);
+    await populateTextRefs(trx, onProgress);
+    await populateJsonRefs(trx, onProgress);
+    await populateBlobRefs(trx, onProgress);
+    await populateHeaderTables(trx, onProgress);
+}

package/lib/populate-ref-tables/populate-text-refs.d.ts

@@ -0,0 +1,32 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `text_refs` from every text-shaped column across `anchors`,
+ * `images`, and `pages` (issue #191).
+ *
+ * Rules:
+ *
+ * - **Content-hash dedup contract**: identical text produces one row. The
+ *   hash is derived by {@link computeContentHash} (currently SHA-256,
+ *   matching `page_html_blobs.hash`; see that function's docs for the
+ *   algorithm-choice rationale).
+ * - **Empty and null are skipped** — an empty `<a>` tag adds no text to
+ *   the dictionary. Anchors with no text still get an `anchor_edges` row
+ *   in 0.13, just with `first_text_id = NULL`.
+ * - **INSERT OR IGNORE on (hash, text)** — the composite UNIQUE from the
+ *   0.13 DDL — makes the step idempotent across partial-failure
+ *   restarts.
+ *
+ * Peak memory is bounded by the count of distinct texts (all texts are
+ * held in a Map before insert). On the reference archive this Map is
+ * about 5 MB of anchor textContent + a few MB of page meta text — well
+ * within the migration process budget.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateTextRefs(trx);
+ * });
+ */
+export declare function populateTextRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;