@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/archive/populate-ref-tables/populate-header-tables.d.ts

@@ -0,0 +1,35 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates the five header decomposition tables (`header_name_refs`,
+ * `header_value_refs`, `header_sets`, `header_set_entries`,
+ * `header_flags`) from every `pages.responseHeaders` and
+ * `resources.responseHeaders` JSON blob (issue #191).
+ *
+ * Strategy:
+ *
+ * 1. **Warm caches** via {@link ./create-header-table-caches.ts} — load
+ *    existing `header_name_refs`, `header_value_refs`, and `header_sets`
+ *    rows into id maps. Idempotent re-runs then reuse ids instead of
+ *    re-issuing an upsert per entry, and new ids are appended as they
+ *    are inserted.
+ * 2. **Stream `pages.responseHeaders` + `resources.responseHeaders`** in
+ *    id-keyset chunks. Each non-null value is decomposed via
+ *    {@link decomposeHeaderSet} and written via
+ *    {@link ./upsert-one-header-set.ts} — the same per-set primitive the
+ *    crawler's live write path uses, so migrated and live-crawled
+ *    archives produce identical header rows.
+ *
+ * Every write is bulk-batched so the migration stays O(chunks × per-
+ * chunk-distinct-decomposed-sets) round-trips against the DB. Idempotency
+ * across full re-runs is guaranteed by the `INSERT OR IGNORE`s and the
+ * `setIdsProcessedThisRun` guard inside the per-set upsert.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateHeaderTables(trx);
+ * });
+ */
+export declare function populateHeaderTables(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-ref-tables/populate-header-tables.js

@@ -0,0 +1,80 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { createHeaderTableCaches } from './create-header-table-caches.js';
+import { decomposeHeaderSet } from './decompose-header-set.js';
+import { upsertOneHeaderSet } from './upsert-one-header-set.js';
+/**
+ * Rows scanned per SELECT chunk against `pages.responseHeaders` /
+ * `resources.responseHeaders`. Keyset-paginated on `id` to avoid loading
+ * a 470k-row `responseHeaders` column into a single query result.
+ */
+const READ_CHUNK_SIZE = 500;
+/**
+ * Populates the five header decomposition tables (`header_name_refs`,
+ * `header_value_refs`, `header_sets`, `header_set_entries`,
+ * `header_flags`) from every `pages.responseHeaders` and
+ * `resources.responseHeaders` JSON blob (issue #191).
+ *
+ * Strategy:
+ *
+ * 1. **Warm caches** via {@link ./create-header-table-caches.ts} — load
+ *    existing `header_name_refs`, `header_value_refs`, and `header_sets`
+ *    rows into id maps. Idempotent re-runs then reuse ids instead of
+ *    re-issuing an upsert per entry, and new ids are appended as they
+ *    are inserted.
+ * 2. **Stream `pages.responseHeaders` + `resources.responseHeaders`** in
+ *    id-keyset chunks. Each non-null value is decomposed via
+ *    {@link decomposeHeaderSet} and written via
+ *    {@link ./upsert-one-header-set.ts} — the same per-set primitive the
+ *    crawler's live write path uses, so migrated and live-crawled
+ *    archives produce identical header rows.
+ *
+ * Every write is bulk-batched so the migration stays O(chunks × per-
+ * chunk-distinct-decomposed-sets) round-trips against the DB. Idempotency
+ * across full re-runs is guaranteed by the `INSERT OR IGNORE`s and the
+ * `setIdsProcessedThisRun` guard inside the per-set upsert.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateHeaderTables(trx);
+ * });
+ */
+export async function populateHeaderTables(trx, onProgress) {
+    const caches = await createHeaderTableCaches(trx);
+    for (const table of ['pages', 'resources']) {
+        const hasTable = await trx.schema.hasTable(table);
+        if (!hasTable) {
+            continue;
+        }
+        const hasColumn = await trx.schema.hasColumn(table, 'responseHeaders');
+        if (!hasColumn) {
+            continue;
+        }
+        const countRows = await trx(table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`header_tables (${table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(table)
+                .select('id', 'responseHeaders')
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                const decomposed = decomposeHeaderSet(row.responseHeaders);
+                if (decomposed === null) {
+                    continue;
+                }
+                await upsertOneHeaderSet(trx, decomposed, caches);
+            }
+        }
+    }
+}

package/lib/archive/populate-ref-tables/populate-json-refs.d.ts

@@ -0,0 +1,29 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `json_refs` from every non-null `pages.meta_extras` value
+ * (issue #191).
+ *
+ * Each value is:
+ *
+ * 1. Hashed with {@link computeContentHash} (32-byte SHA-256; see that
+ *    function's docs for the algorithm-choice rationale).
+ * 2. Compressed with `zstdCompressSync` (`codec='zstd'`) — same encoder
+ *    used by `page_html_blobs`.
+ * 3. Deduplicated by hash — identical raw JSON strings produce one row.
+ *
+ * `size_raw` records the uncompressed byte length (UTF-8) and
+ * `size_stored` records the compressed byte length; consumers can
+ * estimate compression savings without decompressing.
+ *
+ * `INSERT OR IGNORE` on `hash` makes the step idempotent across partial-
+ * failure restarts.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateJsonRefs(trx);
+ * });
+ */
+export declare function populateJsonRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-ref-tables/populate-json-refs.js

@@ -0,0 +1,101 @@
+import { zstdCompressSync } from 'node:zlib';
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { computeContentHash } from './compute-content-hash.js';
+/**
+ * Number of `json_refs` rows sent per INSERT. Each row binds 5 params
+ * (hash + json_text + codec + size_raw + size_stored). 200 rows = 1000
+ * params — comfortable margin under SQLite's default variable limit and
+ * bounds any one INSERT's compressed-blob payload total to a size that
+ * fits in one WAL frame.
+ */
+const INSERT_CHUNK_SIZE = 200;
+/**
+ * Rows scanned per source query. Same rationale as the other populators:
+ * streaming with keyset pagination on `id` avoids loading a 470k-row
+ * `meta_extras` column into a single result set.
+ */
+const READ_CHUNK_SIZE = 1000;
+/**
+ * Populates `json_refs` from every non-null `pages.meta_extras` value
+ * (issue #191).
+ *
+ * Each value is:
+ *
+ * 1. Hashed with {@link computeContentHash} (32-byte SHA-256; see that
+ *    function's docs for the algorithm-choice rationale).
+ * 2. Compressed with `zstdCompressSync` (`codec='zstd'`) — same encoder
+ *    used by `page_html_blobs`.
+ * 3. Deduplicated by hash — identical raw JSON strings produce one row.
+ *
+ * `size_raw` records the uncompressed byte length (UTF-8) and
+ * `size_stored` records the compressed byte length; consumers can
+ * estimate compression savings without decompressing.
+ *
+ * `INSERT OR IGNORE` on `hash` makes the step idempotent across partial-
+ * failure restarts.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of `pages` scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateJsonRefs(trx);
+ * });
+ */
+export async function populateJsonRefs(trx, onProgress) {
+    const hasPages = await trx.schema.hasTable('pages');
+    if (!hasPages) {
+        return;
+    }
+    const hasMetaExtras = await trx.schema.hasColumn('pages', 'meta_extras');
+    if (!hasMetaExtras) {
+        return;
+    }
+    const seen = new Set();
+    const pending = [];
+    const countRows = await trx('pages').count({ n: '*' });
+    const total = Number(countRows[0]?.n ?? 0);
+    const report = createProgressReporter('json_refs (pages)', total, onProgress);
+    let processed = 0;
+    let cursor = 0;
+    while (true) {
+        const rows = await trx('pages')
+            .select('id', 'meta_extras as value')
+            .where('id', '>', cursor)
+            .orderBy('id', 'asc')
+            .limit(READ_CHUNK_SIZE);
+        if (rows.length === 0) {
+            break;
+        }
+        cursor = rows.at(-1).id;
+        processed += rows.length;
+        report(processed);
+        for (const row of rows) {
+            const raw = row.value;
+            if (raw == null || raw === '') {
+                continue;
+            }
+            const rawBytes = Buffer.from(raw, 'utf8');
+            const hash = computeContentHash(rawBytes);
+            const hex = hash.toString('hex');
+            if (seen.has(hex)) {
+                continue;
+            }
+            seen.add(hex);
+            const compressed = zstdCompressSync(rawBytes);
+            pending.push({
+                hash,
+                json_text: compressed,
+                codec: 'zstd',
+                size_raw: rawBytes.byteLength,
+                size_stored: compressed.byteLength,
+            });
+            if (pending.length >= INSERT_CHUNK_SIZE) {
+                await trx('json_refs').insert(pending).onConflict('hash').ignore();
+                pending.length = 0;
+            }
+        }
+    }
+    if (pending.length > 0) {
+        await trx('json_refs').insert(pending).onConflict('hash').ignore();
+    }
+}

package/lib/archive/populate-ref-tables/populate-refs.d.ts

@@ -0,0 +1,51 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Runs the six 0.13 ref-table populates (issue #191) in this fixed
+ * order against an already-connected archive:
+ *
+ * 1. `content_type_refs`
+ * 2. `url_refs`
+ * 3. `text_refs`
+ * 4. `json_refs`
+ * 5. `blob_refs`
+ * 6. Header tables (name / value / set / entries / flags).
+ *
+ * The order is an execution-order invariant, not arbitrary: the entity
+ * populates that run afterwards (`populateEntityTables`) JOIN against
+ * `content_type_refs` when filling `content_items` / `resource_items`,
+ * and look up `url_refs.id` for every URL column, so both dictionaries
+ * must be complete first. Within this function the ordering is:
+ * dictionary tables (1–5) first, header tables (6) last — the header
+ * tables' entries reference name / value refs, but those are inserted
+ * by `populateHeaderTables` itself, so no barrier between sub-populates
+ * is required.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE`
+ * on its natural key. Running this orchestrator twice on the same archive
+ * produces the same rows — no phase marker table is used. This function
+ * does not open its own transaction so the caller controls the boundary;
+ * the whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level.
+ *
+ * `scripts/migrate-to-0.13.mjs` does NOT call this orchestrator directly
+ * — it calls the same six sub-populates itself, each in its own
+ * transaction, so a killed multi-hour migration only re-does the one
+ * table in flight on resume rather than the whole six-table batch. Use
+ * this function directly when that per-table checkpointing is not
+ * needed and a single all-or-nothing transaction is preferred.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * // Single-transaction use: connect via Archive, wrap in a
+ * // transaction, run every sub-step, then rely on the caller's `.bak`
+ * // safety net if any step throws.
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateRefTables(trx);
+ * });
+ * await archive.write();
+ */
+export declare function populateRefTables(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-ref-tables/populate-refs.js

@@ -0,0 +1,62 @@
+import { populateBlobRefs } from './populate-blob-refs.js';
+import { populateContentTypeRefs } from './populate-content-type-refs.js';
+import { populateHeaderTables } from './populate-header-tables.js';
+import { populateJsonRefs } from './populate-json-refs.js';
+import { populateTextRefs } from './populate-text-refs.js';
+import { populateUrlRefs } from './populate-url-refs.js';
+/**
+ * Runs the six 0.13 ref-table populates (issue #191) in this fixed
+ * order against an already-connected archive:
+ *
+ * 1. `content_type_refs`
+ * 2. `url_refs`
+ * 3. `text_refs`
+ * 4. `json_refs`
+ * 5. `blob_refs`
+ * 6. Header tables (name / value / set / entries / flags).
+ *
+ * The order is an execution-order invariant, not arbitrary: the entity
+ * populates that run afterwards (`populateEntityTables`) JOIN against
+ * `content_type_refs` when filling `content_items` / `resource_items`,
+ * and look up `url_refs.id` for every URL column, so both dictionaries
+ * must be complete first. Within this function the ordering is:
+ * dictionary tables (1–5) first, header tables (6) last — the header
+ * tables' entries reference name / value refs, but those are inserted
+ * by `populateHeaderTables` itself, so no barrier between sub-populates
+ * is required.
+ *
+ * Every sub-populate is independently idempotent via `INSERT OR IGNORE`
+ * on its natural key. Running this orchestrator twice on the same archive
+ * produces the same rows — no phase marker table is used. This function
+ * does not open its own transaction so the caller controls the boundary;
+ * the whole invocation is expected to run inside one writer transaction
+ * with `.bak` protection at the caller level.
+ *
+ * `scripts/migrate-to-0.13.mjs` does NOT call this orchestrator directly
+ * — it calls the same six sub-populates itself, each in its own
+ * transaction, so a killed multi-hour migration only re-does the one
+ * table in flight on resume rather than the whole six-table batch. Use
+ * this function directly when that per-table checkpointing is not
+ * needed and a single all-or-nothing transaction is preferred.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink threaded to every sub-populate for
+ *   periodic progress lines; see {@link ../create-progress-reporter.ts}.
+ * @example
+ * // Single-transaction use: connect via Archive, wrap in a
+ * // transaction, run every sub-step, then rely on the caller's `.bak`
+ * // safety net if any step throws.
+ * const archive = await Archive.open(archivePath);
+ * const knex = archive.getKnex();
+ * await knex.transaction(async (trx) => {
+ *   await populateRefTables(trx);
+ * });
+ * await archive.write();
+ */
+export async function populateRefTables(trx, onProgress) {
+    await populateContentTypeRefs(trx);
+    await populateUrlRefs(trx, onProgress);
+    await populateTextRefs(trx, onProgress);
+    await populateJsonRefs(trx, onProgress);
+    await populateBlobRefs(trx, onProgress);
+    await populateHeaderTables(trx, onProgress);
+}

package/lib/archive/populate-ref-tables/populate-text-refs.d.ts

@@ -0,0 +1,32 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `text_refs` from every text-shaped column across `anchors`,
+ * `images`, and `pages` (issue #191).
+ *
+ * Rules:
+ *
+ * - **Content-hash dedup contract**: identical text produces one row. The
+ *   hash is derived by {@link computeContentHash} (currently SHA-256,
+ *   matching `page_html_blobs.hash`; see that function's docs for the
+ *   algorithm-choice rationale).
+ * - **Empty and null are skipped** — an empty `<a>` tag adds no text to
+ *   the dictionary. Anchors with no text still get an `anchor_edges` row
+ *   in 0.13, just with `first_text_id = NULL`.
+ * - **INSERT OR IGNORE on (hash, text)** — the composite UNIQUE from the
+ *   0.13 DDL — makes the step idempotent across partial-failure
+ *   restarts.
+ *
+ * Peak memory is bounded by the count of distinct texts (all texts are
+ * held in a Map before insert). On the reference archive this Map is
+ * about 5 MB of anchor textContent + a few MB of page meta text — well
+ * within the migration process budget.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateTextRefs(trx);
+ * });
+ */
+export declare function populateTextRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;

package/lib/archive/populate-ref-tables/populate-text-refs.js

@@ -0,0 +1,133 @@
+import { createProgressReporter } from '../create-progress-reporter.js';
+import { computeContentHash } from './compute-content-hash.js';
+/**
+ * Number of `text_refs` rows sent per `INSERT ... VALUES (...)` statement.
+ * Each row binds 2 params (hash + text), so 500 rows = 1000 params — well
+ * under SQLite's default `SQLITE_MAX_VARIABLE_NUMBER`.
+ */
+const INSERT_CHUNK_SIZE = 500;
+/**
+ * Rows scanned per keyset-paginated `SELECT` chunk. One SELECT reads all
+ * text columns for the source table's chunk simultaneously so the pages
+ * table is scanned exactly once per populate call, not N-times-once-per-
+ * text-column.
+ */
+const READ_CHUNK_SIZE = 5000;
+/**
+ * Source table + its text-shaped columns. Groups by table so each table
+ * is scanned exactly once with a single SELECT that covers every
+ * relevant column — a per-column scan would run 8 separate `pages`
+ * scans and multiply migration wall-clock accordingly.
+ *
+ * `dom_path` is intentionally absent: dom_path strings are derived from
+ * the stored HTML blob, not from any column of the current write-model,
+ * so this scan has nothing to read. Upserting dom_path strings into
+ * `text_refs` is the job of the image-items populate
+ * (`populate-entity-tables/populate-image-items.ts`), not this pass.
+ */
+const TEXT_SOURCES = [
+    { table: 'anchors', columns: ['textContent'] },
+    { table: 'images', columns: ['alt'] },
+    {
+        table: 'pages',
+        columns: [
+            'title',
+            'description',
+            'keywords',
+            'robots_raw',
+            'og_title',
+            'og_description',
+            'twitter_title',
+            'twitter_description',
+        ],
+    },
+];
+/**
+ * Populates `text_refs` from every text-shaped column across `anchors`,
+ * `images`, and `pages` (issue #191).
+ *
+ * Rules:
+ *
+ * - **Content-hash dedup contract**: identical text produces one row. The
+ *   hash is derived by {@link computeContentHash} (currently SHA-256,
+ *   matching `page_html_blobs.hash`; see that function's docs for the
+ *   algorithm-choice rationale).
+ * - **Empty and null are skipped** — an empty `<a>` tag adds no text to
+ *   the dictionary. Anchors with no text still get an `anchor_edges` row
+ *   in 0.13, just with `first_text_id = NULL`.
+ * - **INSERT OR IGNORE on (hash, text)** — the composite UNIQUE from the
+ *   0.13 DDL — makes the step idempotent across partial-failure
+ *   restarts.
+ *
+ * Peak memory is bounded by the count of distinct texts (all texts are
+ * held in a Map before insert). On the reference archive this Map is
+ * about 5 MB of anchor textContent + a few MB of page meta text — well
+ * within the migration process budget.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateTextRefs(trx);
+ * });
+ */
+export async function populateTextRefs(trx, onProgress) {
+    const seen = new Map();
+    for (const source of TEXT_SOURCES) {
+        const hasTable = await trx.schema.hasTable(source.table);
+        if (!hasTable) {
+            continue;
+        }
+        const presentColumns = [];
+        for (const column of source.columns) {
+            if (await trx.schema.hasColumn(source.table, column)) {
+                presentColumns.push(column);
+            }
+        }
+        if (presentColumns.length === 0) {
+            continue;
+        }
+        const countRows = await trx(source.table).count({ n: '*' });
+        const total = Number(countRows[0]?.n ?? 0);
+        const report = createProgressReporter(`text_refs (${source.table})`, total, onProgress);
+        let processed = 0;
+        let cursor = 0;
+        while (true) {
+            const rows = await trx(source.table)
+                .select('id', ...presentColumns)
+                .where('id', '>', cursor)
+                .orderBy('id', 'asc')
+                .limit(READ_CHUNK_SIZE);
+            if (rows.length === 0) {
+                break;
+            }
+            cursor = rows.at(-1).id;
+            processed += rows.length;
+            report(processed);
+            for (const row of rows) {
+                for (const column of presentColumns) {
+                    const value = row[column];
+                    if (typeof value !== 'string' || value === '') {
+                        continue;
+                    }
+                    const hash = computeContentHash(value);
+                    const key = hash.toString('hex');
+                    if (!seen.has(key)) {
+                        seen.set(key, value);
+                    }
+                }
+            }
+        }
+    }
+    if (seen.size === 0) {
+        return;
+    }
+    const inserts = [];
+    for (const [hex, text] of seen) {
+        inserts.push({ hash: Buffer.from(hex, 'hex'), text });
+    }
+    for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) {
+        const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE);
+        await trx('text_refs').insert(chunk).onConflict(['hash', 'text']).ignore();
+    }
+}

package/lib/archive/populate-ref-tables/populate-url-refs.d.ts

@@ -0,0 +1,28 @@
+import type { ProgressCallback } from '../create-progress-reporter.js';
+import type { Knex } from 'knex';
+/**
+ * Populates `url_refs` from every URL-shaped column across `pages`,
+ * `resources`, and `images` (issue #191).
+ *
+ * Runs two passes per source table:
+ *
+ * 1. **Collect distinct URLs** into an in-process `Set<string>`, streaming
+ *    rows from each source table by keyset pagination on `id`. All the
+ *    table's URL columns are read in a single SELECT so the pages scan
+ *    is O(rows), not O(rows × columns). Peak memory is bounded by the
+ *    count of distinct URLs, not the row count.
+ * 2. **Bulk-insert** into `url_refs` in chunks of {@link INSERT_CHUNK_SIZE},
+ *    with the decomposed columns (`scheme` / `host` / `port` / `path` /
+ *    `query_hash` / `fragment`) derived in JS by {@link decomposeUrl}.
+ *
+ * `INSERT OR IGNORE` on `url_refs.url` makes the step idempotent —
+ * repeated invocations after partial failure only add new URLs.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param onProgress - Optional sink for periodic progress lines (one per
+ *   ~5% of each source table scanned); see {@link ../create-progress-reporter.ts}.
+ * @example
+ * await knex.transaction(async (trx) => {
+ *   await populateUrlRefs(trx);
+ * });
+ */
+export declare function populateUrlRefs(trx: Knex, onProgress?: ProgressCallback): Promise<void>;