@nitpicker/archive 0.0.0 → 0.25.0

package/lib/transfer/copy-dictionaries-for-split.js

@@ -0,0 +1,224 @@
+import { PAGE_META_COLUMN_MAPS } from '../page-meta-column-maps.js';
+import { DICTIONARY_COPY_SPECS } from './dictionary-copy-specs.js';
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+import { TransferIntegrityError } from './types.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+const FULL = TRANSFER_ACTION.full;
+/**
+ * Collects dictionary ids referenced directly by `content_items` /
+ * `resource_items` rows the plan keeps — every plan action for
+ * `content_items` (both `full` and `stub`; a stub still carries its own
+ * `url_id`/`content_type_id`/`header_set_id`, the same fields a genuine
+ * external HEAD-only observation would record) and every kept
+ * (`full`-only, per `plan-resource-items.ts`'s split branch) resource.
+ * @param trx - Transaction with the source ATTACHed.
+ */
+async function collectContentItemLevelNeeds(trx) {
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_url_refs (src_id)
+		SELECT s.url_id FROM ${SRC}.content_items s JOIN xfer_ci_plan p ON p.src_id = s.id
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_content_type_refs (src_id)
+		SELECT s.content_type_id FROM ${SRC}.content_items s JOIN xfer_ci_plan p ON p.src_id = s.id
+		WHERE s.content_type_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_header_sets (src_id)
+		SELECT s.header_set_id FROM ${SRC}.content_items s JOIN xfer_ci_plan p ON p.src_id = s.id
+		WHERE s.header_set_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_url_refs (src_id)
+		SELECT r.url_id FROM ${SRC}.resource_items r JOIN xfer_ri_plan rp ON rp.src_id = r.id
+		WHERE r.url_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_blob_refs (src_id)
+		SELECT r.url_blob_id FROM ${SRC}.resource_items r JOIN xfer_ri_plan rp ON rp.src_id = r.id
+		WHERE r.url_blob_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_content_type_refs (src_id)
+		SELECT r.content_type_id FROM ${SRC}.resource_items r JOIN xfer_ri_plan rp ON rp.src_id = r.id
+		WHERE r.content_type_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_header_sets (src_id)
+		SELECT r.header_set_id FROM ${SRC}.resource_items r JOIN xfer_ri_plan rp ON rp.src_id = r.id
+		WHERE r.header_set_id IS NOT NULL
+	`);
+}
+/**
+ * Collects dictionary ids referenced by page-scoped data (`page_meta`,
+ * `anchor_edges`, `image_items`) — only from `full`
+ * pages, since `stub` rows never get any of these copied
+ * (`copy-page-meta.ts` et al. only ever read `full`-action rows).
+ * @param trx - Transaction with the source ATTACHed.
+ */
+async function collectPageScopedNeeds(trx) {
+    for (const { target } of PAGE_META_COLUMN_MAPS.url) {
+        await trx.raw(`
+			INSERT OR IGNORE INTO xfer_need_url_refs (src_id)
+			SELECT pm.${target} FROM ${SRC}.page_meta pm
+			JOIN xfer_ci_plan p ON p.src_id = pm.page_id AND p.action = ${FULL}
+			WHERE pm.${target} IS NOT NULL
+		`);
+    }
+    for (const { target } of PAGE_META_COLUMN_MAPS.text) {
+        await trx.raw(`
+			INSERT OR IGNORE INTO xfer_need_text_refs (src_id)
+			SELECT pm.${target} FROM ${SRC}.page_meta pm
+			JOIN xfer_ci_plan p ON p.src_id = pm.page_id AND p.action = ${FULL}
+			WHERE pm.${target} IS NOT NULL
+		`);
+    }
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_json_refs (src_id)
+		SELECT pm.meta_extras_json_id FROM ${SRC}.page_meta pm
+		JOIN xfer_ci_plan p ON p.src_id = pm.page_id AND p.action = ${FULL}
+		WHERE pm.meta_extras_json_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_text_refs (src_id)
+		SELECT ae.first_text_id FROM ${SRC}.anchor_edges ae
+		JOIN xfer_ci_plan p ON p.src_id = ae.page_id AND p.action = ${FULL}
+		WHERE ae.first_text_id IS NOT NULL
+	`);
+    for (const column of ['src_url_id', 'current_src_url_id']) {
+        await trx.raw(`
+			INSERT OR IGNORE INTO xfer_need_url_refs (src_id)
+			SELECT ii.${column} FROM ${SRC}.image_items ii
+			JOIN xfer_ci_plan p ON p.src_id = ii.page_id AND p.action = ${FULL}
+			WHERE ii.${column} IS NOT NULL
+		`);
+    }
+    for (const column of ['src_blob_id', 'current_src_blob_id']) {
+        await trx.raw(`
+			INSERT OR IGNORE INTO xfer_need_blob_refs (src_id)
+			SELECT ii.${column} FROM ${SRC}.image_items ii
+			JOIN xfer_ci_plan p ON p.src_id = ii.page_id AND p.action = ${FULL}
+			WHERE ii.${column} IS NOT NULL
+		`);
+    }
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_text_refs (src_id)
+		SELECT ii.alt_text_id FROM ${SRC}.image_items ii
+		JOIN xfer_ci_plan p ON p.src_id = ii.page_id AND p.action = ${FULL}
+		WHERE ii.alt_text_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_text_refs (src_id)
+		SELECT ii.dom_path_text_id FROM ${SRC}.image_items ii
+		JOIN xfer_ci_plan p ON p.src_id = ii.page_id AND p.action = ${FULL}
+	`);
+}
+/**
+ * Collects `console_log_items` referenced by kept pages' `page_console_logs`,
+ * then that dictionary's OWN url/text/json needs — a second-order need
+ * only `console_log_items` has, since it is itself a dictionary that
+ * points into three other dictionaries.
+ * @param trx - Transaction with the source ATTACHed.
+ */
+async function collectConsoleLogNeeds(trx) {
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_console_log_items (src_id)
+		SELECT pcl.consoleLogId FROM ${SRC}.page_console_logs pcl
+		JOIN xfer_ci_plan p ON p.src_id = pcl.pageId AND p.action = ${FULL}
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_url_refs (src_id)
+		SELECT cli.loc_url_id FROM ${SRC}.console_log_items cli
+		JOIN xfer_need_console_log_items n ON n.src_id = cli.id
+		WHERE cli.loc_url_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_text_refs (src_id)
+		SELECT cli.text_id FROM ${SRC}.console_log_items cli
+		JOIN xfer_need_console_log_items n ON n.src_id = cli.id
+		WHERE cli.text_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_text_refs (src_id)
+		SELECT cli.stack_text_id FROM ${SRC}.console_log_items cli
+		JOIN xfer_need_console_log_items n ON n.src_id = cli.id
+		WHERE cli.stack_text_id IS NOT NULL
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_need_json_refs (src_id)
+		SELECT cli.args_json_id FROM ${SRC}.console_log_items cli
+		JOIN xfer_need_console_log_items n ON n.src_id = cli.id
+		WHERE cli.args_json_id IS NOT NULL
+	`);
+}
+/**
+ * Copies only the dictionary rows referenced by kept content — the split
+ * counterpart of `copy-dictionaries-for-concat.ts`'s unconditional
+ * copy-everything. Split's entire purpose is size reduction, so (unlike
+ * concat, where computing a precise needed-set is not worth the
+ * complexity) a dropped page's dictionary rows must not survive into the
+ * output archive.
+ *
+ * Runs in two phases: collect every needed dictionary id into
+ * `temp.xfer_need_<table>` (this function's own private helpers, in
+ * dependency order — `console_log_items`' own needs must be collected
+ * AFTER `console_log_items` itself, since they derive from it), then copy
+ * + map each dictionary table exactly like concat but filtered to the
+ * needed set.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `temp.xfer_ci_plan` AND `temp.xfer_ri_plan` are fully populated.
+ * @throws {import('./types.js').TransferIntegrityError} If any table's
+ *   mapped row count does not equal its needed-row count after the
+ *   natural-key JOIN.
+ * @example
+ * ```ts
+ * await planContentItemsForSplit(trx);
+ * await planResourceItems(trx, 'split');
+ * await copyDictionariesForSplit(trx);
+ * ```
+ */
+export async function copyDictionariesForSplit(trx) {
+    await collectContentItemLevelNeeds(trx);
+    await collectPageScopedNeeds(trx);
+    await collectConsoleLogNeeds(trx);
+    for (const spec of DICTIONARY_COPY_SPECS) {
+        const columns = await listTransferColumns(trx, SRC, spec.table, ['id']);
+        const quotedColumns = columns.map(quoteTransferIdentifier);
+        const columnList = quotedColumns.join(', ');
+        const conflictTarget = spec.naturalKey.map(quoteTransferIdentifier).join(', ');
+        await trx.raw(`
+			INSERT INTO main.${spec.table} (${columnList})
+			SELECT ${columnList} FROM ${SRC}.${spec.table} s
+			JOIN xfer_need_${spec.table} n ON n.src_id = s.id
+			WHERE true
+			ON CONFLICT(${conflictTarget}) DO NOTHING
+		`);
+        const joinCondition = spec.naturalKey
+            .map((col) => `d.${quoteTransferIdentifier(col)} IS s.${quoteTransferIdentifier(col)}`)
+            .join(' AND ');
+        await trx.raw(`
+			INSERT INTO xfer_map_${spec.table} (src_id, dest_id)
+			SELECT s.id, d.id
+			FROM ${SRC}.${spec.table} s
+			JOIN xfer_need_${spec.table} n ON n.src_id = s.id
+			JOIN main.${spec.table} d ON ${joinCondition}
+		`);
+        // Every NEEDED source row must resolve to a destination id (unlike
+        // concat, split's needed-set can legitimately be a strict subset of
+        // the source table — that is the whole point). A gap between
+        // `xfer_need_<table>` and `xfer_map_<table>` means the natural-key
+        // JOIN failed to match a row this same function just inserted,
+        // which should be impossible; treat it as a hard integrity failure.
+        const neededCountRows = await trx(`xfer_need_${spec.table}`).count({ neededCount: '*' });
+        const mappedCountRows = await trx(`xfer_map_${spec.table}`).count({ mappedCount: '*' });
+        const neededCount = neededCountRows[0]?.neededCount ?? 0;
+        const mappedCount = mappedCountRows[0]?.mappedCount ?? 0;
+        if (Number(mappedCount) !== Number(neededCount)) {
+            throw new TransferIntegrityError(`copyDictionariesForSplit: ${spec.table} needs ${neededCount} row(s) but only ${mappedCount} mapped to a destination id`);
+        }
+    }
+}

package/lib/transfer/copy-header-set-children.d.ts

@@ -0,0 +1,27 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `header_set_entries` and `header_flags` — the two children of
+ * `header_sets` that have no id of their own to map (`header_set_entries`
+ * is `WITHOUT ROWID` with a composite PK; `header_flags`' PK IS
+ * `header_set_id`), so both are remapped and copied directly rather than
+ * going through the generic `temp.xfer_map_<table>` machinery
+ * {@link import('./copy-dictionaries-for-concat.js').copyDictionariesForConcat} /
+ * {@link import('./copy-dictionaries-for-split.js').copyDictionariesForSplit}
+ * use for id-keyed dictionaries.
+ *
+ * Both are filtered implicitly: only `header_sets` rows that were
+ * actually copied have an entry in `temp.xfer_map_header_sets`, so a
+ * source header set that split's need-computation skipped (belongs only
+ * to a dropped page) naturally contributes no entries/flags row either —
+ * no separate need-set for these two tables is required.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after `temp.xfer_map_header_sets`,
+ *   `temp.xfer_map_header_name_refs`, and `temp.xfer_map_header_value_refs`
+ *   are populated — i.e. after `copyDictionariesForConcat`/`copyDictionariesForSplit`.
+ * @example
+ * ```ts
+ * await copyDictionariesForConcat(trx); // or copyDictionariesForSplit
+ * await copyHeaderSetChildren(trx);
+ * ```
+ */
+export declare function copyHeaderSetChildren(trx: Knex): Promise<void>;

package/lib/transfer/copy-header-set-children.js

@@ -0,0 +1,51 @@
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Copies `header_set_entries` and `header_flags` — the two children of
+ * `header_sets` that have no id of their own to map (`header_set_entries`
+ * is `WITHOUT ROWID` with a composite PK; `header_flags`' PK IS
+ * `header_set_id`), so both are remapped and copied directly rather than
+ * going through the generic `temp.xfer_map_<table>` machinery
+ * {@link import('./copy-dictionaries-for-concat.js').copyDictionariesForConcat} /
+ * {@link import('./copy-dictionaries-for-split.js').copyDictionariesForSplit}
+ * use for id-keyed dictionaries.
+ *
+ * Both are filtered implicitly: only `header_sets` rows that were
+ * actually copied have an entry in `temp.xfer_map_header_sets`, so a
+ * source header set that split's need-computation skipped (belongs only
+ * to a dropped page) naturally contributes no entries/flags row either —
+ * no separate need-set for these two tables is required.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after `temp.xfer_map_header_sets`,
+ *   `temp.xfer_map_header_name_refs`, and `temp.xfer_map_header_value_refs`
+ *   are populated — i.e. after `copyDictionariesForConcat`/`copyDictionariesForSplit`.
+ * @example
+ * ```ts
+ * await copyDictionariesForConcat(trx); // or copyDictionariesForSplit
+ * await copyHeaderSetChildren(trx);
+ * ```
+ */
+export async function copyHeaderSetChildren(trx) {
+    await trx.raw(`
+		INSERT INTO main.header_set_entries (header_set_id, name_id, occurrence, value_id, is_volatile)
+		SELECT mhs.dest_id, mhn.dest_id, s.occurrence, mhv.dest_id, s.is_volatile
+		FROM ${TRANSFER_SOURCE_ALIAS}.header_set_entries s
+		JOIN xfer_map_header_sets mhs ON mhs.src_id = s.header_set_id
+		JOIN xfer_map_header_name_refs mhn ON mhn.src_id = s.name_id
+		JOIN xfer_map_header_value_refs mhv ON mhv.src_id = s.value_id
+		WHERE true
+		ON CONFLICT(header_set_id, name_id, occurrence) DO NOTHING
+	`);
+    const flagColumns = await listTransferColumns(trx, TRANSFER_SOURCE_ALIAS, 'header_flags', ['header_set_id']);
+    const quotedFlagColumns = flagColumns.map(quoteTransferIdentifier);
+    const selectList = quotedFlagColumns.map((col) => `s.${col}`).join(', ');
+    await trx.raw(`
+		INSERT INTO main.header_flags (header_set_id, ${quotedFlagColumns.join(', ')})
+		SELECT mhs.dest_id, ${selectList}
+		FROM ${TRANSFER_SOURCE_ALIAS}.header_flags s
+		JOIN xfer_map_header_sets mhs ON mhs.src_id = s.header_set_id
+		WHERE true
+		ON CONFLICT(header_set_id) DO NOTHING
+	`);
+}

package/lib/transfer/copy-image-items.d.ts

@@ -0,0 +1,21 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `image_items` for every `full`/`replace`-action page (never
+ * `stub`/`skip` — see `copy-page-meta.ts`'s docs for the same rule).
+ *
+ * Like `page_meta`, several columns are FK-shaped but each points at its
+ * own distinct dictionary row, so each gets a uniquely-aliased join
+ * rather than one shared alias. `dom_path_text_id` is `NOT NULL` in the
+ * schema (every image resolves to a DOM-position label, falling back to
+ * a synthetic `img[unknown]` text at write time — see
+ * `create-entity-tables.ts`), so an unmapped id here would surface
+ * immediately as a constraint violation rather than silently going missing.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copyImageItems(trx);
+ * ```
+ */
+export declare function copyImageItems(trx: Knex): Promise<void>;

package/lib/transfer/copy-image-items.js

@@ -0,0 +1,62 @@
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/** `image_items` columns that resolve through `temp.xfer_map_url_refs`. */
+const URL_COLUMNS = new Set(['src_url_id', 'current_src_url_id']);
+/** `image_items` columns that resolve through `temp.xfer_map_blob_refs`. */
+const BLOB_COLUMNS = new Set(['src_blob_id', 'current_src_blob_id']);
+/**
+ * Copies `image_items` for every `full`/`replace`-action page (never
+ * `stub`/`skip` — see `copy-page-meta.ts`'s docs for the same rule).
+ *
+ * Like `page_meta`, several columns are FK-shaped but each points at its
+ * own distinct dictionary row, so each gets a uniquely-aliased join
+ * rather than one shared alias. `dom_path_text_id` is `NOT NULL` in the
+ * schema (every image resolves to a DOM-position label, falling back to
+ * a synthetic `img[unknown]` text at write time — see
+ * `create-entity-tables.ts`), so an unmapped id here would surface
+ * immediately as a constraint violation rather than silently going missing.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copyImageItems(trx);
+ * ```
+ */
+export async function copyImageItems(trx) {
+    const columns = await listTransferColumns(trx, SRC, 'image_items', ['id', 'page_id']);
+    const joins = [];
+    const exprFor = (column) => {
+        const quotedColumn = quoteTransferIdentifier(column);
+        if (column === 'alt_text_id' || column === 'dom_path_text_id') {
+            const alias = `mt_${column}`;
+            joins.push(`LEFT JOIN xfer_map_text_refs ${alias} ON ${alias}.src_id = ii.${quotedColumn}`);
+            return `${alias}.dest_id`;
+        }
+        if (URL_COLUMNS.has(column)) {
+            const alias = `mu_${column}`;
+            joins.push(`LEFT JOIN xfer_map_url_refs ${alias} ON ${alias}.src_id = ii.${quotedColumn}`);
+            return `${alias}.dest_id`;
+        }
+        if (BLOB_COLUMNS.has(column)) {
+            const alias = `mb_${column}`;
+            joins.push(`LEFT JOIN xfer_map_blob_refs ${alias} ON ${alias}.src_id = ii.${quotedColumn}`);
+            return `${alias}.dest_id`;
+        }
+        return `ii.${quotedColumn}`;
+    };
+    const selectList = columns.map(exprFor).join(', ');
+    const joinList = joins.join('\n');
+    const quotedColumnList = columns.map(quoteTransferIdentifier).join(', ');
+    await trx.raw(`
+		INSERT INTO main.image_items (page_id, ${quotedColumnList})
+		SELECT p.dest_id, ${selectList}
+		FROM ${SRC}.image_items ii
+		JOIN xfer_ci_plan p ON p.src_id = ii.page_id
+			AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		${joinList}
+	`);
+}

package/lib/transfer/copy-inventory-source-lists.d.ts

@@ -0,0 +1,30 @@
+import type Archive from '../archive.js';
+/**
+ * Copies every `inventory/<sha256>.txt` source list from each source's
+ * extracted tmpDir into the destination archive, via
+ * `Archive#saveInventorySourceList` (the same write path `--inventory`/
+ * `--recrawl` themselves use) — a plain filesystem operation, not SQL, so
+ * this takes tmpDir paths and the destination `Archive`, not a `Knex`
+ * transaction.
+ *
+ * Filenames are content-addressed (the sha256 of the list's own bytes),
+ * so the same list appearing in two sources (an operator re-running
+ * `--inventory` with the same file against both, say) copies once — `seen`
+ * dedupes across sources, and `saveInventorySourceList` itself is a
+ * same-name overwrite (idempotent) even without that guard. Anything not
+ * matching {@link INVENTORY_FILE_NAME_PATTERN} — including a source whose
+ * `inventory/` directory does not exist at all, the common case — is
+ * skipped rather than copied verbatim, so an unrelated stray file
+ * (there should never be one, but this function does not trust that)
+ * cannot be smuggled into the output archive.
+ * @param destination - The (writer) destination archive, opened with
+ *   `openPluginData: true` so `write()` will re-tar this directory.
+ * @param sourceTmpDirs - Each source's extracted tmpDir
+ *   (`accessor.tmpDir`), in argument order.
+ * @returns The number of distinct source lists copied.
+ * @example
+ * ```ts
+ * const copied = await copyInventorySourceLists(destination, sources.map((s) => s.tmpDir));
+ * ```
+ */
+export declare function copyInventorySourceLists(destination: Archive, sourceTmpDirs: readonly string[]): Promise<number>;

package/lib/transfer/copy-inventory-source-lists.js

@@ -0,0 +1,61 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+/** Matches `Archive.saveInventorySourceList`'s own naming contract: `<sha256 hex>.txt`. */
+const INVENTORY_FILE_NAME_PATTERN = /^[0-9a-f]{64}\.txt$/;
+/**
+ * Copies every `inventory/<sha256>.txt` source list from each source's
+ * extracted tmpDir into the destination archive, via
+ * `Archive#saveInventorySourceList` (the same write path `--inventory`/
+ * `--recrawl` themselves use) — a plain filesystem operation, not SQL, so
+ * this takes tmpDir paths and the destination `Archive`, not a `Knex`
+ * transaction.
+ *
+ * Filenames are content-addressed (the sha256 of the list's own bytes),
+ * so the same list appearing in two sources (an operator re-running
+ * `--inventory` with the same file against both, say) copies once — `seen`
+ * dedupes across sources, and `saveInventorySourceList` itself is a
+ * same-name overwrite (idempotent) even without that guard. Anything not
+ * matching {@link INVENTORY_FILE_NAME_PATTERN} — including a source whose
+ * `inventory/` directory does not exist at all, the common case — is
+ * skipped rather than copied verbatim, so an unrelated stray file
+ * (there should never be one, but this function does not trust that)
+ * cannot be smuggled into the output archive.
+ * @param destination - The (writer) destination archive, opened with
+ *   `openPluginData: true` so `write()` will re-tar this directory.
+ * @param sourceTmpDirs - Each source's extracted tmpDir
+ *   (`accessor.tmpDir`), in argument order.
+ * @returns The number of distinct source lists copied.
+ * @example
+ * ```ts
+ * const copied = await copyInventorySourceLists(destination, sources.map((s) => s.tmpDir));
+ * ```
+ */
+export async function copyInventorySourceLists(destination, sourceTmpDirs) {
+    const seen = new Set();
+    for (const sourceTmpDir of sourceTmpDirs) {
+        const inventoryDir = path.join(sourceTmpDir, 'inventory');
+        let entries;
+        try {
+            entries = await fs.readdir(inventoryDir);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                continue;
+            }
+            throw error;
+        }
+        for (const entry of entries) {
+            if (!INVENTORY_FILE_NAME_PATTERN.test(entry)) {
+                continue;
+            }
+            const sha256 = entry.slice(0, -'.txt'.length);
+            if (seen.has(sha256)) {
+                continue;
+            }
+            seen.add(sha256);
+            const bytes = await fs.readFile(path.join(inventoryDir, entry));
+            await destination.saveInventorySourceList(sha256, bytes);
+        }
+    }
+    return seen.size;
+}

package/lib/transfer/copy-journal-tables.d.ts

@@ -0,0 +1,30 @@
+import type { Knex } from 'knex';
+/**
+ * Copies the four append-only journal tables with fresh (auto-assigned)
+ * ids: `list_reconcile_runs`, `network_outages`, `dedupe_cap_events`
+ * (none has an FK, so a plain `INSERT ... SELECT` of every non-`id`
+ * column suffices — no id remapping needed for these three), and
+ * `crawl_errors` (also no FK, but split additionally drops a row whose
+ * `url` is no longer present in the output's `url_refs`, so a dropped
+ * page's crawl-time errors do not linger in an archive that no longer
+ * contains it — concat keeps every `crawl_errors` row unconditionally,
+ * since nothing was dropped).
+ *
+ * `content_items.dedupe_cap_event_id` is deliberately NOT remapped here
+ * to point at the copied `dedupe_cap_events` rows — it is always written
+ * `NULL` by `insert-content-items.ts`/`replace-content-items.ts` and the
+ * mandatory viewer read-model rebuild's `backfillDedupeCapEventId` fully
+ * recomputes it afterwards by matching `content_items` rows against
+ * `dedupe_cap_events.shape_key` from scratch, so writing a remapped id
+ * here would only be immediately overwritten.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @param mode - `'split'` filters `crawl_errors` to rows whose `url` (if
+ *   any) survives in the destination's `url_refs`; `'concat'` copies every
+ *   row unconditionally.
+ * @example
+ * ```ts
+ * await copyJournalTables(trx, 'split');
+ * ```
+ */
+export declare function copyJournalTables(trx: Knex, mode: 'concat' | 'split'): Promise<void>;

package/lib/transfer/copy-journal-tables.js

@@ -0,0 +1,56 @@
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies the four append-only journal tables with fresh (auto-assigned)
+ * ids: `list_reconcile_runs`, `network_outages`, `dedupe_cap_events`
+ * (none has an FK, so a plain `INSERT ... SELECT` of every non-`id`
+ * column suffices — no id remapping needed for these three), and
+ * `crawl_errors` (also no FK, but split additionally drops a row whose
+ * `url` is no longer present in the output's `url_refs`, so a dropped
+ * page's crawl-time errors do not linger in an archive that no longer
+ * contains it — concat keeps every `crawl_errors` row unconditionally,
+ * since nothing was dropped).
+ *
+ * `content_items.dedupe_cap_event_id` is deliberately NOT remapped here
+ * to point at the copied `dedupe_cap_events` rows — it is always written
+ * `NULL` by `insert-content-items.ts`/`replace-content-items.ts` and the
+ * mandatory viewer read-model rebuild's `backfillDedupeCapEventId` fully
+ * recomputes it afterwards by matching `content_items` rows against
+ * `dedupe_cap_events.shape_key` from scratch, so writing a remapped id
+ * here would only be immediately overwritten.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @param mode - `'split'` filters `crawl_errors` to rows whose `url` (if
+ *   any) survives in the destination's `url_refs`; `'concat'` copies every
+ *   row unconditionally.
+ * @example
+ * ```ts
+ * await copyJournalTables(trx, 'split');
+ * ```
+ */
+export async function copyJournalTables(trx, mode) {
+    for (const table of ['list_reconcile_runs', 'network_outages', 'dedupe_cap_events']) {
+        const columns = await listTransferColumns(trx, SRC, table, ['id']);
+        const quoted = columns.map(quoteTransferIdentifier);
+        const columnList = quoted.map((col) => `s.${col}`).join(', ');
+        await trx.raw(`
+			INSERT INTO main.${table} (${quoted.join(', ')})
+			SELECT ${columnList} FROM ${SRC}.${table} s
+		`);
+    }
+    const crawlErrorColumns = await listTransferColumns(trx, SRC, 'crawl_errors', ['id']);
+    const quotedCrawlErrorColumns = crawlErrorColumns.map(quoteTransferIdentifier);
+    const crawlErrorSelectList = quotedCrawlErrorColumns
+        .map((col) => `s.${col}`)
+        .join(', ');
+    const urlSurvivesFilter = mode === 'split'
+        ? `WHERE s.url IS NULL OR EXISTS (SELECT 1 FROM main.url_refs u WHERE u.url = s.url)`
+        : '';
+    await trx.raw(`
+		INSERT INTO main.crawl_errors (${quotedCrawlErrorColumns.join(', ')})
+		SELECT ${crawlErrorSelectList} FROM ${SRC}.crawl_errors s
+		${urlSurvivesFilter}
+	`);
+}

package/lib/transfer/copy-page-console-logs.d.ts

@@ -0,0 +1,16 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `page_console_logs` (one row per page/console-log occurrence)
+ * for every `full`/`replace`-action page, remapping `consoleLogId`
+ * through `temp.xfer_map_console_log_items`.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `copyDictionariesForConcat`/`copyDictionariesForSplit` has populated
+ *   `temp.xfer_map_console_log_items`.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copyPageConsoleLogs(trx);
+ * ```
+ */
+export declare function copyPageConsoleLogs(trx: Knex): Promise<void>;

package/lib/transfer/copy-page-console-logs.js

@@ -0,0 +1,27 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies `page_console_logs` (one row per page/console-log occurrence)
+ * for every `full`/`replace`-action page, remapping `consoleLogId`
+ * through `temp.xfer_map_console_log_items`.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `copyDictionariesForConcat`/`copyDictionariesForSplit` has populated
+ *   `temp.xfer_map_console_log_items`.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copyPageConsoleLogs(trx);
+ * ```
+ */
+export async function copyPageConsoleLogs(trx) {
+    await trx.raw(`
+		INSERT INTO main.page_console_logs (pageId, consoleLogId, ts)
+		SELECT p.dest_id, m.dest_id, pcl.ts
+		FROM ${SRC}.page_console_logs pcl
+		JOIN xfer_ci_plan p ON p.src_id = pcl.pageId
+			AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		JOIN xfer_map_console_log_items m ON m.src_id = pcl.consoleLogId
+	`);
+}

package/lib/transfer/copy-page-html-blobs.d.ts

@@ -0,0 +1,31 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `page_html_blobs` rows referenced by kept pages' `page_html_ref`.
+ *
+ * `page_html_blobs.hash` is itself the content-addressed identity (the
+ * table's PK, computed the same way in every archive) — unlike every
+ * other dictionary in `archive/transfer/`, there is no separate integer id
+ * to remap: {@link import('./copy-simple-page-scoped-tables.js').copySimplePageScopedTables}'s
+ * `page_html_ref` copy carries the source's `hash` value straight through
+ * unchanged, and it already resolves in the destination once this
+ * function has run. No `temp.xfer_map_*` table exists for this dictionary.
+ *
+ * The filter (`action IN (full, replace)`) applies identically to both
+ * transfer modes: split's plan never assigns `replace`, so for split this
+ * is exactly "full pages only"; concat's plan never assigns `stub`, so for
+ * concat this is exactly "every page whose `page_html_ref` copy will
+ * actually run" (`stub` rows, split-only, never get a `page_html_ref`
+ * copied either way).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after `temp.xfer_ci_plan` is
+ *   fully populated, before `copy-simple-page-scoped-tables.ts`'s `page_html_ref` copy
+ *   (which assumes the blob it points at already exists — `page_html_ref`
+ *   has an FK on `hash`).
+ * @example
+ * ```ts
+ * await planContentItemsForSplit(trx);
+ * await copyPageHtmlBlobs(trx);
+ * await copySimplePageScopedTables(trx); // copies page_html_ref among other tables
+ * ```
+ */
+export declare function copyPageHtmlBlobs(trx: Knex): Promise<void>;