@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/transfer/copy-page-html-blobs.js

@@ -0,0 +1,43 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Copies `page_html_blobs` rows referenced by kept pages' `page_html_ref`.
+ *
+ * `page_html_blobs.hash` is itself the content-addressed identity (the
+ * table's PK, computed the same way in every archive) — unlike every
+ * other dictionary in `archive/transfer/`, there is no separate integer id
+ * to remap: {@link import('./copy-simple-page-scoped-tables.js').copySimplePageScopedTables}'s
+ * `page_html_ref` copy carries the source's `hash` value straight through
+ * unchanged, and it already resolves in the destination once this
+ * function has run. No `temp.xfer_map_*` table exists for this dictionary.
+ *
+ * The filter (`action IN (full, replace)`) applies identically to both
+ * transfer modes: split's plan never assigns `replace`, so for split this
+ * is exactly "full pages only"; concat's plan never assigns `stub`, so for
+ * concat this is exactly "every page whose `page_html_ref` copy will
+ * actually run" (`stub` rows, split-only, never get a `page_html_ref`
+ * copied either way).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after `temp.xfer_ci_plan` is
+ *   fully populated, before `copy-simple-page-scoped-tables.ts`'s `page_html_ref` copy
+ *   (which assumes the blob it points at already exists — `page_html_ref`
+ *   has an FK on `hash`).
+ * @example
+ * ```ts
+ * await planContentItemsForSplit(trx);
+ * await copyPageHtmlBlobs(trx);
+ * await copySimplePageScopedTables(trx); // copies page_html_ref among other tables
+ * ```
+ */
+export async function copyPageHtmlBlobs(trx) {
+    await trx.raw(`
+		INSERT INTO main.page_html_blobs (hash, body, codec, size_raw, size_stored)
+		SELECT DISTINCT b.hash, b.body, b.codec, b.size_raw, b.size_stored
+		FROM ${TRANSFER_SOURCE_ALIAS}.page_html_blobs b
+		JOIN ${TRANSFER_SOURCE_ALIAS}.page_html_ref ref ON ref.hash = b.hash
+		JOIN xfer_ci_plan p ON p.src_id = ref.page_id
+			AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		WHERE true
+		ON CONFLICT(hash) DO NOTHING
+	`);
+}

package/lib/transfer/copy-page-meta.d.ts

@@ -0,0 +1,27 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `page_meta` for every `full`/`replace`-action page (never
+ * `stub` — a stub carries no page-scoped data at all, per
+ * `transfer-action.ts`'s docs; never `skip`, whose destination row's
+ * existing `page_meta` is untouched).
+ *
+ * `page_meta` has TEN `url_id`-shaped columns and EIGHT `text_id`-shaped
+ * columns ({@link PAGE_META_COLUMN_MAPS}), each potentially pointing at a
+ * DIFFERENT dictionary row — unlike `content_items`' single `url_id`, one
+ * shared join alias cannot serve every column. Each FK column gets its own
+ * uniquely-aliased `LEFT JOIN` against the relevant `temp.xfer_map_*`
+ * table, built dynamically from {@link listTransferColumns} so a future
+ * `page_meta` column addition is picked up without touching this file
+ * (only `page-meta-column-maps.ts` needs updating for a new url/text
+ * column; a new plain column needs nothing here at all).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after every dictionary copy
+ *   AND after `insert-content-items.ts`/`replace-content-items.ts` have
+ *   resolved every kept page's `dest_id`.
+ * @example
+ * ```ts
+ * await insertContentItems(trx, { crawlOrderOffset: 0 });
+ * await copyPageMeta(trx);
+ * ```
+ */
+export declare function copyPageMeta(trx: Knex): Promise<void>;

package/lib/transfer/copy-page-meta.js

@@ -0,0 +1,68 @@
+import { PAGE_META_COLUMN_MAPS } from '../page-meta-column-maps.js';
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+const URL_COLUMNS = new Set(PAGE_META_COLUMN_MAPS.url.map((m) => m.target));
+const TEXT_COLUMNS = new Set(PAGE_META_COLUMN_MAPS.text.map((m) => m.target));
+/**
+ * Copies `page_meta` for every `full`/`replace`-action page (never
+ * `stub` — a stub carries no page-scoped data at all, per
+ * `transfer-action.ts`'s docs; never `skip`, whose destination row's
+ * existing `page_meta` is untouched).
+ *
+ * `page_meta` has TEN `url_id`-shaped columns and EIGHT `text_id`-shaped
+ * columns ({@link PAGE_META_COLUMN_MAPS}), each potentially pointing at a
+ * DIFFERENT dictionary row — unlike `content_items`' single `url_id`, one
+ * shared join alias cannot serve every column. Each FK column gets its own
+ * uniquely-aliased `LEFT JOIN` against the relevant `temp.xfer_map_*`
+ * table, built dynamically from {@link listTransferColumns} so a future
+ * `page_meta` column addition is picked up without touching this file
+ * (only `page-meta-column-maps.ts` needs updating for a new url/text
+ * column; a new plain column needs nothing here at all).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after every dictionary copy
+ *   AND after `insert-content-items.ts`/`replace-content-items.ts` have
+ *   resolved every kept page's `dest_id`.
+ * @example
+ * ```ts
+ * await insertContentItems(trx, { crawlOrderOffset: 0 });
+ * await copyPageMeta(trx);
+ * ```
+ */
+export async function copyPageMeta(trx) {
+    const columns = await listTransferColumns(trx, SRC, 'page_meta', ['page_id']);
+    const joins = [];
+    const exprFor = (column) => {
+        const quotedColumn = quoteTransferIdentifier(column);
+        if (URL_COLUMNS.has(column)) {
+            const alias = `mu_${column}`;
+            joins.push(`LEFT JOIN xfer_map_url_refs ${alias} ON ${alias}.src_id = pm.${quotedColumn}`);
+            return `${alias}.dest_id`;
+        }
+        if (TEXT_COLUMNS.has(column)) {
+            const alias = `mt_${column}`;
+            joins.push(`LEFT JOIN xfer_map_text_refs ${alias} ON ${alias}.src_id = pm.${quotedColumn}`);
+            return `${alias}.dest_id`;
+        }
+        if (column === 'meta_extras_json_id') {
+            joins.push('LEFT JOIN xfer_map_json_refs mj ON mj.src_id = pm.meta_extras_json_id');
+            return 'mj.dest_id';
+        }
+        return `pm.${quotedColumn}`;
+    };
+    const selectList = columns.map(exprFor).join(', ');
+    const joinList = joins.join('\n');
+    const quotedColumnList = columns.map(quoteTransferIdentifier).join(', ');
+    await trx.raw(`
+		INSERT INTO main.page_meta (page_id, ${quotedColumnList})
+		SELECT p.dest_id, ${selectList}
+		FROM ${SRC}.page_meta pm
+		JOIN xfer_ci_plan p ON p.src_id = pm.page_id
+			AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		${joinList}
+		WHERE true
+		ON CONFLICT(page_id) DO NOTHING
+	`);
+}

package/lib/transfer/copy-page-template-clusters.d.ts

@@ -0,0 +1,31 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `page_template_clusters` rows referenced by at least one kept
+ * (`full`/`replace`) page's `page_templates.template_key`, then
+ * recomputes `member_count` (and prunes any cluster left with zero
+ * members) against the destination's OWN `page_templates` table — which
+ * must already reflect this source's copy
+ * ({@link import('./copy-simple-page-scoped-tables.js').copySimplePageScopedTables},
+ * which carries `page_templates` — see that file's table list).
+ *
+ * A `template_key` collision between two DIFFERENT sources' clustering
+ * runs (concat) is a known, accepted simplification: `template_key` is a
+ * blocking-key string from page-cluster's own corpus
+ * classification, computed independently per archive, so the same string
+ * appearing in two sources does not necessarily mean the same semantic
+ * template — `ON CONFLICT(template_key) DO NOTHING` below means the
+ * FIRST source's `reason_json` (cluster-selection evidence) wins and the
+ * merged `member_count` recompute mixes both sources' pages under it.
+ * `concat` therefore re-classifies its output right after this copy,
+ * recomputing clean, merged-corpus clusters (see `docs/concat.md`); the
+ * copied rows only seed label inheritance for that run.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `copySimplePageScopedTables` (which copies `page_templates` itself).
+ * @example
+ * ```ts
+ * await copySimplePageScopedTables(trx); // copies page_templates
+ * await copyPageTemplateClusters(trx);
+ * ```
+ */
+export declare function copyPageTemplateClusters(trx: Knex): Promise<void>;

package/lib/transfer/copy-page-template-clusters.js

@@ -0,0 +1,56 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies `page_template_clusters` rows referenced by at least one kept
+ * (`full`/`replace`) page's `page_templates.template_key`, then
+ * recomputes `member_count` (and prunes any cluster left with zero
+ * members) against the destination's OWN `page_templates` table — which
+ * must already reflect this source's copy
+ * ({@link import('./copy-simple-page-scoped-tables.js').copySimplePageScopedTables},
+ * which carries `page_templates` — see that file's table list).
+ *
+ * A `template_key` collision between two DIFFERENT sources' clustering
+ * runs (concat) is a known, accepted simplification: `template_key` is a
+ * blocking-key string from page-cluster's own corpus
+ * classification, computed independently per archive, so the same string
+ * appearing in two sources does not necessarily mean the same semantic
+ * template — `ON CONFLICT(template_key) DO NOTHING` below means the
+ * FIRST source's `reason_json` (cluster-selection evidence) wins and the
+ * merged `member_count` recompute mixes both sources' pages under it.
+ * `concat` therefore re-classifies its output right after this copy,
+ * recomputing clean, merged-corpus clusters (see `docs/concat.md`); the
+ * copied rows only seed label inheritance for that run.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `copySimplePageScopedTables` (which copies `page_templates` itself).
+ * @example
+ * ```ts
+ * await copySimplePageScopedTables(trx); // copies page_templates
+ * await copyPageTemplateClusters(trx);
+ * ```
+ */
+export async function copyPageTemplateClusters(trx) {
+    await trx.raw(`
+		INSERT INTO main.page_template_clusters
+			(template_key, member_count, reason_json, codec, size_raw, size_stored)
+		SELECT s.template_key, s.member_count, s.reason_json, s.codec, s.size_raw, s.size_stored
+		FROM ${SRC}.page_template_clusters s
+		WHERE EXISTS (
+			SELECT 1
+			FROM ${SRC}.page_templates pt
+			JOIN xfer_ci_plan p ON p.src_id = pt.page_id
+				AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+			WHERE pt.template_key = s.template_key
+		)
+		ON CONFLICT(template_key) DO NOTHING
+	`);
+    await trx.raw(`
+		UPDATE main.page_template_clusters
+		SET member_count = (
+			SELECT COUNT(*) FROM main.page_templates
+			WHERE main.page_templates.template_key = main.page_template_clusters.template_key
+		)
+	`);
+    await trx.raw('DELETE FROM main.page_template_clusters WHERE member_count = 0');
+}

package/lib/transfer/copy-page-template-labels.d.ts

@@ -0,0 +1,34 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `page_template_labels` rows referenced by at least one kept
+ * (`full`/`replace`) page's `page_templates.template_key`, prunes any label
+ * whose key no longer has a member in the destination's OWN
+ * `page_templates`, and then **renumbers label collisions**: two sources
+ * (concat) number their labels independently, so both can carry an
+ * "events template A" under different keys. Leaving both would give the
+ * merged archive two templates with one name — and a later
+ * re-classification of the output (`concat` / `split` run one) would *keep* both, since `replacePageTemplates`
+ * carries labels forward by member overlap rather than renaming. The later
+ * key (by `template_key` order, for determinism) is therefore moved to the
+ * next free ordinal in its section here, at copy time.
+ *
+ * A `template_key` collision (the same key string in both sources) is the
+ * same accepted simplification as in
+ * {@link import('./copy-page-template-clusters.js').copyPageTemplateClusters}:
+ * `ON CONFLICT(template_key) DO NOTHING` keeps the FIRST source's label.
+ *
+ * A source extracted by an older nitpicker — the tar cache is keyed without
+ * a schema version, so an archive first opened before this table existed
+ * stays without it until re-extracted — has no `page_template_labels` at
+ * all; that source simply contributes no labels rather than failing the
+ * whole transfer.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `copySimplePageScopedTables` (which copies `page_templates` itself).
+ * @example
+ * ```ts
+ * await copySimplePageScopedTables(trx); // copies page_templates
+ * await copyPageTemplateLabels(trx);
+ * ```
+ */
+export declare function copyPageTemplateLabels(trx: Knex): Promise<void>;

package/lib/transfer/copy-page-template-labels.js

@@ -0,0 +1,88 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies `page_template_labels` rows referenced by at least one kept
+ * (`full`/`replace`) page's `page_templates.template_key`, prunes any label
+ * whose key no longer has a member in the destination's OWN
+ * `page_templates`, and then **renumbers label collisions**: two sources
+ * (concat) number their labels independently, so both can carry an
+ * "events template A" under different keys. Leaving both would give the
+ * merged archive two templates with one name — and a later
+ * re-classification of the output (`concat` / `split` run one) would *keep* both, since `replacePageTemplates`
+ * carries labels forward by member overlap rather than renaming. The later
+ * key (by `template_key` order, for determinism) is therefore moved to the
+ * next free ordinal in its section here, at copy time.
+ *
+ * A `template_key` collision (the same key string in both sources) is the
+ * same accepted simplification as in
+ * {@link import('./copy-page-template-clusters.js').copyPageTemplateClusters}:
+ * `ON CONFLICT(template_key) DO NOTHING` keeps the FIRST source's label.
+ *
+ * A source extracted by an older nitpicker — the tar cache is keyed without
+ * a schema version, so an archive first opened before this table existed
+ * stays without it until re-extracted — has no `page_template_labels` at
+ * all; that source simply contributes no labels rather than failing the
+ * whole transfer.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `copySimplePageScopedTables` (which copies `page_templates` itself).
+ * @example
+ * ```ts
+ * await copySimplePageScopedTables(trx); // copies page_templates
+ * await copyPageTemplateLabels(trx);
+ * ```
+ */
+export async function copyPageTemplateLabels(trx) {
+    const [sourceTable] = (await trx.raw(`SELECT name FROM ${SRC}.sqlite_master WHERE type = 'table' AND name = 'page_template_labels'`));
+    if (sourceTable) {
+        await trx.raw(`
+			INSERT INTO main.page_template_labels (template_key, section, ordinal)
+			SELECT s.template_key, s.section, s.ordinal
+			FROM ${SRC}.page_template_labels s
+			WHERE EXISTS (
+				SELECT 1
+				FROM ${SRC}.page_templates pt
+				JOIN xfer_ci_plan p ON p.src_id = pt.page_id
+					AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+				WHERE pt.template_key = s.template_key
+			)
+			ON CONFLICT(template_key) DO NOTHING
+		`);
+    }
+    await trx.raw(`
+		DELETE FROM main.page_template_labels
+		WHERE NOT EXISTS (
+			SELECT 1 FROM main.page_templates
+			WHERE main.page_templates.template_key = main.page_template_labels.template_key
+		)
+	`);
+    await renumberLabelCollisions(trx);
+}
+/**
+ * Gives every `(section, ordinal)` pair held by more than one key a fresh,
+ * unused ordinal for all but the first key (by `template_key` order).
+ * @param trx - The transfer transaction.
+ */
+async function renumberLabelCollisions(trx) {
+    const rows = (await trx('main.page_template_labels')
+        .select('template_key as templateKey', 'section', 'ordinal')
+        .orderBy('template_key'));
+    const maxOrdinalBySection = new Map();
+    for (const row of rows) {
+        maxOrdinalBySection.set(row.section, Math.max(maxOrdinalBySection.get(row.section) ?? 0, row.ordinal));
+    }
+    const seen = new Set();
+    for (const row of rows) {
+        const pair = `${row.section ?? '\0'}:${row.ordinal}`;
+        if (!seen.has(pair)) {
+            seen.add(pair);
+            continue;
+        }
+        const next = (maxOrdinalBySection.get(row.section) ?? 0) + 1;
+        maxOrdinalBySection.set(row.section, next);
+        await trx('main.page_template_labels')
+            .where('template_key', row.templateKey)
+            .update({ ordinal: next });
+    }
+}

package/lib/transfer/copy-resource-items.d.ts

@@ -0,0 +1,29 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `resource_items` (and its `technology_js_scan_cache` companion)
+ * for every `full`/`replace`-action row in `temp.xfer_ri_plan`
+ * ({@link import('./plan-resource-items.js').planResourceItems}), then
+ * backfills each `full` row's `dest_id`.
+ *
+ * `is_external` is carried over from the source verbatim here — it is
+ * NOT authoritative yet. `resource_items.is_external` is a pure scope
+ * check (unlike `content_items.is_external`, "was this taken on as a
+ * target"), so
+ * {@link import('./reclassify-resource-externality.js').reclassifyResourceExternality}
+ * always recomputes it against the OUTPUT archive's own (merged or
+ * narrowed) scope once every source has been copied — writing the
+ * source's value here is only a placeholder that step immediately
+ * overwrites.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after the dictionary copies
+ *   (`temp.xfer_map_url_refs`/`xfer_map_blob_refs`/`xfer_map_content_type_refs`/
+ *   `xfer_map_header_sets` populated) and after
+ *   {@link import('./plan-resource-items.js').planResourceItems}.
+ * @example
+ * ```ts
+ * await planResourceItems(trx, 'concat');
+ * await copyDictionariesForConcat(trx);
+ * await copyResourceItems(trx);
+ * ```
+ */
+export declare function copyResourceItems(trx: Knex): Promise<void>;

package/lib/transfer/copy-resource-items.js

@@ -0,0 +1,89 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies `resource_items` (and its `technology_js_scan_cache` companion)
+ * for every `full`/`replace`-action row in `temp.xfer_ri_plan`
+ * ({@link import('./plan-resource-items.js').planResourceItems}), then
+ * backfills each `full` row's `dest_id`.
+ *
+ * `is_external` is carried over from the source verbatim here — it is
+ * NOT authoritative yet. `resource_items.is_external` is a pure scope
+ * check (unlike `content_items.is_external`, "was this taken on as a
+ * target"), so
+ * {@link import('./reclassify-resource-externality.js').reclassifyResourceExternality}
+ * always recomputes it against the OUTPUT archive's own (merged or
+ * narrowed) scope once every source has been copied — writing the
+ * source's value here is only a placeholder that step immediately
+ * overwrites.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after the dictionary copies
+ *   (`temp.xfer_map_url_refs`/`xfer_map_blob_refs`/`xfer_map_content_type_refs`/
+ *   `xfer_map_header_sets` populated) and after
+ *   {@link import('./plan-resource-items.js').planResourceItems}.
+ * @example
+ * ```ts
+ * await planResourceItems(trx, 'concat');
+ * await copyDictionariesForConcat(trx);
+ * await copyResourceItems(trx);
+ * ```
+ */
+export async function copyResourceItems(trx) {
+    await trx.raw(`
+		INSERT INTO main.resource_items
+			(url_id, url_blob_id, is_external, status, status_text, content_type_id, content_length, header_set_id, compress, cdn, source)
+		SELECT
+			mu.dest_id, mb.dest_id, s.is_external, s.status, s.status_text,
+			mct.dest_id, s.content_length, mhs.dest_id, s.compress, s.cdn, s.source
+		FROM ${SRC}.resource_items s
+		JOIN xfer_ri_plan rp ON rp.src_id = s.id AND rp.action = ${TRANSFER_ACTION.full}
+		LEFT JOIN xfer_map_url_refs mu ON mu.src_id = s.url_id
+		LEFT JOIN xfer_map_blob_refs mb ON mb.src_id = s.url_blob_id
+		LEFT JOIN xfer_map_content_type_refs mct ON mct.src_id = s.content_type_id
+		LEFT JOIN xfer_map_header_sets mhs ON mhs.src_id = s.header_set_id
+	`);
+    await trx.raw(`
+		UPDATE xfer_ri_plan
+		SET dest_id = x.dest_id
+		FROM (
+			SELECT rp.src_id AS src_id, d.id AS dest_id
+			FROM xfer_ri_plan rp
+			JOIN ${SRC}.resource_items s ON s.id = rp.src_id
+			LEFT JOIN xfer_map_url_refs mu ON mu.src_id = s.url_id
+			LEFT JOIN xfer_map_blob_refs mb ON mb.src_id = s.url_blob_id
+			JOIN main.resource_items d ON d.url_id = mu.dest_id OR d.url_blob_id = mb.dest_id
+			WHERE rp.action = ${TRANSFER_ACTION.full}
+		) x
+		WHERE xfer_ri_plan.src_id = x.src_id
+	`);
+    await trx.raw(`
+		UPDATE main.resource_items
+		SET
+			status = x.status, status_text = x.status_text, content_type_id = x.content_type_id,
+			content_length = x.content_length, header_set_id = x.header_set_id,
+			compress = x.compress, cdn = x.cdn, source = x.source, is_external = x.is_external
+		FROM (
+			SELECT
+				rp.dest_id AS dest_id, s.status, s.status_text, mct.dest_id AS content_type_id,
+				s.content_length, mhs.dest_id AS header_set_id, s.compress, s.cdn, s.source, s.is_external
+			FROM xfer_ri_plan rp
+			JOIN ${SRC}.resource_items s ON s.id = rp.src_id
+			LEFT JOIN xfer_map_content_type_refs mct ON mct.src_id = s.content_type_id
+			LEFT JOIN xfer_map_header_sets mhs ON mhs.src_id = s.header_set_id
+			WHERE rp.action = ${TRANSFER_ACTION.replace}
+		) x
+		WHERE main.resource_items.id = x.dest_id
+	`);
+    await trx.raw(`
+		INSERT INTO main.technology_js_scan_cache (resourceId, scannedAt, technology, evidence)
+		SELECT rp.dest_id, c.scannedAt, c.technology, c.evidence
+		FROM ${SRC}.technology_js_scan_cache c
+		JOIN xfer_ri_plan rp ON rp.src_id = c.resourceId
+			AND rp.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		WHERE true
+		ON CONFLICT(resourceId) DO UPDATE SET
+			scannedAt = excluded.scannedAt,
+			technology = excluded.technology,
+			evidence = excluded.evidence
+	`);
+}

package/lib/transfer/copy-resource-ref-edges.d.ts

@@ -0,0 +1,20 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `resource_ref_edges` (page → resource references) for every
+ * `full`/`replace`-action page whose referenced resource is also kept
+ * (`temp.xfer_ri_plan`, INNER JOIN — a resource split's
+ * `plan-resource-items.ts` dropped has no destination row, so any edge
+ * pointing at it is correctly skipped rather than inserted with a
+ * dangling `resource_id`).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after both
+ *   `temp.xfer_ci_plan` (with `dest_id` resolved) and `temp.xfer_ri_plan`
+ *   (with `dest_id` resolved by
+ *   {@link import('./copy-resource-items.js').copyResourceItems}).
+ * @example
+ * ```ts
+ * await copyResourceItems(trx, 'concat');
+ * await copyResourceRefEdges(trx);
+ * ```
+ */
+export declare function copyResourceRefEdges(trx: Knex): Promise<void>;

package/lib/transfer/copy-resource-ref-edges.js

@@ -0,0 +1,33 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies `resource_ref_edges` (page → resource references) for every
+ * `full`/`replace`-action page whose referenced resource is also kept
+ * (`temp.xfer_ri_plan`, INNER JOIN — a resource split's
+ * `plan-resource-items.ts` dropped has no destination row, so any edge
+ * pointing at it is correctly skipped rather than inserted with a
+ * dangling `resource_id`).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after both
+ *   `temp.xfer_ci_plan` (with `dest_id` resolved) and `temp.xfer_ri_plan`
+ *   (with `dest_id` resolved by
+ *   {@link import('./copy-resource-items.js').copyResourceItems}).
+ * @example
+ * ```ts
+ * await copyResourceItems(trx, 'concat');
+ * await copyResourceRefEdges(trx);
+ * ```
+ */
+export async function copyResourceRefEdges(trx) {
+    await trx.raw(`
+		INSERT INTO main.resource_ref_edges (resource_id, page_id, count)
+		SELECT rp.dest_id, p.dest_id, e.count
+		FROM ${SRC}.resource_ref_edges e
+		JOIN xfer_ci_plan p ON p.src_id = e.page_id
+			AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		JOIN xfer_ri_plan rp ON rp.src_id = e.resource_id
+		WHERE true
+		ON CONFLICT(resource_id, page_id) DO NOTHING
+	`);
+}

package/lib/transfer/copy-simple-page-scoped-tables.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Copies every table in {@link SIMPLE_PAGE_SCOPED_TABLES}, for every
+ * `full`/`replace`-action page, using {@link listTransferColumns} for the
+ * column list (never a hand-written one — see that function's docs) since
+ * every column here passes straight through unchanged.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copySimplePageScopedTables(trx);
+ * ```
+ */
+export declare function copySimplePageScopedTables(trx: Knex): Promise<void>;

package/lib/transfer/copy-simple-page-scoped-tables.js

@@ -0,0 +1,34 @@
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { SIMPLE_PAGE_SCOPED_TABLES } from './simple-page-scoped-tables.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies every table in {@link SIMPLE_PAGE_SCOPED_TABLES}, for every
+ * `full`/`replace`-action page, using {@link listTransferColumns} for the
+ * column list (never a hand-written one — see that function's docs) since
+ * every column here passes straight through unchanged.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copySimplePageScopedTables(trx);
+ * ```
+ */
+export async function copySimplePageScopedTables(trx) {
+    for (const { table, pageColumn } of SIMPLE_PAGE_SCOPED_TABLES) {
+        const columns = await listTransferColumns(trx, SRC, table, ['id', pageColumn]);
+        const quotedColumns = columns.map(quoteTransferIdentifier);
+        const columnList = quotedColumns.map((col) => `s.${col}`).join(', ');
+        const quotedPageColumn = quoteTransferIdentifier(pageColumn);
+        await trx.raw(`
+			INSERT INTO main.${table} (${quotedPageColumn}, ${quotedColumns.join(', ')})
+			SELECT p.dest_id, ${columnList}
+			FROM ${SRC}.${table} s
+			JOIN xfer_ci_plan p ON p.src_id = s.${quotedPageColumn}
+				AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		`);
+    }
+}

package/lib/transfer/count-external-pages-in-scope.d.ts

@@ -0,0 +1,26 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+import type { Knex } from 'knex';
+/**
+ * Counts `content_items` rows that are `is_external = 1` yet fall inside
+ * the given scope — exactly the set `crawl <out> --append <root>`'s
+ * `repromoteExternalPages` would re-fetch as internal pages. Concat never
+ * runs that promotion itself (see `types.ts`'s `ConcatArchivesResult.externalInScopeCount`
+ * docs for why), so this count is what the CLI surfaces as its
+ * `--append` hint after a merge.
+ *
+ * Split never calls this (its output scope is a SUBSET of its single
+ * source's scope, so nothing newly falls into scope), but the function
+ * is written generically rather than concat-specific for the same reason
+ * `flattenRedirectChains` is: a narrower, single-purpose name would
+ * invite a second near-duplicate implementation later.
+ * @param trx - The destination transaction.
+ * @param scope - The output archive's final scope map.
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+ * @returns The count of in-scope-but-still-external pages.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(mergedConfig.roots, mergedConfig);
+ * const hintCount = await countExternalPagesInScope(knex, scope, mergedConfig);
+ * ```
+ */
+export declare function countExternalPagesInScope(trx: Knex, scope: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<number>;

package/lib/transfer/count-external-pages-in-scope.js

@@ -0,0 +1,40 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { findScopeEntry } from '../scope/find-scope-entry.js';
+/**
+ * Counts `content_items` rows that are `is_external = 1` yet fall inside
+ * the given scope — exactly the set `crawl <out> --append <root>`'s
+ * `repromoteExternalPages` would re-fetch as internal pages. Concat never
+ * runs that promotion itself (see `types.ts`'s `ConcatArchivesResult.externalInScopeCount`
+ * docs for why), so this count is what the CLI surfaces as its
+ * `--append` hint after a merge.
+ *
+ * Split never calls this (its output scope is a SUBSET of its single
+ * source's scope, so nothing newly falls into scope), but the function
+ * is written generically rather than concat-specific for the same reason
+ * `flattenRedirectChains` is: a narrower, single-purpose name would
+ * invite a second near-duplicate implementation later.
+ * @param trx - The destination transaction.
+ * @param scope - The output archive's final scope map.
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+ * @returns The count of in-scope-but-still-external pages.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(mergedConfig.roots, mergedConfig);
+ * const hintCount = await countExternalPagesInScope(knex, scope, mergedConfig);
+ * ```
+ */
+export async function countExternalPagesInScope(trx, scope, options) {
+    const candidates = await trx
+        .select('url_refs.url as url')
+        .from('content_items')
+        .join('url_refs', 'url_refs.id', 'content_items.url_id')
+        .where('content_items.is_external', 1);
+    let count = 0;
+    for (const row of candidates) {
+        const parsed = parseUrl(row.url, options);
+        if (parsed && findScopeEntry(parsed, scope, options) !== null) {
+            count++;
+        }
+    }
+    return count;
+}