@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/transfer/build-split-config.js

@@ -0,0 +1,42 @@
+import { REQUIRED_FORMAT_VERSION } from '../meta/assert-compatible-version.js';
+/**
+ * Derives the output {@link Config} for a `split`, from the source's own
+ * config plus the scope URLs the operator gave on the command line.
+ *
+ * `roots` becomes exactly the given URLs (their `withoutHash` form,
+ * matching how `crawl`'s own roots are recorded); `baseUrl` is the first
+ * one. Every other scalar field is carried over verbatim from the
+ * source — a split narrows WHICH pages are kept, it does not change HOW
+ * they were crawled (`recursive`/`interval`/`userAgent`/etc. all still
+ * describe the original session honestly). `excludes`/`excludeKeywords`/
+ * `excludeUrls` are likewise carried over unchanged (they described the
+ * source's own crawl-time exclusions, still true of the subset that
+ * survives). `version` is pinned to {@link REQUIRED_FORMAT_VERSION} and
+ * `createdCwd` is always `null`, matching `mergeArchiveConfigs`'s same
+ * two fields for the same reasons.
+ *
+ * Callers must reject a `fromList` source BEFORE calling this — `roots`
+ * for such an archive is a URL LIST, not a scope, so "roots = the given
+ * scope URLs" has no coherent meaning for it (see `docs/split.md`).
+ * @param source - The source archive's `getConfig()` result.
+ * @param scopeUrls - The scope URLs the operator gave (`withoutHash` form).
+ * @param name - The output archive's `name` (typically its basename).
+ * @returns The derived {@link Config} to write via `Archive#setConfig`.
+ * @example
+ * ```ts
+ * const config = buildSplitConfig(await source.getConfig(), ['https://example.com/blog/'], 'blog');
+ * ```
+ */
+export function buildSplitConfig(source, scopeUrls, name) {
+    if (scopeUrls.length === 0) {
+        throw new Error('buildSplitConfig: at least one scope URL is required');
+    }
+    return {
+        ...source,
+        version: REQUIRED_FORMAT_VERSION,
+        name,
+        roots: [...scopeUrls],
+        baseUrl: scopeUrls[0],
+        createdCwd: null,
+    };
+}

package/lib/transfer/clear-replaced-page-rows.d.ts

@@ -0,0 +1,23 @@
+import type { Knex } from 'knex';
+/**
+ * Deletes every page-scoped derived row for a concat source's `replace`-
+ * action destination ids (`temp.xfer_ci_plan.dest_id`), across every
+ * table in {@link REMAPPED_PAGE_ROW_TABLES} + {@link SIMPLE_PAGE_SCOPED_TABLES}
+ * combined. The legacy `analysis_violations` rows are also deleted when the
+ * destination archive still carries that table (guarded by `hasTable`),
+ * because its legacy FK to `content_items` would otherwise block the
+ * deletion. Set-based (`WHERE ... IN (subquery)`), so there is no
+ * `SQLITE_LIMIT_VARIABLE_NUMBER` bind-count concern regardless of how many
+ * rows are being replaced.
+ *
+ * Must run BEFORE the page-scoped table copies
+ * (`copy-page-meta.ts` et al.) so their `INSERT`s never collide with
+ * stale rows left over from the losing destination row's own prior data.
+ * @param trx - Transaction with the source ATTACHed.
+ * @example
+ * ```ts
+ * await clearReplacedPageRows(trx);
+ * await replaceContentItems(trx, { crawlOrderOffset: 0 });
+ * ```
+ */
+export declare function clearReplacedPageRows(trx: Knex): Promise<void>;

package/lib/transfer/clear-replaced-page-rows.js

@@ -0,0 +1,61 @@
+import { SIMPLE_PAGE_SCOPED_TABLES } from './simple-page-scoped-tables.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+/**
+ * The page-scoped tables NOT covered by {@link SIMPLE_PAGE_SCOPED_TABLES}
+ * (which this file also clears — see below) — each of these has its own
+ * dedicated FK-remapping copy function (`copy-page-meta.ts`,
+ * `copy-anchor-edges.ts`, `copy-image-items.ts`,
+ * `copy-resource-ref-edges.ts`, `copy-page-console-logs.ts`), so they are
+ * listed explicitly rather than folded into the shared "plain passthrough"
+ * table. Together the two lists are a superset of `clearPageDerivedRows`'
+ * tables (reused by `resetFailedPages`/`repromoteExternalPages` for the same
+ * un-scrape-then-re-insert shape) — the combined set additionally covers
+ * `page_console_logs`/`page_templates`/`page_errors`, which that shared
+ * helper deliberately omits (see its own JSDoc for why:
+ * `repromoteExternalPages` never needed them cleared, and folding them in
+ * would have silently changed its existing behaviour). Concat's replace
+ * path needs all of them — a `replace` is a full Scoped-Replace of every
+ * page-scoped table, not a partial one.
+ */
+const REMAPPED_PAGE_ROW_TABLES = [
+    { table: 'page_meta', pageColumn: 'page_id' },
+    { table: 'anchor_edges', pageColumn: 'page_id' },
+    { table: 'image_items', pageColumn: 'page_id' },
+    { table: 'resource_ref_edges', pageColumn: 'page_id' },
+    { table: 'page_console_logs', pageColumn: 'pageId' },
+];
+/**
+ * Deletes every page-scoped derived row for a concat source's `replace`-
+ * action destination ids (`temp.xfer_ci_plan.dest_id`), across every
+ * table in {@link REMAPPED_PAGE_ROW_TABLES} + {@link SIMPLE_PAGE_SCOPED_TABLES}
+ * combined. The legacy `analysis_violations` rows are also deleted when the
+ * destination archive still carries that table (guarded by `hasTable`),
+ * because its legacy FK to `content_items` would otherwise block the
+ * deletion. Set-based (`WHERE ... IN (subquery)`), so there is no
+ * `SQLITE_LIMIT_VARIABLE_NUMBER` bind-count concern regardless of how many
+ * rows are being replaced.
+ *
+ * Must run BEFORE the page-scoped table copies
+ * (`copy-page-meta.ts` et al.) so their `INSERT`s never collide with
+ * stale rows left over from the losing destination row's own prior data.
+ * @param trx - Transaction with the source ATTACHed.
+ * @example
+ * ```ts
+ * await clearReplacedPageRows(trx);
+ * await replaceContentItems(trx, { crawlOrderOffset: 0 });
+ * ```
+ */
+export async function clearReplacedPageRows(trx) {
+    const tables = [...REMAPPED_PAGE_ROW_TABLES, ...SIMPLE_PAGE_SCOPED_TABLES];
+    if (await trx.schema.hasTable('analysis_violations')) {
+        tables.push({ table: 'analysis_violations', pageColumn: 'page_id' });
+    }
+    for (const { table, pageColumn } of tables) {
+        await trx.raw(`
+			DELETE FROM main.${table}
+			WHERE ${pageColumn} IN (
+				SELECT dest_id FROM xfer_ci_plan WHERE action = ${TRANSFER_ACTION.replace}
+			)
+		`);
+    }
+}

package/lib/transfer/concat-archives.d.ts

@@ -0,0 +1,55 @@
+import type { ArchiveAccessor } from '../archive-accessor.js';
+import type Archive from '../archive.js';
+import type { ConcatArchivesResult, TransferCallbacks } from './types.js';
+/**
+ * Merges two or more read-only-opened source archives into an already
+ * `Archive.create`d, empty destination — the crawler-side half of the
+ * `concat` CLI command. Does not open/close/write anything itself: the
+ * caller owns every accessor's and the destination's lifecycle (see
+ * `docs/concat.md` and the CLI's `commands/concat.ts`, which owns the
+ * full open → transfer → rebuild-read-model → write sequence this is one
+ * step of).
+ *
+ * Sources are transferred ONE AT A TIME, in argument order, each one's
+ * `crawlOrderOffset` set to the destination's current
+ * `MAX(content_items.crawl_order)` so a later source's pages always sort
+ * after an earlier one's — this is what makes "ties go to the later
+ * argument" (see `plan-content-items-for-concat.ts`) match the pages'
+ * OWN display order too, not just their win/lose outcome.
+ *
+ * After every source is transferred: `flattenRedirectChains` (a merge can
+ * reintroduce a two-hop redirect chain no single source ever has),
+ * `reclassifyResourceExternality` against the MERGED scope, and
+ * `countExternalPagesInScope` for the CLI's `--append` hint (concat never
+ * promotes such pages itself — see `types.ts`'s
+ * `ConcatArchivesResult.externalInScopeCount` docs).
+ * @param options - See individual properties.
+ * @param options.sources - Every source, read-only opened
+ *   (`Archive.openCached`), in argument order.
+ * @param options.destination - The empty destination, already
+ *   `Archive.create`d by the caller.
+ * @param options.name - The output archive's `name` config field
+ *   (typically its basename without extension).
+ * @param options.callbacks - Progress callbacks; see `types.ts`.
+ * @returns The merged config, each source's transfer outcome, and the
+ *   two summary counts.
+ * @throws {import('./types.js').ArchiveConfigConflictError} If sources
+ *   disagree on `disableQueries`/`fromList` — see `mergeArchiveConfigs`.
+ * @example
+ * ```ts
+ * const destination = await Archive.create({ filePath: outputPath, cwd });
+ * const sources = await Promise.all(
+ *   inputPaths.map(async (p) => ({ path: p, accessor: await Archive.openCached(p) })),
+ * );
+ * const result = await concatArchives({ sources, destination, name: 'merged' });
+ * ```
+ */
+export declare function concatArchives(options: {
+    readonly sources: readonly {
+        readonly accessor: ArchiveAccessor;
+        readonly path: string;
+    }[];
+    readonly destination: Archive;
+    readonly name: string;
+    readonly callbacks?: TransferCallbacks;
+}): Promise<ConcatArchivesResult>;

package/lib/transfer/concat-archives.js

@@ -0,0 +1,95 @@
+import path from 'node:path';
+import ArchiveDefault from '../archive.js';
+import { buildScopeMap } from '../scope/build-scope-map.js';
+import { copyInventorySourceLists } from './copy-inventory-source-lists.js';
+import { countExternalPagesInScope } from './count-external-pages-in-scope.js';
+import { flattenRedirectChains } from './flatten-redirect-chains.js';
+import { mergeArchiveConfigs } from './merge-archive-configs.js';
+import { reclassifyResourceExternality } from './reclassify-resource-externality.js';
+import { transferArchiveRows } from './transfer-archive-rows.js';
+/**
+ * Merges two or more read-only-opened source archives into an already
+ * `Archive.create`d, empty destination — the crawler-side half of the
+ * `concat` CLI command. Does not open/close/write anything itself: the
+ * caller owns every accessor's and the destination's lifecycle (see
+ * `docs/concat.md` and the CLI's `commands/concat.ts`, which owns the
+ * full open → transfer → rebuild-read-model → write sequence this is one
+ * step of).
+ *
+ * Sources are transferred ONE AT A TIME, in argument order, each one's
+ * `crawlOrderOffset` set to the destination's current
+ * `MAX(content_items.crawl_order)` so a later source's pages always sort
+ * after an earlier one's — this is what makes "ties go to the later
+ * argument" (see `plan-content-items-for-concat.ts`) match the pages'
+ * OWN display order too, not just their win/lose outcome.
+ *
+ * After every source is transferred: `flattenRedirectChains` (a merge can
+ * reintroduce a two-hop redirect chain no single source ever has),
+ * `reclassifyResourceExternality` against the MERGED scope, and
+ * `countExternalPagesInScope` for the CLI's `--append` hint (concat never
+ * promotes such pages itself — see `types.ts`'s
+ * `ConcatArchivesResult.externalInScopeCount` docs).
+ * @param options - See individual properties.
+ * @param options.sources - Every source, read-only opened
+ *   (`Archive.openCached`), in argument order.
+ * @param options.destination - The empty destination, already
+ *   `Archive.create`d by the caller.
+ * @param options.name - The output archive's `name` config field
+ *   (typically its basename without extension).
+ * @param options.callbacks - Progress callbacks; see `types.ts`.
+ * @returns The merged config, each source's transfer outcome, and the
+ *   two summary counts.
+ * @throws {import('./types.js').ArchiveConfigConflictError} If sources
+ *   disagree on `disableQueries`/`fromList` — see `mergeArchiveConfigs`.
+ * @example
+ * ```ts
+ * const destination = await Archive.create({ filePath: outputPath, cwd });
+ * const sources = await Promise.all(
+ *   inputPaths.map(async (p) => ({ path: p, accessor: await Archive.openCached(p) })),
+ * );
+ * const result = await concatArchives({ sources, destination, name: 'merged' });
+ * ```
+ */
+export async function concatArchives(options) {
+    const { sources, destination, name, callbacks } = options;
+    if (sources.length < 2) {
+        throw new Error('concatArchives: at least two sources are required');
+    }
+    const configs = await Promise.all(sources.map((s) => s.accessor.getConfig()));
+    const merged = mergeArchiveConfigs(configs, name);
+    await destination.setConfig(merged);
+    const knex = destination.getKnex();
+    const scope = buildScopeMap(merged.roots, merged);
+    const results = [];
+    for (const [index, source] of sources.entries()) {
+        callbacks?.onSourceStart?.(index, source.path);
+        const sourceDbPath = path.join(source.accessor.tmpDir, ArchiveDefault.SQLITE_DB_FILE_NAME);
+        const maxRows = await knex('content_items').max({
+            max: 'crawl_order',
+        });
+        const crawlOrderOffset = maxRows[0]?.max ?? 0;
+        const result = await transferArchiveRows({
+            knex,
+            sourceDbPath,
+            mode: { kind: 'concat' },
+            crawlOrderOffset,
+            sourceIndex: index,
+            callbacks,
+        });
+        results.push(result);
+    }
+    const postIndex = sources.length;
+    callbacks?.onPhase?.(postIndex, 'flatteningRedirects');
+    await flattenRedirectChains(knex);
+    callbacks?.onPhase?.(postIndex, 'reclassifyingResources');
+    await reclassifyResourceExternality(knex, scope, merged);
+    callbacks?.onPhase?.(postIndex, 'countingExternalInScope');
+    const externalInScopeCount = await countExternalPagesInScope(knex, scope, merged);
+    const inventoryListsCopied = await copyInventorySourceLists(destination, sources.map((s) => s.accessor.tmpDir));
+    return {
+        config: merged,
+        sources: results,
+        externalInScopeCount,
+        inventoryListsCopied,
+    };
+}

package/lib/transfer/concat-source-transfer-phases.d.ts

@@ -0,0 +1,10 @@
+import type { TransferPhase } from './types.js';
+/**
+ * Ordered phase sequence {@link import('./transfer-archive-rows.js').transferArchiveRows}
+ * announces via `onPhase`, once per concat source, unconditionally and
+ * always in this order (see that function's own source — no branch skips
+ * a phase). Single source of truth for the per-source `TaskList` rows a
+ * CLI caller pre-builds (mirrors `VIEWER_READ_MODEL_FULL_BUILD_PHASES`'s
+ * same role for the read-model build).
+ */
+export declare const CONCAT_SOURCE_TRANSFER_PHASES: readonly TransferPhase[];

package/lib/transfer/concat-source-transfer-phases.js

@@ -0,0 +1,17 @@
+/**
+ * Ordered phase sequence {@link import('./transfer-archive-rows.js').transferArchiveRows}
+ * announces via `onPhase`, once per concat source, unconditionally and
+ * always in this order (see that function's own source — no branch skips
+ * a phase). Single source of truth for the per-source `TaskList` rows a
+ * CLI caller pre-builds (mirrors `VIEWER_READ_MODEL_FULL_BUILD_PHASES`'s
+ * same role for the read-model build).
+ */
+export const CONCAT_SOURCE_TRANSFER_PHASES = [
+    'planningContentItems',
+    'copyingDictionaries',
+    'copyingContentItems',
+    'copyingPageRows',
+    'copyingResources',
+    'copyingJournals',
+    'committing',
+];

package/lib/transfer/content-item-rank-sql.d.ts

@@ -0,0 +1,21 @@
+/**
+ * SQL fragment computing a `content_items` row's "richness" rank for
+ * concat's same-URL conflict resolution: `2` (scraped internal page) beats
+ * `1` (scraped external HEAD-only observation) beats `0` (never scraped).
+ * Concat keeps whichever source's row ranks highest for a given URL; a tie
+ * is resolved by processing sources in argument order and overwriting on
+ * `>=` (see {@link import('./plan-content-items-for-concat.js').planContentItemsForConcat}),
+ * so the LAST source with an equal-or-better rank wins.
+ *
+ * This is the single definition of the ranking lattice — every other
+ * ranking (or a change to it) must go through this function so concat's
+ * conflict resolution cannot silently drift from what this doc describes.
+ * @param alias - The table alias (or bare table name) the fragment reads
+ *   `scraped`/`is_external` from, e.g. `'s'` for `xfer_src.content_items s`.
+ * @returns A SQL expression (no trailing semicolon) evaluating to `0`, `1`, or `2`.
+ * @example
+ * ```ts
+ * trx.raw(`SELECT ${contentItemRankSql('s')} as rank FROM xfer_src.content_items s`);
+ * ```
+ */
+export declare function contentItemRankSql(alias: string): string;

package/lib/transfer/content-item-rank-sql.js

@@ -0,0 +1,23 @@
+/**
+ * SQL fragment computing a `content_items` row's "richness" rank for
+ * concat's same-URL conflict resolution: `2` (scraped internal page) beats
+ * `1` (scraped external HEAD-only observation) beats `0` (never scraped).
+ * Concat keeps whichever source's row ranks highest for a given URL; a tie
+ * is resolved by processing sources in argument order and overwriting on
+ * `>=` (see {@link import('./plan-content-items-for-concat.js').planContentItemsForConcat}),
+ * so the LAST source with an equal-or-better rank wins.
+ *
+ * This is the single definition of the ranking lattice — every other
+ * ranking (or a change to it) must go through this function so concat's
+ * conflict resolution cannot silently drift from what this doc describes.
+ * @param alias - The table alias (or bare table name) the fragment reads
+ *   `scraped`/`is_external` from, e.g. `'s'` for `xfer_src.content_items s`.
+ * @returns A SQL expression (no trailing semicolon) evaluating to `0`, `1`, or `2`.
+ * @example
+ * ```ts
+ * trx.raw(`SELECT ${contentItemRankSql('s')} as rank FROM xfer_src.content_items s`);
+ * ```
+ */
+export function contentItemRankSql(alias) {
+    return `(CASE WHEN ${alias}.scraped = 1 THEN 2 - ${alias}.is_external ELSE 0 END)`;
+}

package/lib/transfer/copy-anchor-edges.d.ts

@@ -0,0 +1,28 @@
+import type { Knex } from 'knex';
+/**
+ * Copies `anchor_edges` for every `full`/`replace`-action page's OUTBOUND
+ * links, remapping both `page_id` (the link's own page — must be
+ * `full`/`replace`, same rule as every other page-scoped table) and
+ * `href_page_id` (the link's target — resolved through
+ * `temp.xfer_ci_plan` directly, via an INNER JOIN, NOT `LEFT JOIN`):
+ *
+ * - Concat: every source row has a plan entry (even `skip`), so the
+ *   INNER JOIN always matches — an anchor to a `skip`ped URL still
+ *   resolves to whichever destination row won that URL.
+ * - Split: a target NOT in the plan means "dropped" (out of scope and
+ *   unreferenced by anything else) — the INNER JOIN drops that anchor
+ *   row entirely rather than inserting a dangling `href_page_id`. This is
+ *   intentional: `plan-content-items-for-split.ts`'s own closure already
+ *   guarantees every anchor target of a KEPT page is itself kept (as
+ *   `full` or `stub`), so in practice this INNER JOIN never actually
+ *   filters anything out for split — it exists as a structural safety net,
+ *   not as an active filter.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copyAnchorEdges(trx);
+ * ```
+ */
+export declare function copyAnchorEdges(trx: Knex): Promise<void>;

package/lib/transfer/copy-anchor-edges.js

@@ -0,0 +1,42 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Copies `anchor_edges` for every `full`/`replace`-action page's OUTBOUND
+ * links, remapping both `page_id` (the link's own page — must be
+ * `full`/`replace`, same rule as every other page-scoped table) and
+ * `href_page_id` (the link's target — resolved through
+ * `temp.xfer_ci_plan` directly, via an INNER JOIN, NOT `LEFT JOIN`):
+ *
+ * - Concat: every source row has a plan entry (even `skip`), so the
+ *   INNER JOIN always matches — an anchor to a `skip`ped URL still
+ *   resolves to whichever destination row won that URL.
+ * - Split: a target NOT in the plan means "dropped" (out of scope and
+ *   unreferenced by anything else) — the INNER JOIN drops that anchor
+ *   row entirely rather than inserting a dangling `href_page_id`. This is
+ *   intentional: `plan-content-items-for-split.ts`'s own closure already
+ *   guarantees every anchor target of a KEPT page is itself kept (as
+ *   `full` or `stub`), so in practice this INNER JOIN never actually
+ *   filters anything out for split — it exists as a structural safety net,
+ *   not as an active filter.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @example
+ * ```ts
+ * await copyPageMeta(trx);
+ * await copyAnchorEdges(trx);
+ * ```
+ */
+export async function copyAnchorEdges(trx) {
+    await trx.raw(`
+		INSERT INTO main.anchor_edges (page_id, href_page_id, count, first_hash, first_text_id)
+		SELECT p.dest_id, pd.dest_id, ae.count, ae.first_hash, mt.dest_id
+		FROM ${SRC}.anchor_edges ae
+		JOIN xfer_ci_plan p ON p.src_id = ae.page_id
+			AND p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.replace})
+		JOIN xfer_ci_plan pd ON pd.src_id = ae.href_page_id
+		LEFT JOIN xfer_map_text_refs mt ON mt.src_id = ae.first_text_id
+		WHERE true
+		ON CONFLICT(page_id, href_page_id) DO NOTHING
+	`);
+}

package/lib/transfer/copy-dictionaries-for-concat.d.ts

@@ -0,0 +1,39 @@
+import type { Knex } from 'knex';
+/**
+ * Copies every dictionary table's rows from the attached source into the
+ * destination unconditionally (`ON CONFLICT DO NOTHING` on the natural
+ * key), then builds a `temp.xfer_map_<table>` mapping every source row's id
+ * to its (possibly pre-existing, possibly freshly-inserted) destination
+ * id.
+ *
+ * Concat does not compute a "needed by kept content" filter first, unlike
+ * split (`copy-dictionaries-for-split.ts`) — every dictionary row is
+ * copied whether or not the content_items row(s) that referenced it ended
+ * up `full`/`replace` or `skip`. This can leave a handful of dictionary
+ * rows in the destination with no live referrer (a source's `skip`ped
+ * row's own `text_refs`/`url_refs`, say), the same accepted trade-off this
+ * archive already makes for `page_html_blobs` orphans (pending #23's GC) —
+ * computing a precise per-source needed-set here would roughly double the
+ * SQL in this file for a storage saving that matters far less than
+ * concat's own priority order (viewer compat > read perf > storage, see
+ * `viewer-priority-order.md`).
+ *
+ * `console_log_items` and `page_html_blobs` are copied by
+ * `copy-page-console-logs.ts` / `copy-page-html-blobs.ts` respectively —
+ * both need a source-row need-set derived from `page_console_logs`/
+ * `page_html_ref` regardless of concat/split, so they are not duplicated
+ * here.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./create-transfer-temp-tables.js').createTransferTempTables}.
+ * @throws {import('./types.js').TransferIntegrityError} If any table's
+ *   mapped row count does not equal its source row count after the
+ *   natural-key JOIN — see this function's body for why that should be
+ *   structurally impossible in normal operation.
+ * @example
+ * ```ts
+ * await copyDictionariesForConcat(trx);
+ * // temp.xfer_map_url_refs, temp.xfer_map_text_refs, ... are now populated
+ * ```
+ */
+export declare function copyDictionariesForConcat(trx: Knex): Promise<void>;

package/lib/transfer/copy-dictionaries-for-concat.js

@@ -0,0 +1,81 @@
+import { DICTIONARY_COPY_SPECS } from './dictionary-copy-specs.js';
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+import { TransferIntegrityError } from './types.js';
+/**
+ * Copies every dictionary table's rows from the attached source into the
+ * destination unconditionally (`ON CONFLICT DO NOTHING` on the natural
+ * key), then builds a `temp.xfer_map_<table>` mapping every source row's id
+ * to its (possibly pre-existing, possibly freshly-inserted) destination
+ * id.
+ *
+ * Concat does not compute a "needed by kept content" filter first, unlike
+ * split (`copy-dictionaries-for-split.ts`) — every dictionary row is
+ * copied whether or not the content_items row(s) that referenced it ended
+ * up `full`/`replace` or `skip`. This can leave a handful of dictionary
+ * rows in the destination with no live referrer (a source's `skip`ped
+ * row's own `text_refs`/`url_refs`, say), the same accepted trade-off this
+ * archive already makes for `page_html_blobs` orphans (pending #23's GC) —
+ * computing a precise per-source needed-set here would roughly double the
+ * SQL in this file for a storage saving that matters far less than
+ * concat's own priority order (viewer compat > read perf > storage, see
+ * `viewer-priority-order.md`).
+ *
+ * `console_log_items` and `page_html_blobs` are copied by
+ * `copy-page-console-logs.ts` / `copy-page-html-blobs.ts` respectively —
+ * both need a source-row need-set derived from `page_console_logs`/
+ * `page_html_ref` regardless of concat/split, so they are not duplicated
+ * here.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./create-transfer-temp-tables.js').createTransferTempTables}.
+ * @throws {import('./types.js').TransferIntegrityError} If any table's
+ *   mapped row count does not equal its source row count after the
+ *   natural-key JOIN — see this function's body for why that should be
+ *   structurally impossible in normal operation.
+ * @example
+ * ```ts
+ * await copyDictionariesForConcat(trx);
+ * // temp.xfer_map_url_refs, temp.xfer_map_text_refs, ... are now populated
+ * ```
+ */
+export async function copyDictionariesForConcat(trx) {
+    for (const spec of DICTIONARY_COPY_SPECS) {
+        const columns = await listTransferColumns(trx, TRANSFER_SOURCE_ALIAS, spec.table, [
+            'id',
+        ]);
+        const quotedColumns = columns.map(quoteTransferIdentifier);
+        const columnList = quotedColumns.join(', ');
+        const conflictTarget = spec.naturalKey.map(quoteTransferIdentifier).join(', ');
+        await trx.raw(`
+			INSERT INTO main.${spec.table} (${columnList})
+			SELECT ${columnList} FROM ${TRANSFER_SOURCE_ALIAS}.${spec.table}
+			WHERE true
+			ON CONFLICT(${conflictTarget}) DO NOTHING
+		`);
+        const joinCondition = spec.naturalKey
+            .map((col) => `d.${quoteTransferIdentifier(col)} IS s.${quoteTransferIdentifier(col)}`)
+            .join(' AND ');
+        await trx.raw(`
+			INSERT INTO xfer_map_${spec.table} (src_id, dest_id)
+			SELECT s.id, d.id
+			FROM ${TRANSFER_SOURCE_ALIAS}.${spec.table} s
+			JOIN main.${spec.table} d ON ${joinCondition}
+		`);
+        // Every source row must resolve to a destination id — concat maps
+        // unconditionally (no needed-set filter), so `xfer_map_<table>`
+        // must cover every row of the attached source's own table. A gap
+        // here means the natural-key JOIN above failed to match a row this
+        // same function just inserted, which should be impossible; treat it
+        // as a hard integrity failure rather than let a missing map entry
+        // surface later as a confusing NULL FK deep in an unrelated table.
+        const sourceCountRows = await trx(`${TRANSFER_SOURCE_ALIAS}.${spec.table}`).count({ sourceCount: '*' });
+        const mappedCountRows = await trx(`xfer_map_${spec.table}`).count({ mappedCount: '*' });
+        const sourceCount = sourceCountRows[0]?.sourceCount ?? 0;
+        const mappedCount = mappedCountRows[0]?.mappedCount ?? 0;
+        if (Number(mappedCount) !== Number(sourceCount)) {
+            throw new TransferIntegrityError(`copyDictionariesForConcat: ${spec.table} has ${sourceCount} source row(s) but only ${mappedCount} mapped to a destination id`);
+        }
+    }
+}

package/lib/transfer/copy-dictionaries-for-split.d.ts

@@ -0,0 +1,29 @@
+import type { Knex } from 'knex';
+/**
+ * Copies only the dictionary rows referenced by kept content — the split
+ * counterpart of `copy-dictionaries-for-concat.ts`'s unconditional
+ * copy-everything. Split's entire purpose is size reduction, so (unlike
+ * concat, where computing a precise needed-set is not worth the
+ * complexity) a dropped page's dictionary rows must not survive into the
+ * output archive.
+ *
+ * Runs in two phases: collect every needed dictionary id into
+ * `temp.xfer_need_<table>` (this function's own private helpers, in
+ * dependency order — `console_log_items`' own needs must be collected
+ * AFTER `console_log_items` itself, since they derive from it), then copy
+ * + map each dictionary table exactly like concat but filtered to the
+ * needed set.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   `temp.xfer_ci_plan` AND `temp.xfer_ri_plan` are fully populated.
+ * @throws {import('./types.js').TransferIntegrityError} If any table's
+ *   mapped row count does not equal its needed-row count after the
+ *   natural-key JOIN.
+ * @example
+ * ```ts
+ * await planContentItemsForSplit(trx);
+ * await planResourceItems(trx, 'split');
+ * await copyDictionariesForSplit(trx);
+ * ```
+ */
+export declare function copyDictionariesForSplit(trx: Knex): Promise<void>;