@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/transfer/create-transfer-temp-tables.d.ts

@@ -0,0 +1,30 @@
+import type { Knex } from 'knex';
+/**
+ * Creates every `temp.xfer_*` table one source's transfer needs: the
+ * `content_items` / `resource_items` plans (per-row action + resolved
+ * destination id), the in-scope/referenced staging tables split's plan
+ * computation uses, and one `(src_id PRIMARY KEY)` need-table plus one
+ * `(src_id PRIMARY KEY, dest_id)` map-table per dictionary in
+ * {@link TRANSFER_DICTIONARY_TABLES}.
+ *
+ * Must run on the SAME connection {@link import('./attach-source-database.js').attachSourceDatabase}
+ * attached the source to (SQLite `TEMP` tables are connection-scoped, not
+ * transaction-scoped) — and, unlike `ATTACH`, is safe to run either inside
+ * or outside a transaction. Called once per source, paired with
+ * {@link import('./drop-transfer-temp-tables.js').dropTransferTempTables}
+ * after that source's transaction commits, so a second source's plan
+ * starts from empty tables rather than accumulating the previous source's
+ * rows.
+ * @param knex - Knex query builder connected to the destination archive DB
+ *   (the connection the temp tables attach to).
+ * @example
+ * ```ts
+ * await createTransferTempTables(knex);
+ * try {
+ *   await knex.transaction(async (trx) => { ... });
+ * } finally {
+ *   await dropTransferTempTables(knex);
+ * }
+ * ```
+ */
+export declare function createTransferTempTables(knex: Knex): Promise<void>;

package/lib/transfer/create-transfer-temp-tables.js

@@ -0,0 +1,59 @@
+import { TRANSFER_DICTIONARY_TABLES } from './transfer-dictionary-tables.js';
+/**
+ * Creates every `temp.xfer_*` table one source's transfer needs: the
+ * `content_items` / `resource_items` plans (per-row action + resolved
+ * destination id), the in-scope/referenced staging tables split's plan
+ * computation uses, and one `(src_id PRIMARY KEY)` need-table plus one
+ * `(src_id PRIMARY KEY, dest_id)` map-table per dictionary in
+ * {@link TRANSFER_DICTIONARY_TABLES}.
+ *
+ * Must run on the SAME connection {@link import('./attach-source-database.js').attachSourceDatabase}
+ * attached the source to (SQLite `TEMP` tables are connection-scoped, not
+ * transaction-scoped) — and, unlike `ATTACH`, is safe to run either inside
+ * or outside a transaction. Called once per source, paired with
+ * {@link import('./drop-transfer-temp-tables.js').dropTransferTempTables}
+ * after that source's transaction commits, so a second source's plan
+ * starts from empty tables rather than accumulating the previous source's
+ * rows.
+ * @param knex - Knex query builder connected to the destination archive DB
+ *   (the connection the temp tables attach to).
+ * @example
+ * ```ts
+ * await createTransferTempTables(knex);
+ * try {
+ *   await knex.transaction(async (trx) => { ... });
+ * } finally {
+ *   await dropTransferTempTables(knex);
+ * }
+ * ```
+ */
+export async function createTransferTempTables(knex) {
+    await knex.raw(`
+		CREATE TEMP TABLE xfer_ci_plan (
+			src_id INTEGER PRIMARY KEY,
+			action INTEGER NOT NULL,
+			dest_id INTEGER
+		)
+	`);
+    await knex.raw(`
+		CREATE TEMP TABLE xfer_ci_in_scope (
+			src_id INTEGER PRIMARY KEY
+		)
+	`);
+    await knex.raw(`
+		CREATE TEMP TABLE xfer_ci_ref (
+			src_id INTEGER PRIMARY KEY
+		)
+	`);
+    await knex.raw(`
+		CREATE TEMP TABLE xfer_ri_plan (
+			src_id INTEGER PRIMARY KEY,
+			action INTEGER NOT NULL,
+			dest_id INTEGER
+		)
+	`);
+    for (const table of TRANSFER_DICTIONARY_TABLES) {
+        await knex.raw(`CREATE TEMP TABLE xfer_need_${table} (src_id INTEGER PRIMARY KEY)`);
+        await knex.raw(`CREATE TEMP TABLE xfer_map_${table} (src_id INTEGER PRIMARY KEY, dest_id INTEGER NOT NULL)`);
+    }
+}

package/lib/transfer/dictionary-copy-specs.d.ts

@@ -0,0 +1,32 @@
+/** One content-addressable dictionary table's copy contract. */
+export interface DictionaryCopySpec {
+    /** Table name (same in both `main` and the attached source). */
+    readonly table: string;
+    /**
+     * Column(s) forming the natural (content-addressed) key used both as
+     * the `ON CONFLICT` target on insert and to re-resolve the
+     * destination id for every source row afterwards. This is
+     * DELIBERATELY not always the table's `UNIQUE` constraint used at
+     * write time — see `header_sets`' entry below.
+     */
+    readonly naturalKey: readonly string[];
+}
+/**
+ * Every dictionary table a transfer copies (concat and split alike), in
+ * dependency order (independent dictionaries first; `header_sets` before
+ * its `header_set_entries`/`header_flags` children — handled separately in
+ * `copy-header-set-children.ts` since they have no own id to map;
+ * `console_log_items` has no cross-dictionary FKs of
+ * its own, so ordering relative to the others does not matter).
+ *
+ * `header_sets`' natural key is `raw_hash` alone, not
+ * `(raw_json_hash, raw_hash)` — `raw_json_hash` is BLAKE3 of the raw
+ * `responseHeaders` JSON STRING (kept only for the pre-0.13 migration
+ * lookup, per `create-ref-tables.ts`'s docs) and can differ between two
+ * byte-for-byte-different JSON serialisations of the exact same header
+ * set (key order, whitespace), whereas `raw_hash` is BLAKE3 of the
+ * decomposed, sorted `name=value` pairs — the actual dedup identity every
+ * live write computes against. Two archives crawling the same site
+ * therefore share a `raw_hash` far more reliably than a `raw_json_hash`.
+ */
+export declare const DICTIONARY_COPY_SPECS: readonly DictionaryCopySpec[];

package/lib/transfer/dictionary-copy-specs.js

@@ -0,0 +1,28 @@
+/**
+ * Every dictionary table a transfer copies (concat and split alike), in
+ * dependency order (independent dictionaries first; `header_sets` before
+ * its `header_set_entries`/`header_flags` children — handled separately in
+ * `copy-header-set-children.ts` since they have no own id to map;
+ * `console_log_items` has no cross-dictionary FKs of
+ * its own, so ordering relative to the others does not matter).
+ *
+ * `header_sets`' natural key is `raw_hash` alone, not
+ * `(raw_json_hash, raw_hash)` — `raw_json_hash` is BLAKE3 of the raw
+ * `responseHeaders` JSON STRING (kept only for the pre-0.13 migration
+ * lookup, per `create-ref-tables.ts`'s docs) and can differ between two
+ * byte-for-byte-different JSON serialisations of the exact same header
+ * set (key order, whitespace), whereas `raw_hash` is BLAKE3 of the
+ * decomposed, sorted `name=value` pairs — the actual dedup identity every
+ * live write computes against. Two archives crawling the same site
+ * therefore share a `raw_hash` far more reliably than a `raw_json_hash`.
+ */
+export const DICTIONARY_COPY_SPECS = [
+    { table: 'url_refs', naturalKey: ['url'] },
+    { table: 'content_type_refs', naturalKey: ['raw'] },
+    { table: 'text_refs', naturalKey: ['hash', 'text'] },
+    { table: 'json_refs', naturalKey: ['hash'] },
+    { table: 'blob_refs', naturalKey: ['hash'] },
+    { table: 'header_name_refs', naturalKey: ['name'] },
+    { table: 'header_value_refs', naturalKey: ['hash', 'value'] },
+    { table: 'header_sets', naturalKey: ['raw_hash'] },
+];

package/lib/transfer/drop-transfer-temp-tables.d.ts

@@ -0,0 +1,9 @@
+import type { Knex } from 'knex';
+/**
+ * Drops every temp table
+ * {@link import('./create-transfer-temp-tables.js').createTransferTempTables}
+ * created, so the next source starts from a clean slate. Safe to call even
+ * if creation only partially succeeded (`IF EXISTS`).
+ * @param knex - Knex query builder connected to the destination archive DB.
+ */
+export declare function dropTransferTempTables(knex: Knex): Promise<void>;

package/lib/transfer/drop-transfer-temp-tables.js

@@ -0,0 +1,18 @@
+import { TRANSFER_DICTIONARY_TABLES } from './transfer-dictionary-tables.js';
+/**
+ * Drops every temp table
+ * {@link import('./create-transfer-temp-tables.js').createTransferTempTables}
+ * created, so the next source starts from a clean slate. Safe to call even
+ * if creation only partially succeeded (`IF EXISTS`).
+ * @param knex - Knex query builder connected to the destination archive DB.
+ */
+export async function dropTransferTempTables(knex) {
+    await knex.raw('DROP TABLE IF EXISTS xfer_ci_plan');
+    await knex.raw('DROP TABLE IF EXISTS xfer_ci_in_scope');
+    await knex.raw('DROP TABLE IF EXISTS xfer_ci_ref');
+    await knex.raw('DROP TABLE IF EXISTS xfer_ri_plan');
+    for (const table of TRANSFER_DICTIONARY_TABLES) {
+        await knex.raw(`DROP TABLE IF EXISTS xfer_need_${table}`);
+        await knex.raw(`DROP TABLE IF EXISTS xfer_map_${table}`);
+    }
+}

package/lib/transfer/flatten-redirect-chains.d.ts

@@ -0,0 +1,44 @@
+import type { Knex } from 'knex';
+/**
+ * Re-flattens `content_items.redirect_dest_id` to a single hop after
+ * concat, breaking any cycle it finds.
+ *
+ * Every source archive individually satisfies the "redirect destination
+ * is always the final hop" invariant (`linkRedirectSources` pre-flattens
+ * at write time), but merging two sources can reintroduce a chain: if
+ * source A has `P → X` (X external/unresolved in A) and source B's
+ * winning row for X is itself `X → Y`, the merged destination now has a
+ * two-hop `P → X → Y` — a shape no single crawl ever produces, and every
+ * reader in this codebase (`resolveAliasAndRedirectChain`,
+ * `list-links.ts`, …) assumes at most one hop.
+ *
+ * Runs a bounded loop: each pass sets every row's `redirect_dest_id` to
+ * its CURRENT target's `redirect_dest_id` when that target itself
+ * redirects further, stopping either when a pass changes zero rows or
+ * after `maxHops` passes (a cross-source cycle — A→B, B→A — would
+ * otherwise loop forever alternating between the two, since neither
+ * satisfies "my target has no further redirect"). Any row still part of
+ * a chain after the cap, or found to point at itself, is a cycle: its
+ * `redirect_dest_id` is set to `NULL` rather than left pointing at a
+ * still-chained row.
+ *
+ * This is whole-archive (not source-scoped) and split-safe: a single,
+ * not-yet-merged source is already flat (this function is still called
+ * for split too, for uniformity — every pass finds zero rows to change on
+ * the first iteration, a cheap no-op).
+ * @param trx - The destination transaction (no source ATTACHed — this
+ *   runs once, after every source has been transferred, not per-source).
+ * @param maxHops - Maximum flatten passes before treating any remaining
+ *   chain as a cycle. Defaults to {@link DEFAULT_MAX_HOPS}.
+ * @returns The number of rows flattened across every pass, and the number
+ *   of cycle rows NULLed out.
+ * @example
+ * ```ts
+ * for (const source of sources) { await transferArchiveRows(...); }
+ * const { flattened, cyclesBroken } = await flattenRedirectChains(knex);
+ * ```
+ */
+export declare function flattenRedirectChains(trx: Knex, maxHops?: number): Promise<{
+    flattened: number;
+    cyclesBroken: number;
+}>;

package/lib/transfer/flatten-redirect-chains.js

@@ -0,0 +1,74 @@
+/** Default cap on flatten iterations — see {@link flattenRedirectChains}'s docs. */
+const DEFAULT_MAX_HOPS = 32;
+/**
+ * Re-flattens `content_items.redirect_dest_id` to a single hop after
+ * concat, breaking any cycle it finds.
+ *
+ * Every source archive individually satisfies the "redirect destination
+ * is always the final hop" invariant (`linkRedirectSources` pre-flattens
+ * at write time), but merging two sources can reintroduce a chain: if
+ * source A has `P → X` (X external/unresolved in A) and source B's
+ * winning row for X is itself `X → Y`, the merged destination now has a
+ * two-hop `P → X → Y` — a shape no single crawl ever produces, and every
+ * reader in this codebase (`resolveAliasAndRedirectChain`,
+ * `list-links.ts`, …) assumes at most one hop.
+ *
+ * Runs a bounded loop: each pass sets every row's `redirect_dest_id` to
+ * its CURRENT target's `redirect_dest_id` when that target itself
+ * redirects further, stopping either when a pass changes zero rows or
+ * after `maxHops` passes (a cross-source cycle — A→B, B→A — would
+ * otherwise loop forever alternating between the two, since neither
+ * satisfies "my target has no further redirect"). Any row still part of
+ * a chain after the cap, or found to point at itself, is a cycle: its
+ * `redirect_dest_id` is set to `NULL` rather than left pointing at a
+ * still-chained row.
+ *
+ * This is whole-archive (not source-scoped) and split-safe: a single,
+ * not-yet-merged source is already flat (this function is still called
+ * for split too, for uniformity — every pass finds zero rows to change on
+ * the first iteration, a cheap no-op).
+ * @param trx - The destination transaction (no source ATTACHed — this
+ *   runs once, after every source has been transferred, not per-source).
+ * @param maxHops - Maximum flatten passes before treating any remaining
+ *   chain as a cycle. Defaults to {@link DEFAULT_MAX_HOPS}.
+ * @returns The number of rows flattened across every pass, and the number
+ *   of cycle rows NULLed out.
+ * @example
+ * ```ts
+ * for (const source of sources) { await transferArchiveRows(...); }
+ * const { flattened, cyclesBroken } = await flattenRedirectChains(knex);
+ * ```
+ */
+export async function flattenRedirectChains(trx, maxHops = DEFAULT_MAX_HOPS) {
+    let flattened = 0;
+    for (let hop = 0; hop < maxHops; hop++) {
+        const result = await trx.raw(`
+			UPDATE content_items
+			SET redirect_dest_id = t.redirect_dest_id
+			FROM content_items t
+			WHERE t.id = content_items.redirect_dest_id
+				AND t.redirect_dest_id IS NOT NULL
+				AND t.redirect_dest_id != content_items.id
+		`);
+        const changes = result?.changes ?? 0;
+        flattened += changes;
+        if (changes === 0) {
+            break;
+        }
+    }
+    const selfLoops = await trx
+        .select('id')
+        .from('content_items')
+        .whereRaw('redirect_dest_id = id');
+    const stillChained = await trx
+        .select('t1.id as id')
+        .from({ t1: 'content_items' })
+        .join({ t2: 'content_items' }, 't2.id', 't1.redirect_dest_id')
+        .whereNotNull('t1.redirect_dest_id')
+        .whereNotNull('t2.redirect_dest_id');
+    const cycleIds = [...new Set([...selfLoops, ...stillChained].map((r) => r.id))];
+    if (cycleIds.length > 0) {
+        await trx('content_items').whereIn('id', cycleIds).update({ redirect_dest_id: null });
+    }
+    return { flattened, cyclesBroken: cycleIds.length };
+}

package/lib/transfer/insert-content-items.d.ts

@@ -0,0 +1,35 @@
+import type { Knex } from 'knex';
+/**
+ * Inserts every `full`/`stub`-action `content_items` row from
+ * `temp.xfer_ci_plan` into the destination, then backfills each plan row's
+ * `dest_id` with the id the insert assigned.
+ *
+ * Every column is looked up via {@link listTransferColumns} (never a
+ * hand-written list — see that function's docs for why) and given an
+ * explicit SELECT expression by {@link columnExpression}: dictionary FK
+ * columns resolve through their `temp.xfer_map_*` table, `is_external`/
+ * `is_target` are forced for `stub` rows, and `redirect_dest_id`/
+ * `alias_of_id`/`dedupe_cap_event_id` are always written `NULL` (see
+ * {@link columnExpression}'s docs). Every other column passes through
+ * unchanged.
+ *
+ * Must run AFTER every dictionary copy this source needs
+ * (`copy-dictionaries-for-concat.ts` / `copy-dictionaries-for-split.ts`,
+ * `copy-header-set-children.ts`) — a `full`/`stub` row's `url_id` is NOT
+ * NULL, so a missing `temp.xfer_map_url_refs` entry surfaces immediately
+ * as a constraint violation rather than silently inserting garbage.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @param options - Options.
+ * @param options.crawlOrderOffset - Added to every copied row's
+ *   `crawl_order` (concat: the destination's current `MAX(crawl_order)`,
+ *   so a second source's pages sort after the first's; split: `0`, since
+ *   a single source's relative order is preserved as-is).
+ * @example
+ * ```ts
+ * await insertContentItems(trx, { crawlOrderOffset: 0 });
+ * ```
+ */
+export declare function insertContentItems(trx: Knex, options: {
+    crawlOrderOffset: number;
+}): Promise<void>;

package/lib/transfer/insert-content-items.js

@@ -0,0 +1,116 @@
+import { listTransferColumns } from './list-transfer-columns.js';
+import { quoteTransferIdentifier } from './quote-transfer-identifier.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+const SRC = TRANSFER_SOURCE_ALIAS;
+/**
+ * Builds the per-column SELECT expression for one `content_items` column,
+ * given its plan-driven remap rule.
+ * @param column - The destination column name.
+ * @param crawlOrderOffset - See {@link insertContentItems}.
+ */
+function columnExpression(column, crawlOrderOffset) {
+    switch (column) {
+        case 'url_id': {
+            return 'mu.dest_id';
+        }
+        case 'content_type_id': {
+            return 'mct.dest_id';
+        }
+        case 'header_set_id': {
+            return 'mhs.dest_id';
+        }
+        case 'redirect_dest_id':
+        case 'alias_of_id':
+        case 'dedupe_cap_event_id': {
+            // Never carried over verbatim: `redirect_dest_id` is remapped in a
+            // later pass once every source id has a destination id
+            // (`remap-redirect-dest-ids.ts`); `alias_of_id`/`dedupe_cap_event_id`
+            // are always recomputed from scratch by the mandatory viewer
+            // read-model rebuild's backfills, so writing a stale source value
+            // here would only be overwritten moments later.
+            return 'NULL';
+        }
+        case 'is_external': {
+            // A `stub` row is forced external regardless of what it was in the
+            // source archive (it may well have been a fully-scraped internal
+            // page there) — that IS the definition of "kept as a stub".
+            return `CASE WHEN p.action = ${TRANSFER_ACTION.stub} THEN 1 ELSE s.is_external END`;
+        }
+        case 'is_target': {
+            return `CASE WHEN p.action = ${TRANSFER_ACTION.stub} THEN 0 ELSE s.is_target END`;
+        }
+        case 'crawl_order': {
+            // `s.crawl_order + offset` is NULL-safe for free: SQLite's `+`
+            // against a NULL operand yields NULL, matching a source row whose
+            // `crawl_order` was never set (pre-#... archives, or rows written
+            // outside the normal crawl-order-assigning path).
+            return `s.crawl_order + ${crawlOrderOffset}`;
+        }
+        default: {
+            return `s.${quoteTransferIdentifier(column)}`;
+        }
+    }
+}
+/**
+ * Inserts every `full`/`stub`-action `content_items` row from
+ * `temp.xfer_ci_plan` into the destination, then backfills each plan row's
+ * `dest_id` with the id the insert assigned.
+ *
+ * Every column is looked up via {@link listTransferColumns} (never a
+ * hand-written list — see that function's docs for why) and given an
+ * explicit SELECT expression by {@link columnExpression}: dictionary FK
+ * columns resolve through their `temp.xfer_map_*` table, `is_external`/
+ * `is_target` are forced for `stub` rows, and `redirect_dest_id`/
+ * `alias_of_id`/`dedupe_cap_event_id` are always written `NULL` (see
+ * {@link columnExpression}'s docs). Every other column passes through
+ * unchanged.
+ *
+ * Must run AFTER every dictionary copy this source needs
+ * (`copy-dictionaries-for-concat.ts` / `copy-dictionaries-for-split.ts`,
+ * `copy-header-set-children.ts`) — a `full`/`stub` row's `url_id` is NOT
+ * NULL, so a missing `temp.xfer_map_url_refs` entry surfaces immediately
+ * as a constraint violation rather than silently inserting garbage.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @param options - Options.
+ * @param options.crawlOrderOffset - Added to every copied row's
+ *   `crawl_order` (concat: the destination's current `MAX(crawl_order)`,
+ *   so a second source's pages sort after the first's; split: `0`, since
+ *   a single source's relative order is preserved as-is).
+ * @example
+ * ```ts
+ * await insertContentItems(trx, { crawlOrderOffset: 0 });
+ * ```
+ */
+export async function insertContentItems(trx, options) {
+    const columns = await listTransferColumns(trx, SRC, 'content_items', ['id']);
+    const columnList = columns.map(quoteTransferIdentifier).join(', ');
+    const selectList = columns
+        .map((col) => columnExpression(col, options.crawlOrderOffset))
+        .join(', ');
+    await trx.raw(`
+		INSERT INTO main.content_items (${columnList})
+		SELECT ${selectList}
+		FROM ${SRC}.content_items s
+		JOIN xfer_ci_plan p ON p.src_id = s.id
+		LEFT JOIN xfer_map_url_refs mu ON mu.src_id = s.url_id
+		LEFT JOIN xfer_map_content_type_refs mct ON mct.src_id = s.content_type_id
+		LEFT JOIN xfer_map_header_sets mhs ON mhs.src_id = s.header_set_id
+		WHERE p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.stub})
+		ORDER BY s.id
+	`);
+    await trx.raw(`
+		UPDATE xfer_ci_plan
+		SET dest_id = x.dest_id
+		FROM (
+			SELECT p.src_id AS src_id, d.id AS dest_id
+			FROM xfer_ci_plan p
+			JOIN ${SRC}.content_items s ON s.id = p.src_id
+			JOIN xfer_map_url_refs mu ON mu.src_id = s.url_id
+			JOIN main.content_items d ON d.url_id = mu.dest_id
+			WHERE p.action IN (${TRANSFER_ACTION.full}, ${TRANSFER_ACTION.stub})
+		) x
+		WHERE xfer_ci_plan.src_id = x.src_id
+	`);
+}

package/lib/transfer/list-transfer-columns.d.ts

@@ -0,0 +1,35 @@
+import type { Knex } from 'knex';
+/**
+ * Lists a table's column names via SQLite's `pragma_table_info` table-valued
+ * function, in declaration order, minus any names in `exclude`.
+ *
+ * A transfer copy must NEVER build its column list from a hand-written
+ * literal or from `SELECT *` — migrated archives add columns via `ALTER
+ * TABLE ADD COLUMN`, which always appends at the end, so a fresh archive's
+ * `content_items` (say) can have `alias_of_id` earlier in declaration order
+ * than a migrated archive's copy of the same table. A positional `INSERT
+ * ... SELECT * FROM other_table` across two archives with different column
+ * orders would silently shift values into the wrong columns. Reading each
+ * side's own column list and building an explicit, named `SELECT
+ * <col1>, <col2>, ...` (this function's contract) sidesteps the problem
+ * entirely — column order never matters when every column is named.
+ *
+ * The pragma table-valued function must be schema-qualified as
+ * `<schema>.pragma_table_info(<table>)` — `pragma_table_info('schema.table')`
+ * silently returns zero rows instead of erroring, so `schema` and `table`
+ * are deliberately separate parameters rather than one dotted string a
+ * caller could get wrong.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param schema - The schema the table lives in: `'main'` for the
+ *   destination, or the attached source's alias (`TRANSFER_SOURCE_ALIAS`).
+ * @param table - Bare table name (no schema prefix).
+ * @param exclude - Column names to omit (typically the PK, which a copy
+ *   assigns fresh via `AUTOINCREMENT` rather than carrying over).
+ * @returns Column names in declaration order, minus `exclude`.
+ * @example
+ * ```ts
+ * const columns = await listTransferColumns(trx, 'xfer_src', 'content_items', ['id']);
+ * // ['url_id', 'is_external', 'scraped', ...] — in THIS archive's own order
+ * ```
+ */
+export declare function listTransferColumns(knex: Knex, schema: string, table: string, exclude?: readonly string[]): Promise<string[]>;

package/lib/transfer/list-transfer-columns.js

@@ -0,0 +1,45 @@
+/**
+ * Lists a table's column names via SQLite's `pragma_table_info` table-valued
+ * function, in declaration order, minus any names in `exclude`.
+ *
+ * A transfer copy must NEVER build its column list from a hand-written
+ * literal or from `SELECT *` — migrated archives add columns via `ALTER
+ * TABLE ADD COLUMN`, which always appends at the end, so a fresh archive's
+ * `content_items` (say) can have `alias_of_id` earlier in declaration order
+ * than a migrated archive's copy of the same table. A positional `INSERT
+ * ... SELECT * FROM other_table` across two archives with different column
+ * orders would silently shift values into the wrong columns. Reading each
+ * side's own column list and building an explicit, named `SELECT
+ * <col1>, <col2>, ...` (this function's contract) sidesteps the problem
+ * entirely — column order never matters when every column is named.
+ *
+ * The pragma table-valued function must be schema-qualified as
+ * `<schema>.pragma_table_info(<table>)` — `pragma_table_info('schema.table')`
+ * silently returns zero rows instead of erroring, so `schema` and `table`
+ * are deliberately separate parameters rather than one dotted string a
+ * caller could get wrong.
+ * @param knex - Knex query builder connected to the archive DB.
+ * @param schema - The schema the table lives in: `'main'` for the
+ *   destination, or the attached source's alias (`TRANSFER_SOURCE_ALIAS`).
+ * @param table - Bare table name (no schema prefix).
+ * @param exclude - Column names to omit (typically the PK, which a copy
+ *   assigns fresh via `AUTOINCREMENT` rather than carrying over).
+ * @returns Column names in declaration order, minus `exclude`.
+ * @example
+ * ```ts
+ * const columns = await listTransferColumns(trx, 'xfer_src', 'content_items', ['id']);
+ * // ['url_id', 'is_external', 'scraped', ...] — in THIS archive's own order
+ * ```
+ */
+export async function listTransferColumns(knex, schema, table, exclude = []) {
+    const excludeSet = new Set(exclude);
+    const rows = await knex
+        .select('name')
+        .from(knex.raw(`${schema}.pragma_table_info(?)`, [table]))
+        .orderBy('cid');
+    const columns = rows.map((row) => row.name).filter((name) => !excludeSet.has(name));
+    if (columns.length === 0) {
+        throw new Error(`No columns found for table "${schema}.${table}" — does it exist?`);
+    }
+    return columns;
+}

package/lib/transfer/mark-in-scope-content-items.d.ts

@@ -0,0 +1,31 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+import type { Knex } from 'knex';
+/**
+ * Determines which of the source's internal (`is_external = 0`)
+ * `content_items` rows fall inside `scope` (split's new, narrower root
+ * set), writing matching ids into `temp.xfer_ci_in_scope`.
+ *
+ * Mirrors `repromoteExternalPages`'s shape (load every candidate URL into
+ * memory, classify with {@link findScopeEntry} in JS, write matches back
+ * in bound-count-safe chunks) — the same accepted pattern for a one-off,
+ * whole-archive scope reclassification. External rows are never
+ * candidates here: split's scope check only ever needs to know "was this
+ * an internal page that is now out of scope", since the "external
+ * everywhere but in scope" case is a concat-only concept (split has a
+ * single source, so there is nothing to promote).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./create-transfer-temp-tables.js').createTransferTempTables}.
+ * @param scope - The new (narrower) scope map, from
+ *   {@link import('../scope/build-scope-map.js').buildScopeMap} over
+ *   the split's scope URLs.
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}
+ *   (the source archive's own `disableQueries`, typically).
+ * @returns The number of ids written to `temp.xfer_ci_in_scope`.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(['https://example.com/blog/']);
+ * const inScopeCount = await markInScopeContentItems(trx, scope, sourceConfig);
+ * ```
+ */
+export declare function markInScopeContentItems(trx: Knex, scope: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<number>;

package/lib/transfer/mark-in-scope-content-items.js

@@ -0,0 +1,54 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { findScopeEntry } from '../scope/find-scope-entry.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Determines which of the source's internal (`is_external = 0`)
+ * `content_items` rows fall inside `scope` (split's new, narrower root
+ * set), writing matching ids into `temp.xfer_ci_in_scope`.
+ *
+ * Mirrors `repromoteExternalPages`'s shape (load every candidate URL into
+ * memory, classify with {@link findScopeEntry} in JS, write matches back
+ * in bound-count-safe chunks) — the same accepted pattern for a one-off,
+ * whole-archive scope reclassification. External rows are never
+ * candidates here: split's scope check only ever needs to know "was this
+ * an internal page that is now out of scope", since the "external
+ * everywhere but in scope" case is a concat-only concept (split has a
+ * single source, so there is nothing to promote).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./create-transfer-temp-tables.js').createTransferTempTables}.
+ * @param scope - The new (narrower) scope map, from
+ *   {@link import('../scope/build-scope-map.js').buildScopeMap} over
+ *   the split's scope URLs.
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}
+ *   (the source archive's own `disableQueries`, typically).
+ * @returns The number of ids written to `temp.xfer_ci_in_scope`.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(['https://example.com/blog/']);
+ * const inScopeCount = await markInScopeContentItems(trx, scope, sourceConfig);
+ * ```
+ */
+export async function markInScopeContentItems(trx, scope, options) {
+    const candidates = await trx
+        .select(`${TRANSFER_SOURCE_ALIAS}.content_items.id as id`, 'url_refs.url as url')
+        .from(`${TRANSFER_SOURCE_ALIAS}.content_items`)
+        .join({ url_refs: `${TRANSFER_SOURCE_ALIAS}.url_refs` }, 'url_refs.id', `${TRANSFER_SOURCE_ALIAS}.content_items.url_id`)
+        .where(`${TRANSFER_SOURCE_ALIAS}.content_items.is_external`, 0);
+    const inScopeIds = [];
+    for (const row of candidates) {
+        const parsed = parseUrl(row.url, options);
+        if (!parsed) {
+            continue;
+        }
+        if (findScopeEntry(parsed, scope, options) !== null) {
+            inScopeIds.push(row.id);
+        }
+    }
+    const chunkSize = 500;
+    for (let i = 0; i < inScopeIds.length; i += chunkSize) {
+        const chunk = inScopeIds.slice(i, i + chunkSize);
+        await trx.insert(chunk.map((id) => ({ src_id: id }))).into('xfer_ci_in_scope');
+    }
+    return inScopeIds.length;
+}