@nitpicker/archive 0.0.0 → 0.25.1

package/lib/transfer/merge-archive-configs.d.ts

@@ -0,0 +1,51 @@
+import type { Config } from '../types.js';
+/**
+ * Merges two or more source archives' {@link Config} into the config
+ * concat's output archive is created with.
+ *
+ * - `disableQueries` and `fromList` must be IDENTICAL across every
+ *   source, or this throws {@link ArchiveConfigConflictError}:
+ *   - `disableQueries` changes how every URL is normalised
+ *     (`normalizeArchiveUrl`'s `ExURL.withoutHashAndAuth` vs. a
+ *     query-stripped form) — sources crawled with different settings
+ *     have incomparable `url_refs.url` values, so URL-string matching
+ *     (`plan-content-items-for-concat.ts`'s whole mechanism) would
+ *     silently misclassify identical URLs as distinct.
+ *   - `fromList` archives restrict which internal rows the viewer read
+ *     model admits to exactly `config.roots` (`computeFromListAllowedPageIds`) —
+ *     mixing a `fromList` source with a recursive one has no single
+ *     correct restriction rule.
+ * - `roots` is the union of every source's roots, in argument order,
+ *   deduplicated (a root appearing in two sources counts once).
+ *   `baseUrl` is the merged `roots[0]`.
+ * - `excludes` / `excludeKeywords` / `excludeUrls` are unioned the same
+ *   way, order-preserving.
+ * - `requestHeaderNames` is unioned too: it only records which header names
+ *   ANY source needed (never values), so dropping a later source's names would
+ *   silence the "re-supply these headers" warning for that source's pages.
+ * - Every other scalar field (`recursive`, `interval`, `image`,
+ *   `fetchExternal`, `parallels`, `maxExcludedDepth`, `retry`,
+ *   `userAgent`, `ignoreRobots`, `mainContentSelector`) is taken from the
+ *   FIRST source — these only ever describe how a crawl session was RUN,
+ *   never the shape of the data, so there is no principled way to merge
+ *   two different values and "first argument wins" is at least
+ *   predictable.
+ * - `version` is always {@link REQUIRED_FORMAT_VERSION} (the output is a
+ *   freshly-created archive on the current format) and `createdCwd` is
+ *   always `null` (never persisted into a shared `.nitpicker`, per
+ *   `Archive.write()`'s own scrub — see the DB schema doc's `info.createdCwd`
+ *   entry).
+ * @param configs - Every source's `getConfig()` result, in argument order.
+ * @param name - The output archive's `name` (typically its basename).
+ * @returns The merged {@link Config} to write via `Archive#setConfig`.
+ * @throws {ArchiveConfigConflictError} If `disableQueries` or `fromList`
+ *   differ across `configs`.
+ * @example
+ * ```ts
+ * const merged = mergeArchiveConfigs(
+ *   [await a.getConfig(), await b.getConfig()],
+ *   'merged',
+ * );
+ * ```
+ */
+export declare function mergeArchiveConfigs(configs: readonly Config[], name: string): Config;

package/lib/transfer/merge-archive-configs.js

@@ -0,0 +1,85 @@
+import { REQUIRED_FORMAT_VERSION } from '../meta/assert-compatible-version.js';
+import { ArchiveConfigConflictError } from './types.js';
+/** {@link Config} fields that must be IDENTICAL across every source, or concat refuses. */
+const STRICT_MATCH_FIELDS = ['disableQueries', 'fromList'];
+/**
+ * Merges two or more source archives' {@link Config} into the config
+ * concat's output archive is created with.
+ *
+ * - `disableQueries` and `fromList` must be IDENTICAL across every
+ *   source, or this throws {@link ArchiveConfigConflictError}:
+ *   - `disableQueries` changes how every URL is normalised
+ *     (`normalizeArchiveUrl`'s `ExURL.withoutHashAndAuth` vs. a
+ *     query-stripped form) — sources crawled with different settings
+ *     have incomparable `url_refs.url` values, so URL-string matching
+ *     (`plan-content-items-for-concat.ts`'s whole mechanism) would
+ *     silently misclassify identical URLs as distinct.
+ *   - `fromList` archives restrict which internal rows the viewer read
+ *     model admits to exactly `config.roots` (`computeFromListAllowedPageIds`) —
+ *     mixing a `fromList` source with a recursive one has no single
+ *     correct restriction rule.
+ * - `roots` is the union of every source's roots, in argument order,
+ *   deduplicated (a root appearing in two sources counts once).
+ *   `baseUrl` is the merged `roots[0]`.
+ * - `excludes` / `excludeKeywords` / `excludeUrls` are unioned the same
+ *   way, order-preserving.
+ * - `requestHeaderNames` is unioned too: it only records which header names
+ *   ANY source needed (never values), so dropping a later source's names would
+ *   silence the "re-supply these headers" warning for that source's pages.
+ * - Every other scalar field (`recursive`, `interval`, `image`,
+ *   `fetchExternal`, `parallels`, `maxExcludedDepth`, `retry`,
+ *   `userAgent`, `ignoreRobots`, `mainContentSelector`) is taken from the
+ *   FIRST source — these only ever describe how a crawl session was RUN,
+ *   never the shape of the data, so there is no principled way to merge
+ *   two different values and "first argument wins" is at least
+ *   predictable.
+ * - `version` is always {@link REQUIRED_FORMAT_VERSION} (the output is a
+ *   freshly-created archive on the current format) and `createdCwd` is
+ *   always `null` (never persisted into a shared `.nitpicker`, per
+ *   `Archive.write()`'s own scrub — see the DB schema doc's `info.createdCwd`
+ *   entry).
+ * @param configs - Every source's `getConfig()` result, in argument order.
+ * @param name - The output archive's `name` (typically its basename).
+ * @returns The merged {@link Config} to write via `Archive#setConfig`.
+ * @throws {ArchiveConfigConflictError} If `disableQueries` or `fromList`
+ *   differ across `configs`.
+ * @example
+ * ```ts
+ * const merged = mergeArchiveConfigs(
+ *   [await a.getConfig(), await b.getConfig()],
+ *   'merged',
+ * );
+ * ```
+ */
+export function mergeArchiveConfigs(configs, name) {
+    if (configs.length === 0) {
+        throw new Error('mergeArchiveConfigs: at least one source config is required');
+    }
+    for (const field of STRICT_MATCH_FIELDS) {
+        const values = configs.map((c) => c[field]);
+        const distinct = new Set(values);
+        if (distinct.size > 1) {
+            throw new ArchiveConfigConflictError(field, values);
+        }
+    }
+    const roots = [...new Set(configs.flatMap((c) => c.roots))];
+    const excludes = [...new Set(configs.flatMap((c) => c.excludes))];
+    const excludeKeywords = [...new Set(configs.flatMap((c) => c.excludeKeywords))];
+    const excludeUrls = [...new Set(configs.flatMap((c) => c.excludeUrls))];
+    const requestHeaderNames = [
+        ...new Set(configs.flatMap((c) => c.requestHeaderNames ?? [])),
+    ];
+    const first = configs[0];
+    return {
+        ...first,
+        version: REQUIRED_FORMAT_VERSION,
+        name,
+        roots,
+        baseUrl: roots[0],
+        excludes,
+        excludeKeywords,
+        excludeUrls,
+        requestHeaderNames,
+        createdCwd: null,
+    };
+}

package/lib/transfer/plan-content-items-for-concat.d.ts

@@ -0,0 +1,33 @@
+import type { Knex } from 'knex';
+/**
+ * Populates `temp.xfer_ci_plan` for one concat source: every source
+ * `content_items` row is matched against the destination by URL string
+ * (`url_refs.url`, not `url_id` — the two archives assign unrelated ids to
+ * the same URL) and classified as:
+ *
+ * - `full` — no destination row exists yet for this URL.
+ * - `replace` — a destination row exists, and this source's rank
+ *   ({@link contentItemRankSql}) is greater than or equal to the
+ *   destination's. `>=` (not `>`) is deliberate: sources are merged one at
+ *   a time in argument order, always compared against the CURRENT
+ *   destination state, so an equal rank still overwrites — this is what
+ *   makes "ties go to the later argument" fall out naturally, with no
+ *   extra tie-breaking logic needed.
+ * - `skip` — a destination row exists and outranks this source's row; the
+ *   destination's existing (better) observation is left untouched.
+ *
+ * Every source row gets a plan entry (concat never drops a row outright —
+ * that only happens in split, where a row can be out of the new scope AND
+ * unreferenced). `dest_id` is left `NULL` for `full` rows here; it is
+ * filled in once {@link import('./insert-content-items.js').insertContentItems}
+ * assigns the new destination id.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./create-transfer-temp-tables.js').createTransferTempTables}.
+ * @example
+ * ```ts
+ * await planContentItemsForConcat(trx);
+ * const { full, replace, skip } = await countPlanActions(trx); // for progress reporting
+ * ```
+ */
+export declare function planContentItemsForConcat(trx: Knex): Promise<void>;

package/lib/transfer/plan-content-items-for-concat.js

@@ -0,0 +1,51 @@
+import { contentItemRankSql } from './content-item-rank-sql.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Populates `temp.xfer_ci_plan` for one concat source: every source
+ * `content_items` row is matched against the destination by URL string
+ * (`url_refs.url`, not `url_id` — the two archives assign unrelated ids to
+ * the same URL) and classified as:
+ *
+ * - `full` — no destination row exists yet for this URL.
+ * - `replace` — a destination row exists, and this source's rank
+ *   ({@link contentItemRankSql}) is greater than or equal to the
+ *   destination's. `>=` (not `>`) is deliberate: sources are merged one at
+ *   a time in argument order, always compared against the CURRENT
+ *   destination state, so an equal rank still overwrites — this is what
+ *   makes "ties go to the later argument" fall out naturally, with no
+ *   extra tie-breaking logic needed.
+ * - `skip` — a destination row exists and outranks this source's row; the
+ *   destination's existing (better) observation is left untouched.
+ *
+ * Every source row gets a plan entry (concat never drops a row outright —
+ * that only happens in split, where a row can be out of the new scope AND
+ * unreferenced). `dest_id` is left `NULL` for `full` rows here; it is
+ * filled in once {@link import('./insert-content-items.js').insertContentItems}
+ * assigns the new destination id.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./create-transfer-temp-tables.js').createTransferTempTables}.
+ * @example
+ * ```ts
+ * await planContentItemsForConcat(trx);
+ * const { full, replace, skip } = await countPlanActions(trx); // for progress reporting
+ * ```
+ */
+export async function planContentItemsForConcat(trx) {
+    await trx.raw(`
+		INSERT INTO xfer_ci_plan (src_id, action, dest_id)
+		SELECT
+			s.id,
+			CASE
+				WHEN d.id IS NULL THEN ${TRANSFER_ACTION.full}
+				WHEN ${contentItemRankSql('s')} >= ${contentItemRankSql('d')} THEN ${TRANSFER_ACTION.replace}
+				ELSE ${TRANSFER_ACTION.skip}
+			END,
+			d.id
+		FROM ${TRANSFER_SOURCE_ALIAS}.content_items s
+		JOIN ${TRANSFER_SOURCE_ALIAS}.url_refs su ON su.id = s.url_id
+		LEFT JOIN main.url_refs du ON du.url = su.url
+		LEFT JOIN main.content_items d ON d.url_id = du.id
+	`);
+}

package/lib/transfer/plan-content-items-for-split.d.ts

@@ -0,0 +1,42 @@
+import type { Knex } from 'knex';
+/**
+ * Populates `temp.xfer_ci_plan` for split, in three passes over the
+ * source's `content_items`:
+ *
+ * 1. **Core** — every id `temp.xfer_ci_in_scope` marked (in-scope,
+ *    internal — see
+ *    {@link import('./mark-in-scope-content-items.js').markInScopeContentItems})
+ *    is kept in `full`.
+ * 2. **Redirect-destination promotion** — a core row's `redirect_dest_id`,
+ *    when it points at an INTERNAL row, joins core as `full` too, even if
+ *    that destination's own path falls outside the split scope. This is
+ *    the same invariant `insertPage` documents: "the redirect destination
+ *    of an in-scope request is recorded internal even when its path is out
+ *    of scope" (a soft-404 landing page is the canonical example) — split
+ *    must not treat that page differently than a live crawl would. Only
+ *    ONE hop is honoured here, matching `redirect_dest_id`'s own
+ *    contract: crawler writes always pre-flatten a redirect chain to its
+ *    final destination at write time, so a single JOIN already reaches the
+ *    terminal page — there is no multi-hop chain left to walk in a
+ *    single, not-yet-merged source archive (concat, which CAN reintroduce
+ *    a multi-hop chain by combining two sources, re-flattens separately —
+ *    see `flatten-redirect-chains.ts`).
+ * 3. **Referenced-row stubbing** — any row NOT in core that a core row's
+ *    `anchor_edges` targets, or that a core row's `redirect_dest_id`
+ *    points at (now only the EXTERNAL case survives this filter, since
+ *    every internal destination was already absorbed into core by pass
+ *    2), is kept as a `stub`: the `content_items` row only, forced
+ *    external — see `insert-content-items.ts`. A row reachable by neither
+ *    path is left out of the plan entirely, which is split's definition
+ *    of "dropped".
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./mark-in-scope-content-items.js').markInScopeContentItems}
+ *   has populated `temp.xfer_ci_in_scope`.
+ * @example
+ * ```ts
+ * await markInScopeContentItems(trx, scope, sourceConfig);
+ * await planContentItemsForSplit(trx);
+ * ```
+ */
+export declare function planContentItemsForSplit(trx: Knex): Promise<void>;

package/lib/transfer/plan-content-items-for-split.js

@@ -0,0 +1,83 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Populates `temp.xfer_ci_plan` for split, in three passes over the
+ * source's `content_items`:
+ *
+ * 1. **Core** — every id `temp.xfer_ci_in_scope` marked (in-scope,
+ *    internal — see
+ *    {@link import('./mark-in-scope-content-items.js').markInScopeContentItems})
+ *    is kept in `full`.
+ * 2. **Redirect-destination promotion** — a core row's `redirect_dest_id`,
+ *    when it points at an INTERNAL row, joins core as `full` too, even if
+ *    that destination's own path falls outside the split scope. This is
+ *    the same invariant `insertPage` documents: "the redirect destination
+ *    of an in-scope request is recorded internal even when its path is out
+ *    of scope" (a soft-404 landing page is the canonical example) — split
+ *    must not treat that page differently than a live crawl would. Only
+ *    ONE hop is honoured here, matching `redirect_dest_id`'s own
+ *    contract: crawler writes always pre-flatten a redirect chain to its
+ *    final destination at write time, so a single JOIN already reaches the
+ *    terminal page — there is no multi-hop chain left to walk in a
+ *    single, not-yet-merged source archive (concat, which CAN reintroduce
+ *    a multi-hop chain by combining two sources, re-flattens separately —
+ *    see `flatten-redirect-chains.ts`).
+ * 3. **Referenced-row stubbing** — any row NOT in core that a core row's
+ *    `anchor_edges` targets, or that a core row's `redirect_dest_id`
+ *    points at (now only the EXTERNAL case survives this filter, since
+ *    every internal destination was already absorbed into core by pass
+ *    2), is kept as a `stub`: the `content_items` row only, forced
+ *    external — see `insert-content-items.ts`. A row reachable by neither
+ *    path is left out of the plan entirely, which is split's definition
+ *    of "dropped".
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. Must run after
+ *   {@link import('./mark-in-scope-content-items.js').markInScopeContentItems}
+ *   has populated `temp.xfer_ci_in_scope`.
+ * @example
+ * ```ts
+ * await markInScopeContentItems(trx, scope, sourceConfig);
+ * await planContentItemsForSplit(trx);
+ * ```
+ */
+export async function planContentItemsForSplit(trx) {
+    await trx.raw(`
+		INSERT INTO xfer_ci_plan (src_id, action)
+		SELECT src_id, ${TRANSFER_ACTION.full} FROM xfer_ci_in_scope
+	`);
+    await trx.raw(`
+		INSERT INTO xfer_ci_plan (src_id, action)
+		SELECT DISTINCT d.id, ${TRANSFER_ACTION.full}
+		FROM ${TRANSFER_SOURCE_ALIAS}.content_items s
+		JOIN xfer_ci_plan p ON p.src_id = s.id
+		JOIN ${TRANSFER_SOURCE_ALIAS}.content_items d ON d.id = s.redirect_dest_id
+		WHERE d.is_external = 0
+			AND d.id NOT IN (SELECT src_id FROM xfer_ci_plan)
+	`);
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_ci_ref (src_id)
+		SELECT DISTINCT ae.href_page_id
+		FROM ${TRANSFER_SOURCE_ALIAS}.anchor_edges ae
+		JOIN xfer_ci_plan p ON p.src_id = ae.page_id
+		WHERE ae.href_page_id NOT IN (SELECT src_id FROM xfer_ci_plan)
+	`);
+    // `OR IGNORE` on BOTH inserts (not just this one) is deliberate, not
+    // redundant: this table's PK is `src_id`, and an anchor target can also
+    // be a redirect target of some other core row, so the two passes can
+    // legitimately overlap. The first pass alone cannot collide with itself
+    // (`SELECT DISTINCT` on an empty destination table), but relying on
+    // that ordering to justify leaving it unguarded would silently break if
+    // a third insert into `xfer_ci_ref` were ever added ahead of it.
+    await trx.raw(`
+		INSERT OR IGNORE INTO xfer_ci_ref (src_id)
+		SELECT DISTINCT s.redirect_dest_id
+		FROM ${TRANSFER_SOURCE_ALIAS}.content_items s
+		JOIN xfer_ci_plan p ON p.src_id = s.id
+		WHERE s.redirect_dest_id IS NOT NULL
+			AND s.redirect_dest_id NOT IN (SELECT src_id FROM xfer_ci_plan)
+	`);
+    await trx.raw(`
+		INSERT INTO xfer_ci_plan (src_id, action)
+		SELECT src_id, ${TRANSFER_ACTION.stub} FROM xfer_ci_ref
+	`);
+}

package/lib/transfer/plan-resource-items.d.ts

@@ -0,0 +1,40 @@
+import type { Knex } from 'knex';
+/**
+ * Populates `temp.xfer_ri_plan` for `resource_items`, one call per source,
+ * for either transfer mode:
+ *
+ * - **`concat`** — every source resource is matched against the
+ *   destination by its identity (`url_id` resolved through `url_refs.url`,
+ *   or `url_blob_id` resolved through `blob_refs.hash` — the two are
+ *   mutually exclusive per `resource_items`' own CHECK constraint, so at
+ *   most one of the two LEFT JOINs below ever produces a match for a given
+ *   row) and classified `full` (no destination row yet), `replace`
+ *   (destination exists but has never recorded a response — `status IS
+ *   NULL` — while this source has), or `skip` (destination already has a
+ *   recorded response). Matching happens directly against raw
+ *   `url`/`hash` values, not through `temp.xfer_map_url_refs`/
+ *   `xfer_map_blob_refs` — those dictionary maps are built AFTER this
+ *   plan (`copy-dictionaries-for-concat.ts` needs to know which resources
+ *   are kept before it can compute needed dictionary ids for split), so
+ *   this plan cannot depend on them.
+ * - **`split`** — a resource is kept (`full`) only when at least one
+ *   `full`-action page in `temp.xfer_ci_plan` references it via
+ *   `resource_ref_edges`. A resource with no surviving referrer is left
+ *   out of the plan (dropped) — split does not preserve an
+ *   otherwise-orphan resource just because its URL happens to still be
+ *   in scope (accepted simplification: a resource with zero referring
+ *   pages has no page context left to display it against in the output
+ *   archive anyway).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. For `'split'`, must run after
+ *   `temp.xfer_ci_plan` has been fully populated (every `full` page
+ *   decided) — see
+ *   {@link import('./plan-content-items-for-split.js').planContentItemsForSplit}.
+ * @param mode - Which transfer this plan is for.
+ * @example
+ * ```ts
+ * await planContentItemsForSplit(trx);
+ * await planResourceItems(trx, 'split');
+ * ```
+ */
+export declare function planResourceItems(trx: Knex, mode: 'concat' | 'split'): Promise<void>;

package/lib/transfer/plan-resource-items.js

@@ -0,0 +1,69 @@
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Populates `temp.xfer_ri_plan` for `resource_items`, one call per source,
+ * for either transfer mode:
+ *
+ * - **`concat`** — every source resource is matched against the
+ *   destination by its identity (`url_id` resolved through `url_refs.url`,
+ *   or `url_blob_id` resolved through `blob_refs.hash` — the two are
+ *   mutually exclusive per `resource_items`' own CHECK constraint, so at
+ *   most one of the two LEFT JOINs below ever produces a match for a given
+ *   row) and classified `full` (no destination row yet), `replace`
+ *   (destination exists but has never recorded a response — `status IS
+ *   NULL` — while this source has), or `skip` (destination already has a
+ *   recorded response). Matching happens directly against raw
+ *   `url`/`hash` values, not through `temp.xfer_map_url_refs`/
+ *   `xfer_map_blob_refs` — those dictionary maps are built AFTER this
+ *   plan (`copy-dictionaries-for-concat.ts` needs to know which resources
+ *   are kept before it can compute needed dictionary ids for split), so
+ *   this plan cannot depend on them.
+ * - **`split`** — a resource is kept (`full`) only when at least one
+ *   `full`-action page in `temp.xfer_ci_plan` references it via
+ *   `resource_ref_edges`. A resource with no surviving referrer is left
+ *   out of the plan (dropped) — split does not preserve an
+ *   otherwise-orphan resource just because its URL happens to still be
+ *   in scope (accepted simplification: a resource with zero referring
+ *   pages has no page context left to display it against in the output
+ *   archive anyway).
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}. For `'split'`, must run after
+ *   `temp.xfer_ci_plan` has been fully populated (every `full` page
+ *   decided) — see
+ *   {@link import('./plan-content-items-for-split.js').planContentItemsForSplit}.
+ * @param mode - Which transfer this plan is for.
+ * @example
+ * ```ts
+ * await planContentItemsForSplit(trx);
+ * await planResourceItems(trx, 'split');
+ * ```
+ */
+export async function planResourceItems(trx, mode) {
+    if (mode === 'split') {
+        await trx.raw(`
+			INSERT INTO xfer_ri_plan (src_id, action)
+			SELECT DISTINCT r.id, ${TRANSFER_ACTION.full}
+			FROM ${TRANSFER_SOURCE_ALIAS}.resource_items r
+			JOIN ${TRANSFER_SOURCE_ALIAS}.resource_ref_edges e ON e.resource_id = r.id
+			JOIN xfer_ci_plan p ON p.src_id = e.page_id AND p.action = ${TRANSFER_ACTION.full}
+		`);
+        return;
+    }
+    await trx.raw(`
+		INSERT INTO xfer_ri_plan (src_id, action, dest_id)
+		SELECT
+			s.id,
+			CASE
+				WHEN d.id IS NULL THEN ${TRANSFER_ACTION.full}
+				WHEN (s.status IS NOT NULL) >= (d.status IS NOT NULL) THEN ${TRANSFER_ACTION.replace}
+				ELSE ${TRANSFER_ACTION.skip}
+			END,
+			d.id
+		FROM ${TRANSFER_SOURCE_ALIAS}.resource_items s
+		LEFT JOIN ${TRANSFER_SOURCE_ALIAS}.url_refs su ON su.id = s.url_id
+		LEFT JOIN main.url_refs du ON du.url = su.url
+		LEFT JOIN ${TRANSFER_SOURCE_ALIAS}.blob_refs sb ON sb.id = s.url_blob_id
+		LEFT JOIN main.blob_refs db ON db.hash = sb.hash
+		LEFT JOIN main.resource_items d ON d.url_id = du.id OR d.url_blob_id = db.id
+	`);
+}

package/lib/transfer/quote-transfer-identifier.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Double-quotes a bare SQL identifier (`"order"`, not `order`) so a column
+ * name that happens to collide with a SQLite keyword (`order`, `key`,
+ * `references`, …) never breaks a transfer statement built by
+ * interpolating {@link import('./list-transfer-columns.js').listTransferColumns}'
+ * output directly into raw SQL — every generic (not hand-written)
+ * column-list builder in `archive/transfer/` must quote through this
+ * function rather than splicing bare names.
+ * @param name - A bare, trusted column or table identifier (never
+ *   user-supplied — every caller sources `name` from `pragma_table_info`
+ *   against this codebase's own fixed schema).
+ * @returns The identifier wrapped in double quotes.
+ * @example
+ * ```ts
+ * columns.map(quoteTransferIdentifier).join(', ') // '"id", "order", "text"'
+ * ```
+ */
+export declare function quoteTransferIdentifier(name: string): string;

package/lib/transfer/quote-transfer-identifier.js

@@ -0,0 +1,20 @@
+/**
+ * Double-quotes a bare SQL identifier (`"order"`, not `order`) so a column
+ * name that happens to collide with a SQLite keyword (`order`, `key`,
+ * `references`, …) never breaks a transfer statement built by
+ * interpolating {@link import('./list-transfer-columns.js').listTransferColumns}'
+ * output directly into raw SQL — every generic (not hand-written)
+ * column-list builder in `archive/transfer/` must quote through this
+ * function rather than splicing bare names.
+ * @param name - A bare, trusted column or table identifier (never
+ *   user-supplied — every caller sources `name` from `pragma_table_info`
+ *   against this codebase's own fixed schema).
+ * @returns The identifier wrapped in double quotes.
+ * @example
+ * ```ts
+ * columns.map(quoteTransferIdentifier).join(', ') // '"id", "order", "text"'
+ * ```
+ */
+export function quoteTransferIdentifier(name) {
+    return `"${name.replaceAll('"', '""')}"`;
+}

package/lib/transfer/reclassify-resource-externality.d.ts

@@ -0,0 +1,37 @@
+import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
+import type { Knex } from 'knex';
+/**
+ * Recomputes `resource_items.is_external` for the WHOLE destination
+ * archive against its final (merged or narrowed) scope, once every source
+ * has been transferred.
+ *
+ * Unlike `content_items.is_external` ("was this taken on as a crawl
+ * target" — a write-time fact this transfer never re-derives, per
+ * ARCHITECTURE.md's invariant), `resource_items.is_external` is a pure
+ * scope check computed the same way at crawl time
+ * (`@nitpicker/crawler`'s `Crawler#handleResources`:
+ * `findScopeEntry(url, this.#scope) === null`). Concat's merged scope is a union of every source's roots,
+ * so a resource one source recorded as external can legitimately become
+ * internal-by-scope in the output even though nothing about the resource
+ * itself changed — recomputing this is the resource-table equivalent of
+ * what `content_items` deliberately does NOT do, and the two must not be
+ * "unified" into the same rule.
+ *
+ * Mirrors `repromoteExternalPages`'s shape (load every resource's URL,
+ * classify in JS, write matches back in bound-safe chunks) rather than a
+ * single SQL scope join — `resource_items.url_id` is nullable
+ * (`url_blob_id` rows have no URL to classify and are left as recorded),
+ * so the candidate set is every resource with a non-null `url_id`.
+ * @param trx - The destination transaction (no source ATTACHed).
+ * @param scope - The output archive's final scope map
+ *   (`buildScopeMap(mergedOrNarrowedRoots, config)`).
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}
+ *   (the output archive's own `disableQueries`).
+ * @returns The number of resources whose `is_external` flipped.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(config.roots, config);
+ * const flipped = await reclassifyResourceExternality(knex, scope, config);
+ * ```
+ */
+export declare function reclassifyResourceExternality(trx: Knex, scope: ReadonlyMap<string, readonly ExURL[]>, options?: ParseURLOptions): Promise<number>;

package/lib/transfer/reclassify-resource-externality.js

@@ -0,0 +1,66 @@
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { findScopeEntry } from '../scope/find-scope-entry.js';
+/**
+ * Recomputes `resource_items.is_external` for the WHOLE destination
+ * archive against its final (merged or narrowed) scope, once every source
+ * has been transferred.
+ *
+ * Unlike `content_items.is_external` ("was this taken on as a crawl
+ * target" — a write-time fact this transfer never re-derives, per
+ * ARCHITECTURE.md's invariant), `resource_items.is_external` is a pure
+ * scope check computed the same way at crawl time
+ * (`@nitpicker/crawler`'s `Crawler#handleResources`:
+ * `findScopeEntry(url, this.#scope) === null`). Concat's merged scope is a union of every source's roots,
+ * so a resource one source recorded as external can legitimately become
+ * internal-by-scope in the output even though nothing about the resource
+ * itself changed — recomputing this is the resource-table equivalent of
+ * what `content_items` deliberately does NOT do, and the two must not be
+ * "unified" into the same rule.
+ *
+ * Mirrors `repromoteExternalPages`'s shape (load every resource's URL,
+ * classify in JS, write matches back in bound-safe chunks) rather than a
+ * single SQL scope join — `resource_items.url_id` is nullable
+ * (`url_blob_id` rows have no URL to classify and are left as recorded),
+ * so the candidate set is every resource with a non-null `url_id`.
+ * @param trx - The destination transaction (no source ATTACHed).
+ * @param scope - The output archive's final scope map
+ *   (`buildScopeMap(mergedOrNarrowedRoots, config)`).
+ * @param options - URL parsing options forwarded to {@link findScopeEntry}
+ *   (the output archive's own `disableQueries`).
+ * @returns The number of resources whose `is_external` flipped.
+ * @example
+ * ```ts
+ * const scope = buildScopeMap(config.roots, config);
+ * const flipped = await reclassifyResourceExternality(knex, scope, config);
+ * ```
+ */
+export async function reclassifyResourceExternality(trx, scope, options) {
+    const candidates = await trx
+        .select('resource_items.id as id', 'url_refs.url as url', 'resource_items.is_external')
+        .from('resource_items')
+        .join('url_refs', 'url_refs.id', 'resource_items.url_id');
+    const toInternal = [];
+    const toExternal = [];
+    for (const row of candidates) {
+        const parsed = parseUrl(row.url, options);
+        const isExternal = parsed ? findScopeEntry(parsed, scope, options) === null : true;
+        if (isExternal && row.is_external === 0) {
+            toExternal.push(row.id);
+        }
+        else if (!isExternal && row.is_external === 1) {
+            toInternal.push(row.id);
+        }
+    }
+    const chunkSize = 500;
+    for (const [ids, value] of [
+        [toInternal, 0],
+        [toExternal, 1],
+    ]) {
+        for (let i = 0; i < ids.length; i += chunkSize) {
+            await trx('resource_items')
+                .whereIn('id', ids.slice(i, i + chunkSize))
+                .update({ is_external: value });
+        }
+    }
+    return toInternal.length + toExternal.length;
+}

package/lib/transfer/remap-redirect-dest-ids.d.ts

@@ -0,0 +1,34 @@
+import type { Knex } from 'knex';
+/**
+ * Remaps `redirect_dest_id` for every `full`/`stub`/`replace`-action
+ * destination row this source just wrote, from the source's own id space
+ * into the destination's, via `temp.xfer_ci_plan` (which by this point
+ * maps EVERY source id to a destination id, not just the ones this source
+ * inserted — `skip`/`replace` rows already had `dest_id` set at plan
+ * time).
+ *
+ * `skip`-action rows are deliberately excluded from the `SET` target
+ * (via the `p.action IN (...)` filter) — a `skip` row's `content_items`
+ * row belongs to a PRIOR, better-ranked source, and this (losing) source
+ * has no business touching its `redirect_dest_id`.
+ *
+ * Must run AFTER both
+ * {@link import('./insert-content-items.js').insertContentItems} and
+ * {@link import('./replace-content-items.js').replaceContentItems} —
+ * every plan row needs a resolved `dest_id` (its own AND its redirect
+ * target's) before this can run. A source row whose `redirect_dest_id`
+ * points outside this source's own plan (should not happen — concat's
+ * plan covers every source row, and split's plan closure always pulls a
+ * kept row's redirect target in too, see `plan-content-items-for-split.ts`)
+ * silently leaves that destination row's `redirect_dest_id` at `NULL`
+ * rather than erroring — a dangling-but-absent redirect is a safe
+ * degrade, never a wrong one.
+ * @param trx - Transaction with the source ATTACHed as
+ *   {@link TRANSFER_SOURCE_ALIAS}.
+ * @example
+ * ```ts
+ * await insertContentItems(trx, { crawlOrderOffset: 0 });
+ * await remapRedirectDestIds(trx);
+ * ```
+ */
+export declare function remapRedirectDestIds(trx: Knex): Promise<void>;