@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/transfer/transfer-archive-rows.js

@@ -0,0 +1,156 @@
+import { attachSourceDatabase } from './attach-source-database.js';
+import { clearReplacedPageRows } from './clear-replaced-page-rows.js';
+import { copyAnchorEdges } from './copy-anchor-edges.js';
+import { copyDictionariesForConcat } from './copy-dictionaries-for-concat.js';
+import { copyDictionariesForSplit } from './copy-dictionaries-for-split.js';
+import { copyHeaderSetChildren } from './copy-header-set-children.js';
+import { copyImageItems } from './copy-image-items.js';
+import { copyJournalTables } from './copy-journal-tables.js';
+import { copyPageConsoleLogs } from './copy-page-console-logs.js';
+import { copyPageHtmlBlobs } from './copy-page-html-blobs.js';
+import { copyPageMeta } from './copy-page-meta.js';
+import { copyPageTemplateClusters } from './copy-page-template-clusters.js';
+import { copyPageTemplateLabels } from './copy-page-template-labels.js';
+import { copyResourceItems } from './copy-resource-items.js';
+import { copyResourceRefEdges } from './copy-resource-ref-edges.js';
+import { copySimplePageScopedTables } from './copy-simple-page-scoped-tables.js';
+import { createTransferTempTables } from './create-transfer-temp-tables.js';
+import { dropTransferTempTables } from './drop-transfer-temp-tables.js';
+import { insertContentItems } from './insert-content-items.js';
+import { markInScopeContentItems } from './mark-in-scope-content-items.js';
+import { planContentItemsForConcat } from './plan-content-items-for-concat.js';
+import { planContentItemsForSplit } from './plan-content-items-for-split.js';
+import { planResourceItems } from './plan-resource-items.js';
+import { remapRedirectDestIds } from './remap-redirect-dest-ids.js';
+import { replaceContentItems } from './replace-content-items.js';
+import { TRANSFER_ACTION } from './transfer-action.js';
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Computes {@link TransferSourceResult} from the finished plan tables.
+ * `dropped` (split only) is `sourceTotal - (inserted + stubbed)` — the
+ * plan never lists dropped rows explicitly, so this is a subtraction
+ * against the source's own total row count rather than a query for
+ * absence.
+ * @param trx - The open transaction.
+ * @param mode - `'split'` computes `dropped`; `'concat'` always reports `0`.
+ */
+async function summarizeTransferResult(trx, mode) {
+    const rows = await trx
+        .select('action')
+        .count({ count: '*' })
+        .from('xfer_ci_plan')
+        .groupBy('action');
+    const byAction = new Map(rows.map((r) => [r.action, Number(r.count)]));
+    const inserted = byAction.get(TRANSFER_ACTION.full) ?? 0;
+    const replaced = byAction.get(TRANSFER_ACTION.replace) ?? 0;
+    const skipped = byAction.get(TRANSFER_ACTION.skip) ?? 0;
+    const stubbed = byAction.get(TRANSFER_ACTION.stub) ?? 0;
+    let dropped = 0;
+    if (mode === 'split') {
+        const [totalRow] = await trx
+            .count({ total: '*' })
+            .from(`${TRANSFER_SOURCE_ALIAS}.content_items`);
+        dropped = Number(totalRow?.total ?? 0) - (inserted + stubbed);
+    }
+    return { inserted, replaced, skipped, stubbed, dropped };
+}
+/**
+ * Transfers ONE source archive's rows into the destination — the shared
+ * engine {@link import('./concat-archives.js').concatArchives} and
+ * {@link import('./split-archive.js').splitArchive} both drive, once per
+ * source. Attaches the source, plans, copies every table in dependency
+ * order, and detaches — always, even on failure (`finally`).
+ *
+ * Order within the transaction (see each step's own docs for why this
+ * exact order, not a different one):
+ * 1. Plan `content_items` (mode-specific) and `resource_items`.
+ * 2. Copy every dictionary (mode-specific), then their `header_sets`
+ *    children and `page_html_blobs` (both depend on the dictionary maps
+ *    dictionaries step just built).
+ * 3. Concat only: clear the old page-scoped rows of every `replace`
+ *    destination, then overwrite its `content_items` row. Both modes:
+ *    insert every `full`/`stub` row, backfilling `dest_id`; remap
+ *    `redirect_dest_id` now that every row has one.
+ * 4. Copy every page-scoped table.
+ * 5. Copy `resource_items` (and its js-scan cache), then
+ *    `resource_ref_edges` (needs the resource plan's `dest_id`, just
+ *    resolved).
+ * 6. Copy the append-only journal tables.
+ * @param options - See individual properties.
+ * @param options.knex - Knex connected to the destination archive DB (the
+ *   connection ATTACH/temp-tables/transaction all operate against).
+ * @param options.sourceDbPath - Absolute path to the source's `db.sqlite`
+ *   (e.g. `path.join(accessor.tmpDir, Archive.SQLITE_DB_FILE_NAME)`).
+ * @param options.mode - `{ kind: 'concat' }`, or `{ kind: 'split',
+ *   scopeMap, parseOptions? }` for the narrower-scope pass.
+ * @param options.crawlOrderOffset - Forwarded to
+ *   `insertContentItems`/`replaceContentItems` — see their docs.
+ * @param options.sourceIndex - This source's 0-based position, forwarded
+ *   verbatim to `callbacks` — purely for display, not used in any query.
+ * @param options.callbacks - Progress callbacks; see `types.ts`.
+ * @returns Per-action row counts — see {@link TransferSourceResult}.
+ * @example
+ * ```ts
+ * const result = await transferArchiveRows({
+ *   knex: destination.getKnex(),
+ *   sourceDbPath: path.join(accessor.tmpDir, Archive.SQLITE_DB_FILE_NAME),
+ *   mode: { kind: 'concat' },
+ *   crawlOrderOffset: 0,
+ *   sourceIndex: 0,
+ * });
+ * ```
+ */
+export async function transferArchiveRows(options) {
+    const { knex, sourceDbPath, mode, crawlOrderOffset, sourceIndex, callbacks } = options;
+    const detach = await attachSourceDatabase(knex, sourceDbPath);
+    await createTransferTempTables(knex);
+    try {
+        return await knex.transaction(async (trx) => {
+            callbacks?.onPhase?.(sourceIndex, 'planningContentItems');
+            if (mode.kind === 'split') {
+                await markInScopeContentItems(trx, mode.scopeMap, mode.parseOptions);
+                await planContentItemsForSplit(trx);
+                await planResourceItems(trx, 'split');
+            }
+            else {
+                await planContentItemsForConcat(trx);
+                await planResourceItems(trx, 'concat');
+            }
+            callbacks?.onPhase?.(sourceIndex, 'copyingDictionaries');
+            if (mode.kind === 'split') {
+                await copyDictionariesForSplit(trx);
+            }
+            else {
+                await copyDictionariesForConcat(trx);
+            }
+            await copyHeaderSetChildren(trx);
+            await copyPageHtmlBlobs(trx);
+            callbacks?.onPhase?.(sourceIndex, 'copyingContentItems');
+            if (mode.kind === 'concat') {
+                await clearReplacedPageRows(trx);
+                await replaceContentItems(trx, { crawlOrderOffset });
+            }
+            await insertContentItems(trx, { crawlOrderOffset });
+            await remapRedirectDestIds(trx);
+            callbacks?.onPhase?.(sourceIndex, 'copyingPageRows');
+            await copyPageMeta(trx);
+            await copyImageItems(trx);
+            await copyAnchorEdges(trx);
+            await copySimplePageScopedTables(trx);
+            await copyPageTemplateClusters(trx);
+            await copyPageTemplateLabels(trx);
+            await copyPageConsoleLogs(trx);
+            callbacks?.onPhase?.(sourceIndex, 'copyingResources');
+            await copyResourceItems(trx);
+            await copyResourceRefEdges(trx);
+            callbacks?.onPhase?.(sourceIndex, 'copyingJournals');
+            await copyJournalTables(trx, mode.kind);
+            callbacks?.onPhase?.(sourceIndex, 'committing');
+            return await summarizeTransferResult(trx, mode.kind);
+        });
+    }
+    finally {
+        await dropTransferTempTables(knex);
+        await detach();
+    }
+}

package/lib/transfer/transfer-dictionary-tables.d.ts

@@ -0,0 +1,12 @@
+/**
+ * The fixed set of dictionary tables a transfer's need-based dictionary
+ * copy tracks per-id "is this row referenced by anything kept" state for.
+ * Drives the loop in {@link import('./create-transfer-temp-tables.js').createTransferTempTables} /
+ * {@link import('./drop-transfer-temp-tables.js').dropTransferTempTables}
+ * that creates/drops one `(src_id PRIMARY KEY)` need-table and one
+ * `(src_id PRIMARY KEY, dest_id)` map-table per dictionary — every
+ * dictionary table shares that exact temp-table shape, only the
+ * population query differs (see `copy-dictionaries-for-concat.ts` /
+ * `copy-dictionaries-for-split.ts`).
+ */
+export declare const TRANSFER_DICTIONARY_TABLES: readonly string[];

package/lib/transfer/transfer-dictionary-tables.js

@@ -0,0 +1,22 @@
+/**
+ * The fixed set of dictionary tables a transfer's need-based dictionary
+ * copy tracks per-id "is this row referenced by anything kept" state for.
+ * Drives the loop in {@link import('./create-transfer-temp-tables.js').createTransferTempTables} /
+ * {@link import('./drop-transfer-temp-tables.js').dropTransferTempTables}
+ * that creates/drops one `(src_id PRIMARY KEY)` need-table and one
+ * `(src_id PRIMARY KEY, dest_id)` map-table per dictionary — every
+ * dictionary table shares that exact temp-table shape, only the
+ * population query differs (see `copy-dictionaries-for-concat.ts` /
+ * `copy-dictionaries-for-split.ts`).
+ */
+export const TRANSFER_DICTIONARY_TABLES = [
+    'url_refs',
+    'content_type_refs',
+    'text_refs',
+    'json_refs',
+    'blob_refs',
+    'header_name_refs',
+    'header_value_refs',
+    'header_sets',
+    'console_log_items',
+];

package/lib/transfer/transfer-post-phases.d.ts

@@ -0,0 +1,10 @@
+import type { TransferPhase } from './types.js';
+/**
+ * Ordered whole-operation phase sequence `concatArchives`/`splitArchive`
+ * announce (via `sourceIndex === sources.length`) once every source has
+ * been transferred: `flattenRedirectChains` → `reclassifyResourceExternality`
+ * → `countExternalPagesInScope`. Identical for both commands — split's
+ * single source is already flat and its scope only shrinks, so the first
+ * two are cheap no-ops there, but the sequence itself does not branch.
+ */
+export declare const TRANSFER_POST_PHASES: readonly TransferPhase[];

package/lib/transfer/transfer-post-phases.js

@@ -0,0 +1,13 @@
+/**
+ * Ordered whole-operation phase sequence `concatArchives`/`splitArchive`
+ * announce (via `sourceIndex === sources.length`) once every source has
+ * been transferred: `flattenRedirectChains` → `reclassifyResourceExternality`
+ * → `countExternalPagesInScope`. Identical for both commands — split's
+ * single source is already flat and its scope only shrinks, so the first
+ * two are cheap no-ops there, but the sequence itself does not branch.
+ */
+export const TRANSFER_POST_PHASES = [
+    'flatteningRedirects',
+    'reclassifyingResources',
+    'countingExternalInScope',
+];

package/lib/transfer/transfer-source-alias.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The fixed schema alias a source archive's `db.sqlite` is `ATTACH`ed under
+ * during a transfer. A constant (not a per-call parameter) because every
+ * transfer SQL statement in `archive/transfer/` hard-codes this name —
+ * allowing a caller-supplied alias would mean threading it through every
+ * one of those statements for no real benefit (a transfer only ever
+ * attaches one source at a time; see `transfer-archive-rows.ts`).
+ */
+export declare const TRANSFER_SOURCE_ALIAS = "xfer_src";

package/lib/transfer/transfer-source-alias.js

@@ -0,0 +1,9 @@
+/**
+ * The fixed schema alias a source archive's `db.sqlite` is `ATTACH`ed under
+ * during a transfer. A constant (not a per-call parameter) because every
+ * transfer SQL statement in `archive/transfer/` hard-codes this name —
+ * allowing a caller-supplied alias would mean threading it through every
+ * one of those statements for no real benefit (a transfer only ever
+ * attaches one source at a time; see `transfer-archive-rows.ts`).
+ */
+export const TRANSFER_SOURCE_ALIAS = 'xfer_src';

package/lib/transfer/types.d.ts

@@ -0,0 +1,111 @@
+import type { Config } from '../types.js';
+/**
+ * Ordered phase sequence a single source's transfer announces via
+ * `onPhase`, unconditionally and always in this order (mirrors
+ * `ViewerReadModelBuildPhase`'s contract — no branch skips a phase, so a
+ * caller can pre-build one `TaskList` row per phase and rely on them
+ * settling in order).
+ */
+export type TransferPhase = 'planningContentItems' | 'copyingDictionaries' | 'copyingContentItems' | 'copyingPageRows' | 'copyingResources' | 'copyingJournals' | 'committing' | 'flatteningRedirects' | 'reclassifyingResources' | 'countingExternalInScope';
+/**
+ * Progress callbacks a caller (the CLI's `concat`/`split` commands) passes
+ * into {@link import('./concat-archives.js').concatArchives} /
+ * {@link import('./split-archive.js').splitArchive} to drive a `TaskList`
+ * display. `onPhase` fires once per phase per source (plus once per
+ * whole-operation post phase); `onProgress` fires zero or more times while
+ * a phase is active, with a phase-specific unit (rows copied, ids
+ * classified, …).
+ */
+export interface TransferCallbacks {
+    /** Called when a new source's extraction begins (0-based index). */
+    readonly onSourceStart?: (sourceIndex: number, filePath: string) => void;
+    /**
+     * Called whenever the active phase changes, for the source at
+     * `sourceIndex` (or after every source has finished, for the
+     * whole-operation post phases — `flatteningRedirects` /
+     * `reclassifyingResources` / `countingExternalInScope` — where
+     * `sourceIndex` is `sources.length`).
+     */
+    readonly onPhase?: (sourceIndex: number, phase: TransferPhase) => void;
+    /** Called zero or more times while a phase is active. */
+    readonly onProgress?: (processed: number, total: number) => void;
+}
+/** Per-source outcome of {@link import('./transfer-archive-rows.js').transferArchiveRows}. */
+export interface TransferSourceResult {
+    /** Number of `content_items` rows copied as brand-new rows (no prior dest row for that URL). */
+    readonly inserted: number;
+    /**
+     * Number of existing dest `content_items` rows overwritten because this
+     * source's observation outranked the dest's (concat only; always `0`
+     * for split, which never merges into an existing archive).
+     */
+    readonly replaced: number;
+    /** Number of source rows skipped because dest already held an equal-or-better observation (concat only). */
+    readonly skipped: number;
+    /** Number of source rows kept as an external stub, not a full copy (split only; always `0` for concat). */
+    readonly stubbed: number;
+    /** Number of source rows dropped entirely — out of scope and unreferenced (split only; always `0` for concat). */
+    readonly dropped: number;
+}
+/** Result of {@link import('./concat-archives.js').concatArchives}. */
+export interface ConcatArchivesResult {
+    /** The merged config written to the destination's `info` row. */
+    readonly config: Config;
+    /** Per-source transfer outcome, in argument order. */
+    readonly sources: readonly TransferSourceResult[];
+    /**
+     * Number of URLs that fall inside the merged scope but were never
+     * fetched as an internal page by any source (still `is_external = 1`
+     * in the output) — the count the CLI surfaces with a
+     * `crawl <out> --append <root>` hint, since concat never re-promotes
+     * external rows on its own (that would fabricate a `scraped = 1`
+     * observation with no body/meta behind it).
+     */
+    readonly externalInScopeCount: number;
+    /** Number of `inventory/<sha256>.txt` source lists copied (deduplicated by filename across sources). */
+    readonly inventoryListsCopied: number;
+}
+/** Result of {@link import('./split-archive.js').splitArchive}. */
+export interface SplitArchiveResult {
+    /** The derived config (roots = the given scope URLs) written to the destination's `info` row. */
+    readonly config: Config;
+    /** The single source's transfer outcome. */
+    readonly source: TransferSourceResult;
+    /**
+     * Usually `0`, but not structurally guaranteed to be: the operator's
+     * scope URLs are not required to be a subset of the source's original
+     * roots, so a scope broader than (or merely different from) the
+     * original can bring an external-only (never crawled) page inside the
+     * new scope, the same way concat's own external-in-scope count can be
+     * non-zero.
+     */
+    readonly externalInScopeCount: number;
+    /** Number of `inventory/<sha256>.txt` source lists copied. */
+    readonly inventoryListsCopied: number;
+}
+/**
+ * Thrown by {@link import('./merge-archive-configs.js').mergeArchiveConfigs}
+ * when two source archives disagree on a config field that concat cannot
+ * silently reconcile (`disableQueries`, `fromList`) — see that function's
+ * docs for why these two specifically cannot fall back to "first archive
+ * wins" like every other scalar field.
+ */
+export declare class ArchiveConfigConflictError extends Error {
+    readonly field: string;
+    readonly values: readonly unknown[];
+    /**
+     * @param field - The conflicting `Config` field name.
+     * @param values - The distinct values seen across sources, in argument order.
+     */
+    constructor(field: string, values: readonly unknown[]);
+}
+/**
+ * Thrown when a dictionary-table copy's post-copy id map does not cover
+ * every id it needed to map (a source row referencing a value that failed
+ * to insert or resolve) — a defensive integrity check, since a silently
+ * incomplete map would surface much later as a confusing "no such row" or
+ * a dangling NULL FK deep in an unrelated read path.
+ */
+export declare class TransferIntegrityError extends Error {
+    constructor(message: string);
+}

package/lib/transfer/types.js

@@ -0,0 +1,35 @@
+/**
+ * Thrown by {@link import('./merge-archive-configs.js').mergeArchiveConfigs}
+ * when two source archives disagree on a config field that concat cannot
+ * silently reconcile (`disableQueries`, `fromList`) — see that function's
+ * docs for why these two specifically cannot fall back to "first archive
+ * wins" like every other scalar field.
+ */
+export class ArchiveConfigConflictError extends Error {
+    field;
+    values;
+    /**
+     * @param field - The conflicting `Config` field name.
+     * @param values - The distinct values seen across sources, in argument order.
+     */
+    constructor(field, values) {
+        super(`Cannot merge archives: "${field}" differs across sources (${values.map((v) => JSON.stringify(v)).join(' vs ')}). ` +
+            'Re-crawl the sources with matching settings before concatenating them.');
+        this.field = field;
+        this.values = values;
+        this.name = 'ArchiveConfigConflictError';
+    }
+}
+/**
+ * Thrown when a dictionary-table copy's post-copy id map does not cover
+ * every id it needed to map (a source row referencing a value that failed
+ * to insert or resolve) — a defensive integrity check, since a silently
+ * incomplete map would surface much later as a confusing "no such row" or
+ * a dangling NULL FK deep in an unrelated read path.
+ */
+export class TransferIntegrityError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'TransferIntegrityError';
+    }
+}