@nitpicker/archive 0.0.0 → 0.25.0

package/lib/test-utils/build-full-schema-test-db.js

@@ -0,0 +1,74 @@
+import knex from 'knex';
+import { setConfig } from '../db-ops/config/set-config.js';
+import { init } from '../db-ops/lifecycle/init.js';
+import { LibsqlDialect } from '../libsql-dialect.js';
+/**
+ * Default {@link Config} fields for {@link buildFullSchemaTestDb} — a
+ * single-root, recursive, non-list crawl of `https://example.com/`, no
+ * excludes. Overridable per-call for scope/`fromList`/`disableQueries`
+ * tests.
+ */
+const DEFAULT_TEST_CONFIG = {
+    version: '0.13.0',
+    name: 'test',
+    baseUrl: 'https://example.com/',
+    roots: ['https://example.com/'],
+    recursive: true,
+    interval: 0,
+    image: false,
+    fetchExternal: false,
+    parallels: 1,
+    excludes: [],
+    excludeKeywords: [],
+    excludeUrls: [],
+    maxExcludedDepth: 0,
+    retry: 3,
+    fromList: false,
+    disableQueries: false,
+    userAgent: 'test',
+    ignoreRobots: false,
+    mainContentSelector: null,
+};
+/**
+ * Builds a file-backed knex connection against the FULL current archive
+ * schema (every ref/entity/adjunct table, including columns only a
+ * migration adds to a legacy archive — `alias_of_id`, `dedupe_cap_event_id`,
+ * `body_hash`, …) by running the exact same {@link init} boot sequence a
+ * real `Archive.create`/`Archive.open` uses. Shared by every
+ * `archive/transfer/` spec that needs a realistic source or destination
+ * database — hand-rolling `createRefTables`/`createEntityTables`/
+ * `createAdjunctTables` directly would produce the schema `initSchema`
+ * creates but skip the migrations that add columns for archives created
+ * before those features existed (which happen to be a no-op on a fresh
+ * DB today, but drift is exactly the risk this helper avoids).
+ *
+ * A DB file must be used (not `:memory:`) because `ATTACH DATABASE` in
+ * the transfer primitives needs a real path — see `attach-source-database.ts`.
+ * @param filename - Absolute path to the SQLite file. The caller is
+ *   responsible for removing it (`afterEach`) — see existing specs for the
+ *   convention.
+ * @param config - Overrides merged over {@link DEFAULT_TEST_CONFIG} and
+ *   written via `setConfig` once the schema exists. Pass `null` to skip
+ *   writing an `info` row entirely (some specs seed `content_items`
+ *   directly and never read `getConfig()`).
+ * @returns The connected knex instance, on the current schema, with an
+ *   `info` row unless `config` is `null`.
+ * @example
+ * ```ts
+ * const db = await buildFullSchemaTestDb(path.resolve(workingDir, 'source.sqlite'), {
+ *   roots: ['https://a.example.com/'],
+ * });
+ * ```
+ */
+export async function buildFullSchemaTestDb(filename, config = {}) {
+    const instance = knex({
+        client: LibsqlDialect,
+        connection: { filename },
+        useNullAsDefault: true,
+    });
+    await init(instance, false);
+    if (config !== null) {
+        await setConfig(instance, { ...DEFAULT_TEST_CONFIG, ...config });
+    }
+    return instance;
+}

package/lib/test-utils/create-legacy-analysis-tables.d.ts

@@ -0,0 +1,13 @@
+import type { Knex } from 'knex';
+/**
+ * Creates the `analysis_text_refs` / `analysis_violations` tables in the shape
+ * an archive written by an earlier version carries (FK → `content_items(id)`).
+ * Current archives do not create these tables, so specs that exercise the
+ * guarded cleanup of legacy archives provision them through this helper.
+ * @param db - Knex connected to the test DB (or a transaction).
+ * @example
+ * await createAdjunctTables(db);
+ * await createLegacyAnalysisTables(db);
+ * await db('analysis_text_refs').insert({ text: 'msg', sha256: 'abc' });
+ */
+export declare function createLegacyAnalysisTables(db: Knex): Promise<void>;

package/lib/test-utils/create-legacy-analysis-tables.js

@@ -0,0 +1,37 @@
+/**
+ * Creates the `analysis_text_refs` / `analysis_violations` tables in the shape
+ * an archive written by an earlier version carries (FK → `content_items(id)`).
+ * Current archives do not create these tables, so specs that exercise the
+ * guarded cleanup of legacy archives provision them through this helper.
+ * @param db - Knex connected to the test DB (or a transaction).
+ * @example
+ * await createAdjunctTables(db);
+ * await createLegacyAnalysisTables(db);
+ * await db('analysis_text_refs').insert({ text: 'msg', sha256: 'abc' });
+ */
+export async function createLegacyAnalysisTables(db) {
+    await db.raw(`
+		CREATE TABLE analysis_text_refs (
+			id integer primary key,
+			text text not null,
+			sha256 text not null,
+			unique(sha256, text)
+		)
+	`);
+    await db.raw(`
+		CREATE TABLE analysis_violations (
+			id integer primary key,
+			page_id integer not null references content_items(id),
+			validator text not null,
+			severity text not null,
+			rule text not null,
+			message_text_id integer not null references analysis_text_refs(id),
+			code_text_id integer references analysis_text_refs(id),
+			page_url_sort_key text not null,
+			message_sort_key text not null,
+			code_sort_key text not null,
+			line integer,
+			col integer
+		)
+	`);
+}

package/lib/test-utils/fk-parent-tables.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Returns the set of parent table names referenced by `table`'s FK
+ * clauses, read via SQLite's `pragma_foreign_key_list` table-valued
+ * function. Shared by the specs that assert FK declarations were
+ * retargeted from the legacy `pages(id)` to `content_items(id)`.
+ * @param db - Knex connected to the test DB.
+ * @param table - Table whose FK declarations to inspect.
+ * @returns Parent table names, deduplicated.
+ * @example
+ * const parents = await fkParentTables(db, 'page_errors');
+ * expect(parents.has('content_items')).toBe(true);
+ * expect(parents.has('pages')).toBe(false);
+ */
+export declare function fkParentTables(db: Knex, table: string): Promise<Set<string>>;

package/lib/test-utils/fk-parent-tables.js

@@ -0,0 +1,19 @@
+/**
+ * Returns the set of parent table names referenced by `table`'s FK
+ * clauses, read via SQLite's `pragma_foreign_key_list` table-valued
+ * function. Shared by the specs that assert FK declarations were
+ * retargeted from the legacy `pages(id)` to `content_items(id)`.
+ * @param db - Knex connected to the test DB.
+ * @param table - Table whose FK declarations to inspect.
+ * @returns Parent table names, deduplicated.
+ * @example
+ * const parents = await fkParentTables(db, 'page_errors');
+ * expect(parents.has('content_items')).toBe(true);
+ * expect(parents.has('pages')).toBe(false);
+ */
+export async function fkParentTables(db, table) {
+    const rows = await db
+        .select('table')
+        .from(db.raw('pragma_foreign_key_list(?)', [table]));
+    return new Set(rows.map((row) => row.table));
+}

package/lib/test-utils/seed-content-item.d.ts

@@ -0,0 +1,35 @@
+import type { PageSource } from '../types.js';
+import type { Knex } from 'knex';
+/**
+ * Optional column overrides for {@link seedContentItem}. Defaults model a
+ * scraped, internal, targeted page with no content type.
+ */
+interface SeedContentItemOptions {
+    /** `content_items.scraped` (default 1). */
+    scraped?: 0 | 1;
+    /** `content_items.is_target` (default 1). */
+    isTarget?: 0 | 1;
+    /** `content_items.is_external` (default 0). */
+    isExternal?: 0 | 1;
+    /** `content_items.source` (default: schema default `'crawled'`). */
+    source?: PageSource;
+    /** Registers the value in `content_type_refs` and links it (default: none). */
+    contentType?: string;
+}
+/**
+ * Inserts one `url_refs` + `content_items` pair — the minimal seeding
+ * contract for specs that exercise entity-table readers. Shared so a
+ * schema change to the pair (e.g. a new NOT NULL column) is fixed in one
+ * place instead of drifting across per-spec copies.
+ * @param db - Knex connected to the test DB.
+ * @param url - URL string to register in `url_refs`.
+ * @param options - Column overrides; see {@link SeedContentItemOptions}.
+ * @returns The inserted `content_items.id`.
+ * @example
+ * const pageId = await seedContentItem(db, 'https://example.com/');
+ * const pdfId = await seedContentItem(db, 'https://example.com/doc.pdf', {
+ *   contentType: 'application/pdf',
+ * });
+ */
+export declare function seedContentItem(db: Knex, url: string, options?: SeedContentItemOptions): Promise<number>;
+export {};

package/lib/test-utils/seed-content-item.js

@@ -0,0 +1,42 @@
+/**
+ * Inserts one `url_refs` + `content_items` pair — the minimal seeding
+ * contract for specs that exercise entity-table readers. Shared so a
+ * schema change to the pair (e.g. a new NOT NULL column) is fixed in one
+ * place instead of drifting across per-spec copies.
+ * @param db - Knex connected to the test DB.
+ * @param url - URL string to register in `url_refs`.
+ * @param options - Column overrides; see {@link SeedContentItemOptions}.
+ * @returns The inserted `content_items.id`.
+ * @example
+ * const pageId = await seedContentItem(db, 'https://example.com/');
+ * const pdfId = await seedContentItem(db, 'https://example.com/doc.pdf', {
+ *   contentType: 'application/pdf',
+ * });
+ */
+export async function seedContentItem(db, url, options = {}) {
+    const [urlRef] = await db('url_refs').insert({ url }).returning('id');
+    let contentTypeId = null;
+    if (options.contentType !== undefined) {
+        const [ctRef] = await db('content_type_refs')
+            .insert({
+            raw: options.contentType,
+            normalized: options.contentType,
+            category: 'other',
+        })
+            .onConflict('raw')
+            .merge({ raw: options.contentType })
+            .returning('id');
+        contentTypeId = ctRef.id;
+    }
+    const [item] = await db('content_items')
+        .insert({
+        url_id: urlRef.id,
+        scraped: options.scraped ?? 1,
+        is_target: options.isTarget ?? 1,
+        is_external: options.isExternal ?? 0,
+        content_type_id: contentTypeId,
+        ...(options.source === undefined ? {} : { source: options.source }),
+    })
+        .returning('id');
+    return item.id;
+}

package/lib/test-utils/setup-legacy-fk-db.d.ts

@@ -0,0 +1,34 @@
+import type { Knex } from 'knex';
+/**
+ * Provisions the pre-0.13 legacy table shape on a test database:
+ * the five legacy write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) plus the five adjunct tables in
+ * their old form whose FK columns still point at `pages(id)`
+ * (`page_html_ref` / `page_tags` / `page_jsonld` / `page_errors` /
+ * `analysis_violations`, with their companion tables `page_html_blobs` /
+ * `analysis_text_refs`). The two `analysis_*` tables are the input of the
+ * 0.13 migration's drop path (no current-schema counterpart).
+ *
+ * `initSchema` no longer creates any of these shapes — fresh archives get
+ * neither the legacy write-model tables nor `pages(id)`-referencing
+ * adjunct tables — so specs that exercise the migration input format
+ * (`scripts/migrate-to-0.13.mjs` integration, `retargetLegacyFkTables`,
+ * `dropLegacyTables`) build it through this helper instead. The DDL is a
+ * verbatim copy of what pre-0.13 archives actually contain: the legacy
+ * write-model tables match the pre-0.13 `initSchema`, and the adjunct
+ * tables match the `scripts/migrate-to-0.10.mjs` / lazy-runtime-migration
+ * output of that era.
+ *
+ * Any current-form adjunct tables already present (e.g. created by
+ * `Archive.create()` on the current schema) are dropped first so the
+ * old-FK versions can take their place — callers get a deterministic
+ * "genuine pre-0.13 archive" shape regardless of how the underlying DB
+ * was provisioned.
+ * @param db - Knex connected to the test DB (or a transaction).
+ * @example
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await setupLegacyFkDb(db);
+ * await db('pages').insert({ url: 'https://example.com/', scraped: 1, isTarget: 1 });
+ */
+export declare function setupLegacyFkDb(db: Knex): Promise<void>;

package/lib/test-utils/setup-legacy-fk-db.js

@@ -0,0 +1,271 @@
+/**
+ * Provisions the pre-0.13 legacy table shape on a test database:
+ * the five legacy write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) plus the five adjunct tables in
+ * their old form whose FK columns still point at `pages(id)`
+ * (`page_html_ref` / `page_tags` / `page_jsonld` / `page_errors` /
+ * `analysis_violations`, with their companion tables `page_html_blobs` /
+ * `analysis_text_refs`). The two `analysis_*` tables are the input of the
+ * 0.13 migration's drop path (no current-schema counterpart).
+ *
+ * `initSchema` no longer creates any of these shapes — fresh archives get
+ * neither the legacy write-model tables nor `pages(id)`-referencing
+ * adjunct tables — so specs that exercise the migration input format
+ * (`scripts/migrate-to-0.13.mjs` integration, `retargetLegacyFkTables`,
+ * `dropLegacyTables`) build it through this helper instead. The DDL is a
+ * verbatim copy of what pre-0.13 archives actually contain: the legacy
+ * write-model tables match the pre-0.13 `initSchema`, and the adjunct
+ * tables match the `scripts/migrate-to-0.10.mjs` / lazy-runtime-migration
+ * output of that era.
+ *
+ * Any current-form adjunct tables already present (e.g. created by
+ * `Archive.create()` on the current schema) are dropped first so the
+ * old-FK versions can take their place — callers get a deterministic
+ * "genuine pre-0.13 archive" shape regardless of how the underlying DB
+ * was provisioned.
+ * @param db - Knex connected to the test DB (or a transaction).
+ * @example
+ * await createRefTables(db);
+ * await createEntityTables(db);
+ * await setupLegacyFkDb(db);
+ * await db('pages').insert({ url: 'https://example.com/', scraped: 1, isTarget: 1 });
+ */
+export async function setupLegacyFkDb(db) {
+    // Children before parents so the drops never trip FK enforcement
+    // when the caller has `PRAGMA foreign_keys = ON`.
+    await db.raw('DROP TABLE IF EXISTS "page_html_ref"');
+    await db.raw('DROP TABLE IF EXISTS "page_html_blobs"');
+    await db.raw('DROP TABLE IF EXISTS "analysis_violations"');
+    await db.raw('DROP TABLE IF EXISTS "analysis_text_refs"');
+    await db.raw('DROP TABLE IF EXISTS "page_tags"');
+    await db.raw('DROP TABLE IF EXISTS "page_jsonld"');
+    await db.raw('DROP TABLE IF EXISTS "page_errors"');
+    await db.schema
+        .createTable('pages', (t) => {
+        t.increments('id');
+        t.string('url', 8190).notNullable().unique();
+        t.integer('redirectDestId').unsigned().references('pages.id').defaultTo(null);
+        t.boolean('scraped').notNullable();
+        t.boolean('isTarget').notNullable();
+        t.boolean('isExternal');
+        t.integer('status');
+        t.string('statusText');
+        t.string('contentType').nullable();
+        t.integer('contentLength').unsigned().nullable();
+        t.json('responseHeaders').nullable();
+        // Document basics
+        t.string('lang');
+        t.string('dir');
+        t.string('charset');
+        t.string('baseHref');
+        t.text('viewport_raw');
+        t.string('themeColor');
+        t.string('applicationName');
+        t.string('author');
+        t.string('generator');
+        t.string('publisher');
+        // Title / description / keywords (top-level Meta fields)
+        t.string('title');
+        t.text('description');
+        t.text('keywords');
+        // Robots
+        t.text('robots_raw');
+        t.integer('robots_noindex');
+        t.integer('robots_nofollow');
+        t.integer('robots_noarchive');
+        t.integer('robots_noimageindex');
+        t.string('googlebot');
+        // Link (1:1 only — array shapes live in meta_extras)
+        t.string('canonical', 8190);
+        t.string('amphtml', 8190);
+        t.string('manifest', 8190);
+        t.string('icon_href', 8190);
+        t.string('appleTouchIcon_href', 8190);
+        // Open Graph
+        t.string('og_type');
+        t.string('og_title');
+        t.string('og_url', 8190);
+        t.string('og_site_name');
+        t.text('og_description');
+        t.string('og_image', 8190);
+        t.string('og_image_alt');
+        t.string('og_image_width');
+        t.string('og_image_height');
+        t.string('og_locale');
+        t.string('og_article_published_time');
+        t.string('og_article_modified_time');
+        // Twitter
+        t.string('twitter_card');
+        t.string('twitter_site');
+        t.string('twitter_creator');
+        t.string('twitter_title');
+        t.text('twitter_description');
+        t.string('twitter_image', 8190);
+        // One-offs
+        t.string('fb_app_id');
+        t.string('verification_google');
+        t.integer('formatDetection_telephone');
+        // Within-archive observation timestamps (UNIX ms)
+        t.integer('firstCrawledAt');
+        t.integer('lastCrawledAt');
+        // Denormalised aggregates
+        t.integer('tag_count');
+        t.integer('jsonld_count');
+        t.text('tags_providers_csv');
+        // Catch-all JSON for nested Meta sub-objects not flattened above
+        t.json('meta_extras');
+        // Crawl lifecycle
+        t.boolean('isSkipped');
+        t.string('skipReason');
+        t.integer('order').unsigned().nullable();
+        t.string('source').notNullable().defaultTo('crawled');
+        t.index('isExternal');
+        t.index('contentType');
+        t.index('scraped');
+        t.index('redirectDestId');
+        t.index('order');
+        t.index('robots_noindex');
+        t.index('og_type');
+        t.index('source');
+    })
+        .createTable('anchors', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable().unsigned().references('pages.id');
+        t.integer('hrefId').notNullable().unsigned().references('pages.id');
+        t.string('hash');
+        t.string('textContent').nullable();
+        t.index('pageId');
+        t.index('hrefId');
+    })
+        .createTable('images', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable().unsigned().references('pages.id');
+        t.string('src', 8190);
+        t.string('currentSrc', 8190);
+        t.string('alt');
+        t.float('width').unsigned().notNullable();
+        t.float('height').unsigned().notNullable();
+        t.integer('naturalWidth').unsigned().notNullable();
+        t.integer('naturalHeight').unsigned().notNullable();
+        t.boolean('isLazy');
+        t.integer('viewportWidth').unsigned().notNullable();
+        t.string('sourceCode');
+        t.index('pageId');
+    })
+        .createTable('resources', (t) => {
+        t.increments('id');
+        t.string('url', 8190).notNullable().unique();
+        t.boolean('isExternal');
+        t.integer('status');
+        t.string('statusText');
+        t.string('contentType').nullable();
+        t.integer('contentLength').unsigned().nullable();
+        t.string('compress').nullable();
+        t.string('cdn').nullable();
+        t.json('responseHeaders').nullable();
+        t.string('source').notNullable().defaultTo('crawled');
+        t.index('source');
+    })
+        .createTable('resources-referrers', (t) => {
+        t.increments('id');
+        t.integer('resourceId').notNullable().unsigned().references('resources.id');
+        t.integer('pageId').notNullable().unsigned().references('pages.id');
+        t.unique(['resourceId', 'pageId']);
+        t.index('resourceId');
+        t.index('pageId');
+    });
+    // Adjunct tables in their pre-0.13 shape: FK columns point at
+    // `pages(id)` — the exact declarations that
+    // `scripts/migrate-to-0.10.mjs` and the runtime lazy migrations of
+    // that era produced.
+    await db.schema.createTable('page_errors', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable().unsigned().references('pages.id');
+        t.string('phase').notNullable();
+        t.text('message').notNullable();
+        t.integer('createdAt').notNullable();
+        t.index('pageId');
+    });
+    await db.schema.createTable('page_tags', (t) => {
+        t.increments('id');
+        t.integer('pageId')
+            .notNullable()
+            .unsigned()
+            .references('pages.id')
+            .onDelete('CASCADE');
+        t.string('provider').notNullable();
+        t.string('category');
+        t.string('externalId');
+        t.string('version');
+        t.integer('confidence');
+        t.json('categories');
+        t.json('sources');
+        t.index('pageId');
+        t.index('provider');
+        t.index('externalId');
+    });
+    await db.raw('CREATE INDEX page_tags_provider_extId ON page_tags(provider, externalId)');
+    await db.raw('CREATE INDEX page_tags_provider_pageId ON page_tags(provider, pageId)');
+    await db.schema.createTable('page_jsonld', (t) => {
+        t.increments('id');
+        t.integer('pageId')
+            .notNullable()
+            .unsigned()
+            .references('pages.id')
+            .onDelete('CASCADE');
+        t.string('kind').notNullable();
+        t.string('type');
+        t.text('raw').notNullable();
+        t.json('parsed');
+        t.text('parseError');
+        t.index('pageId');
+        t.index('type');
+    });
+    await db.raw('CREATE INDEX page_jsonld_type_pageId ON page_jsonld(type, pageId)');
+    await db.raw(`
+		CREATE TABLE analysis_text_refs (
+			id integer primary key,
+			text text not null,
+			sha256 text not null,
+			unique(sha256, text)
+		)
+	`);
+    await db.raw(`
+		CREATE TABLE analysis_violations (
+			id integer primary key,
+			page_id integer not null references pages(id),
+			validator text not null,
+			severity text not null,
+			rule text not null,
+			message_text_id integer not null references analysis_text_refs(id),
+			code_text_id integer references analysis_text_refs(id),
+			page_url_sort_key text not null,
+			message_sort_key text not null,
+			code_sort_key text not null
+		)
+	`);
+    await db.raw('CREATE INDEX av_url_order ON analysis_violations(page_url_sort_key, id)');
+    await db.raw('CREATE INDEX av_filter_url ON analysis_violations(validator, severity, rule, page_url_sort_key, id)');
+    await db.raw('CREATE INDEX av_validator_url ON analysis_violations(validator, page_url_sort_key, id)');
+    await db.raw('CREATE INDEX av_severity_url ON analysis_violations(severity, page_url_sort_key, id)');
+    await db.raw('CREATE INDEX av_rule_url ON analysis_violations(rule, page_url_sort_key, id)');
+    await db.raw('CREATE INDEX av_message_order ON analysis_violations(message_sort_key, id)');
+    await db.raw('CREATE INDEX av_code_order ON analysis_violations(code_sort_key, id)');
+    await db.raw('CREATE INDEX av_page ON analysis_violations(page_id, id)');
+    await db.raw(`
+		CREATE TABLE page_html_blobs (
+			hash         BLOB PRIMARY KEY,
+			body         BLOB NOT NULL,
+			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+			size_raw     INTEGER NOT NULL,
+			size_stored  INTEGER NOT NULL
+		) WITHOUT ROWID
+	`);
+    await db.raw(`
+		CREATE TABLE page_html_ref (
+			page_id  INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
+			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
+		) WITHOUT ROWID
+	`);
+    await db.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
+}

package/lib/transfer/attach-source-database.d.ts

@@ -0,0 +1,31 @@
+import type { Knex } from 'knex';
+/**
+ * Attaches a source archive's `db.sqlite` file to the destination
+ * connection under the {@link TRANSFER_SOURCE_ALIAS} schema alias, so a
+ * transfer can read the source and write the destination in the same SQL
+ * statement (`INSERT INTO main.content_items SELECT ... FROM xfer_src.content_items`).
+ *
+ * `ATTACH DATABASE` must run OUTSIDE any transaction — SQLite rejects it
+ * with "cannot ATTACH database within transaction" otherwise — so this is
+ * always called before `knex.transaction(...)` opens, never inside it. The
+ * returned function detaches and must be called after the transaction
+ * commits or rolls back, in a `finally`.
+ * @param knex - Knex query builder connected to the destination archive DB
+ *   (the connection whose schema this attaches to — not a transaction).
+ * @param sourceDbPath - Absolute path to the source's `db.sqlite` file
+ *   (typically `path.join(accessor.tmpDir, Archive.SQLITE_DB_FILE_NAME)`).
+ * @returns An async function that detaches the source. Idempotent-safe to
+ *   call even if the attach itself failed partway (best-effort).
+ * @throws {Error} If `ATTACH DATABASE` fails (e.g. the path does not exist,
+ *   or a transaction is already open on this connection).
+ * @example
+ * ```ts
+ * const detach = await attachSourceDatabase(knex, sourceDbPath);
+ * try {
+ *   await knex.transaction(async (trx) => { ... trx.raw('SELECT * FROM xfer_src.content_items') ... });
+ * } finally {
+ *   await detach();
+ * }
+ * ```
+ */
+export declare function attachSourceDatabase(knex: Knex, sourceDbPath: string): Promise<() => Promise<void>>;

package/lib/transfer/attach-source-database.js

@@ -0,0 +1,40 @@
+import { TRANSFER_SOURCE_ALIAS } from './transfer-source-alias.js';
+/**
+ * Attaches a source archive's `db.sqlite` file to the destination
+ * connection under the {@link TRANSFER_SOURCE_ALIAS} schema alias, so a
+ * transfer can read the source and write the destination in the same SQL
+ * statement (`INSERT INTO main.content_items SELECT ... FROM xfer_src.content_items`).
+ *
+ * `ATTACH DATABASE` must run OUTSIDE any transaction — SQLite rejects it
+ * with "cannot ATTACH database within transaction" otherwise — so this is
+ * always called before `knex.transaction(...)` opens, never inside it. The
+ * returned function detaches and must be called after the transaction
+ * commits or rolls back, in a `finally`.
+ * @param knex - Knex query builder connected to the destination archive DB
+ *   (the connection whose schema this attaches to — not a transaction).
+ * @param sourceDbPath - Absolute path to the source's `db.sqlite` file
+ *   (typically `path.join(accessor.tmpDir, Archive.SQLITE_DB_FILE_NAME)`).
+ * @returns An async function that detaches the source. Idempotent-safe to
+ *   call even if the attach itself failed partway (best-effort).
+ * @throws {Error} If `ATTACH DATABASE` fails (e.g. the path does not exist,
+ *   or a transaction is already open on this connection).
+ * @example
+ * ```ts
+ * const detach = await attachSourceDatabase(knex, sourceDbPath);
+ * try {
+ *   await knex.transaction(async (trx) => { ... trx.raw('SELECT * FROM xfer_src.content_items') ... });
+ * } finally {
+ *   await detach();
+ * }
+ * ```
+ */
+export async function attachSourceDatabase(knex, sourceDbPath) {
+    await knex.raw('ATTACH DATABASE ? AS ' + TRANSFER_SOURCE_ALIAS, [sourceDbPath]);
+    const attached = await knex.raw('PRAGMA database_list');
+    if (!attached.some((row) => row.name === TRANSFER_SOURCE_ALIAS)) {
+        throw new Error(`Failed to attach source database at ${sourceDbPath} — PRAGMA database_list does not list "${TRANSFER_SOURCE_ALIAS}" after ATTACH.`);
+    }
+    return async () => {
+        await knex.raw('DETACH DATABASE ' + TRANSFER_SOURCE_ALIAS).catch(() => { });
+    };
+}

package/lib/transfer/build-split-config.d.ts

@@ -0,0 +1,30 @@
+import type { Config } from '../types.js';
+/**
+ * Derives the output {@link Config} for a `split`, from the source's own
+ * config plus the scope URLs the operator gave on the command line.
+ *
+ * `roots` becomes exactly the given URLs (their `withoutHash` form,
+ * matching how `crawl`'s own roots are recorded); `baseUrl` is the first
+ * one. Every other scalar field is carried over verbatim from the
+ * source — a split narrows WHICH pages are kept, it does not change HOW
+ * they were crawled (`recursive`/`interval`/`userAgent`/etc. all still
+ * describe the original session honestly). `excludes`/`excludeKeywords`/
+ * `excludeUrls` are likewise carried over unchanged (they described the
+ * source's own crawl-time exclusions, still true of the subset that
+ * survives). `version` is pinned to {@link REQUIRED_FORMAT_VERSION} and
+ * `createdCwd` is always `null`, matching `mergeArchiveConfigs`'s same
+ * two fields for the same reasons.
+ *
+ * Callers must reject a `fromList` source BEFORE calling this — `roots`
+ * for such an archive is a URL LIST, not a scope, so "roots = the given
+ * scope URLs" has no coherent meaning for it (see `docs/split.md`).
+ * @param source - The source archive's `getConfig()` result.
+ * @param scopeUrls - The scope URLs the operator gave (`withoutHash` form).
+ * @param name - The output archive's `name` (typically its basename).
+ * @returns The derived {@link Config} to write via `Archive#setConfig`.
+ * @example
+ * ```ts
+ * const config = buildSplitConfig(await source.getConfig(), ['https://example.com/blog/'], 'blog');
+ * ```
+ */
+export declare function buildSplitConfig(source: Config, scopeUrls: readonly string[], name: string): Config;