@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/init-schema.js

@@ -0,0 +1,124 @@
+import { createAdjunctTables } from './create-adjunct-tables.js';
+import { createEntityTables } from './create-entity-tables.js';
+import { createRefTables } from './create-ref-tables.js';
+import { createTemplateTables } from './create-template-tables.js';
+/**
+ * Initializes the archive database schema if tables do not exist.
+ *
+ * The schema is composed of five groups, each owned by a dedicated DDL
+ * function so the migration script (`scripts/migrate-to-0.13.mjs`) can
+ * provision the exact same shapes on archives it upgrades:
+ *
+ * - **`info`** (inline below): single-row crawl configuration.
+ * - **Ref / header dictionary tables** ({@link createRefTables}):
+ *   `url_refs`, `content_type_refs`, `text_refs`, `json_refs`,
+ *   `blob_refs`, `header_name_refs`, `header_value_refs`, `header_sets`,
+ *   `header_set_entries`, `header_flags`. See that file for column-level
+ *   rationale and the reason `blob_refs` uses a regular rowid PK instead
+ *   of WITHOUT ROWID.
+ * - **Entity / edge tables** ({@link createEntityTables}):
+ *   `content_items`, `page_meta`, `resource_items`, `anchor_edges`,
+ *   `resource_ref_edges`, `image_items` — the write-model the crawler
+ *   writes during a crawl and every reader queries. Must run AFTER
+ *   `createRefTables` because most entity tables reference ref-table PKs.
+ * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
+ *   `crawl_errors`, `technology_signals`, `page_technologies`, `page_jsonld`, `list_reconcile_runs`,
+ *   `page_html_blobs` +
+ *   `page_html_ref`. Must run AFTER `createEntityTables` because the
+ *   page-scoped tables FK into `content_items(id)`.
+ * - **Template classification tables** ({@link createTemplateTables}):
+ *   `page_templates`, `page_template_clusters`, `page_template_labels` —
+ *   the crawl-end DOM-structure classification, kept apart from the
+ *   adjunct tables because it has its own producer and write path.
+ *
+ * The legacy flat write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) are deliberately NOT created:
+ * they only exist inside pre-0.13 archives, where they serve as the
+ * populate source for `scripts/migrate-to-0.13.mjs` before that script
+ * drops them.
+ *
+ * **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ * `CREATE TABLE` because SQLite only honors `page_size` changes against
+ * an empty database, and `journal_mode = WAL` is persistent. Other
+ * per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.13 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects older archives with a friendly
+ * error naming the migration scripts to run; `v0.x` policy allows
+ * breaking changes.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function initSchema(instance) {
+    // Only the one-shot work (PRAGMAs + `info` creation) is gated on the
+    // `info` table's existence. The DDL groups below run on EVERY
+    // call: each is internally idempotent (sentinel / IF NOT EXISTS /
+    // per-table guards), and re-running them self-heals an archive whose
+    // provisioning crashed partway through — `info` created but a later
+    // group missing. With a single all-or-nothing gate, such a stub would
+    // short-circuit here forever and every subsequent write would die with
+    // `no such table` and no way back.
+    if (!(await instance.schema.hasTable('info'))) {
+        // Page size must be set on an empty database file; once any data is
+        // written, only VACUUM can change it. journal_mode is also one-shot
+        // (persistent) and so stays here.
+        await instance.raw('PRAGMA page_size = 16384');
+        await instance.raw('PRAGMA journal_mode = WAL');
+        await instance.schema.createTable('info', (t) => {
+            t.increments('id');
+            t.string('version');
+            t.string('name');
+            t.string('baseUrl');
+            t.json('roots');
+            t.boolean('recursive');
+            t.integer('interval');
+            t.boolean('image');
+            t.boolean('fetchExternal');
+            t.integer('parallels');
+            t.json('excludes');
+            t.json('excludeKeywords');
+            t.json('excludeUrls');
+            t.integer('maxExcludedDepth');
+            t.integer('retry');
+            t.boolean('fromList');
+            t.boolean('disableQueries');
+            t.string('userAgent');
+            t.boolean('ignoreRobots');
+            t.string('mainContentSelector');
+            t.json('requestHeaderNames');
+            t.string('createdCwd');
+        });
+    }
+    // 0.13 ref / header dictionary tables. DDL + column-level rationale
+    // lives in {@link createRefTables}, shared with the migration script's
+    // schema catch-up — a divergence between the two paths would silently
+    // break the population step's UNIQUE / CHECK contract.
+    // `createRefTables` uses bare CREATE TABLE, so it needs a sentinel
+    // guard here (all 10 tables are created together — any one of them
+    // works as the sentinel).
+    if (!(await instance.schema.hasTable('url_refs'))) {
+        await createRefTables(instance);
+    }
+    // 0.13 core entity / edge tables — the write-model the crawler writes
+    // during a crawl and every reader queries. MUST run after
+    // {@link createRefTables} because `content_items`, `page_meta`,
+    // `resource_items`, `anchor_edges`, and `image_items` all reference
+    // the ref tables (`url_refs`, `content_type_refs`, `text_refs`,
+    // `json_refs`, `blob_refs`, `header_sets`) via FK clauses.
+    await createEntityTables(instance);
+    // Adjunct tables that FK into `content_items` (page_errors /
+    // technology_signals / page_technologies / page_jsonld /
+    // page_html_*) plus the standalone log tables
+    // (crawl_errors / list_reconcile_runs). MUST run after
+    // {@link createEntityTables} so the FK targets exist. DDL +
+    // column-level rationale lives in {@link createAdjunctTables}, which is
+    // shared with `scripts/migrate-to-0.13.mjs` — a divergence between the
+    // two paths is exactly how migrated archives ended up with stale
+    // `REFERENCES pages(id)` declarations in the pre-0.13 era.
+    await createAdjunctTables(instance);
+    // Template classification tables (page_templates / page_template_clusters /
+    // page_template_labels). Separate from the adjunct group because the
+    // producer is the crawl-end classification step, not the crawl write path.
+    // `page_templates` FKs into `content_items(id)`, so this too
+    // MUST run after {@link createEntityTables}.
+    await createTemplateTables(instance);
+}

package/lib/is-inventory-source.d.ts

@@ -0,0 +1,21 @@
+import type { PageSource } from './types.js';
+/**
+ * Predicate that returns `true` when the given page source value belongs to
+ * the inventory chain — i.e. it is one of the `'inventory-*'` variants of
+ * {@link PageSource}.
+ *
+ * Centralises the membership check that decides whether lineage
+ * propagation should fire. Three call sites used to inline
+ * `s === 'inventory-seed' || s === 'inventory-discovered'`, which is both
+ * a DRY violation AND a future-proofing trap: when a new inventory-family
+ * label gets added (e.g. `'inventory-promoted'`), every inlined check has
+ * to be located and updated by hand. Routing through this predicate
+ * keeps the membership rule in one place.
+ *
+ * Returns `false` for `undefined` so callers can pass the raw `source`
+ * column value (which is non-NULL in the DB schema but reads as
+ * `undefined` from a missing row in JS) without a separate null check.
+ * @param source - The source value to test, or `undefined` when no row matched.
+ * @returns `true` if the source is in the inventory chain, `false` otherwise.
+ */
+export declare function isInventorySource(source: PageSource | undefined): boolean;

package/lib/is-inventory-source.js

@@ -0,0 +1,22 @@
+/**
+ * Predicate that returns `true` when the given page source value belongs to
+ * the inventory chain — i.e. it is one of the `'inventory-*'` variants of
+ * {@link PageSource}.
+ *
+ * Centralises the membership check that decides whether lineage
+ * propagation should fire. Three call sites used to inline
+ * `s === 'inventory-seed' || s === 'inventory-discovered'`, which is both
+ * a DRY violation AND a future-proofing trap: when a new inventory-family
+ * label gets added (e.g. `'inventory-promoted'`), every inlined check has
+ * to be located and updated by hand. Routing through this predicate
+ * keeps the membership rule in one place.
+ *
+ * Returns `false` for `undefined` so callers can pass the raw `source`
+ * column value (which is non-NULL in the DB schema but reads as
+ * `undefined` from a missing row in JS) without a separate null check.
+ * @param source - The source value to test, or `undefined` when no row matched.
+ * @returns `true` if the source is in the inventory chain, `false` otherwise.
+ */
+export function isInventorySource(source) {
+    return source === 'inventory-seed' || source === 'inventory-discovered';
+}

package/lib/is-within-outage-window.d.ts

@@ -0,0 +1,49 @@
+/**
+ * A resolved (closed) network-outage interval, as consumed by
+ * {@link isWithinOutageWindow}.
+ *
+ * This is deliberately NOT the raw `network_outages` row shape: a row whose
+ * `ended_at` is still `NULL` (the crawl session crashed before recovery was
+ * observed) must be resolved to a concrete `endedAt` by the caller — e.g.
+ * `listNetworkOutages` clamping it to the latest observed timestamp in the
+ * archive — before it reaches this function. Accepting a nullable `endedAt`
+ * here would let a crashed session's outage silently swallow every
+ * subsequent timestamp as "network-caused" forever.
+ */
+export interface OutageWindow {
+    /** Epoch ms the outage is considered to have started (already backdated to the earliest triggering error). */
+    readonly startedAt: number;
+    /** Epoch ms the outage is considered to have ended. Never `NULL` — see the interface docstring. */
+    readonly endedAt: number;
+}
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export declare function isWithinOutageWindow(timestamp: number, windows: readonly OutageWindow[]): boolean;

package/lib/is-within-outage-window.js

@@ -0,0 +1,33 @@
+/**
+ * Decide whether a timestamp falls inside any recorded network-outage
+ * window — i.e. whether an error observed at that instant is more likely
+ * attributable to the operator's own network than to the target site.
+ *
+ * Pure and dependency-free by design: no DB handle, no `Crawler` instance.
+ * This is the single predicate every consumer (`resetFailedPages`,
+ * `listDnsBurnedHostCandidates`, `getSummary`'s attribution split) is
+ * expected to call, so the inclusive/exclusive boundary decision below is
+ * made exactly once.
+ *
+ * **Boundaries are inclusive on both ends.** A timestamp equal to
+ * `startedAt` or `endedAt` counts as inside the window. This errs toward
+ * attributing borderline errors to the outage rather than to the site:
+ * classifying a genuinely network-caused failure as "unknown/site kills it
+ * forever" (a false negative) is worse than the reverse (a false positive
+ * merely costs one extra retry pass before the kind classifier sorts it out
+ * again).
+ * @param timestamp - Epoch ms to test (typically an error's `createdAt`).
+ * @param windows - Resolved outage windows to test against. An empty array
+ *   always yields `false` — this is what makes an archive with no
+ *   `network_outages` rows (every archive created before this feature, or
+ *   any crawl with no detected outage) behave identically to today.
+ * @returns `true` if `timestamp` falls within any window.
+ * @example
+ * ```ts
+ * isWithinOutageWindow(1_000, [{ startedAt: 500, endedAt: 1_500 }]); // true
+ * isWithinOutageWindow(1_000, []); // false — no recorded outages
+ * ```
+ */
+export function isWithinOutageWindow(timestamp, windows) {
+    return windows.some((window) => timestamp >= window.startedAt && timestamp <= window.endedAt);
+}

package/lib/isolated-cluster/create-isolated-cluster-run.d.ts

@@ -0,0 +1,24 @@
+import type { IsolatedClusterRun } from '@nitpicker/core/types';
+/**
+ * Starts the computation of the connected components of the isolated pages
+ * (the inventory pages linked to only from other inventory pages), fed with
+ * the link edges as SQLite returns them and never holding them.
+ *
+ * `@nitpicker/query` reaches the native addon through this archive function
+ * (see ARCHITECTURE.md's dependency rule); what an edge counts as, how
+ * redirect and alias hops are followed and how the components are numbered
+ * are the docs of `createIsolatedClusterRun` in `@nitpicker/core`, which this
+ * calls as it is.
+ * @param candidateIds - The candidate page ids, in the order the components
+ *   are numbered in (ascending page id for the viewer read model's `component_id`).
+ * @param redirects - The redirect and alias hops of the whole archive,
+ *   `[source, destination, ...]` by page id.
+ * @returns The run.
+ * @example
+ * ```ts
+ * const run = createIsolatedClusterRun([10, 20, 30], [40, 30]);
+ * run.unionEdges([10, 20, 20, 40]);
+ * run.labels(); // Uint32Array [0, 0, 0]
+ * ```
+ */
+export declare function createIsolatedClusterRun(candidateIds: ArrayLike<number>, redirects: ArrayLike<number>): IsolatedClusterRun;

package/lib/isolated-cluster/create-isolated-cluster-run.js

@@ -0,0 +1,26 @@
+import { createIsolatedClusterRun as createIsolatedClusterRunNative } from '@nitpicker/core/create-isolated-cluster-run';
+/**
+ * Starts the computation of the connected components of the isolated pages
+ * (the inventory pages linked to only from other inventory pages), fed with
+ * the link edges as SQLite returns them and never holding them.
+ *
+ * `@nitpicker/query` reaches the native addon through this archive function
+ * (see ARCHITECTURE.md's dependency rule); what an edge counts as, how
+ * redirect and alias hops are followed and how the components are numbered
+ * are the docs of `createIsolatedClusterRun` in `@nitpicker/core`, which this
+ * calls as it is.
+ * @param candidateIds - The candidate page ids, in the order the components
+ *   are numbered in (ascending page id for the viewer read model's `component_id`).
+ * @param redirects - The redirect and alias hops of the whole archive,
+ *   `[source, destination, ...]` by page id.
+ * @returns The run.
+ * @example
+ * ```ts
+ * const run = createIsolatedClusterRun([10, 20, 30], [40, 30]);
+ * run.unionEdges([10, 20, 20, 40]);
+ * run.labels(); // Uint32Array [0, 0, 0]
+ * ```
+ */
+export function createIsolatedClusterRun(candidateIds, redirects) {
+    return createIsolatedClusterRunNative(candidateIds, redirects);
+}

package/lib/libsql-dialect.d.ts

@@ -0,0 +1,115 @@
+import type { Knex } from 'knex';
+/**
+ * Minimal shape of the query object Knex passes to `_query`.
+ * `response` / `context` are what the `better-sqlite3` dialect fills in for
+ * statements that return no rows.
+ */
+interface DialectQuery {
+    sql?: string;
+    response?: unknown;
+    context?: unknown;
+}
+/**
+ * Type of Knex's internal `better-sqlite3` dialect: `Knex.Client` plus the two
+ * members {@link LibsqlDialect} relies on that the published declaration lacks.
+ * Declared as a class (not a constructor type) so the static side of
+ * `Knex.Client` is kept and `knex({ client: LibsqlDialect })` still type-checks.
+ */
+declare class BetterSqlite3Client extends Knex.Client {
+    /**
+     * `true` only on the per-transaction client Knex derives from the main one
+     * (`makeTxClient`), through which every query of a transaction runs.
+     */
+    transacting?: boolean;
+    protected _query(connection: unknown, query: DialectQuery): Promise<DialectQuery>;
+}
+/**
+ * Untyped reference to Knex's internal `better-sqlite3` dialect.
+ *
+ * Knex's published `exports` do not include this subpath, so TypeScript cannot
+ * resolve a declaration file for it. The dialect is a concrete subclass of
+ * `Knex.Client` at runtime, so the safe cast is intentional.
+ */
+declare const Base: typeof BetterSqlite3Client;
+/**
+ * Knex dialect that reuses the `better-sqlite3` adapter logic but swaps the
+ * underlying driver for `libsql`. `libsql` ships pre-compiled binaries via the
+ * `@libsql/<platform>-<arch>` optionalDependencies pattern, so no postinstall
+ * download from GitHub Releases is required.
+ *
+ * A transaction that SQLite rolls back by itself (for example on a full disk)
+ * rejects with the original error, not with the failure of the `ROLLBACK`
+ * Knex sends afterwards.
+ * @example
+ * ```ts
+ * import knex from 'knex';
+ *
+ * import { LibsqlDialect } from './libsql-dialect.js';
+ *
+ * const db = knex({
+ *   client: LibsqlDialect,
+ *   connection: { filename: 'archive.sqlite' },
+ *   useNullAsDefault: true,
+ * });
+ *
+ * try {
+ *   await db.transaction(async (trx) => {
+ *     await trx.raw('INSERT INTO t (v) VALUES (?)', ['...']);
+ *   });
+ * } catch (error) {
+ *   // `SQLITE_FULL` when the disk filled up mid-transaction,
+ *   // not `ROLLBACK - cannot rollback - no transaction is active`.
+ *   console.error((error as { code?: string }).code);
+ * }
+ * ```
+ */
+export declare class LibsqlDialect extends Base {
+    /**
+     * Returns the `libsql` constructor as the SQLite driver.
+     *
+     * Typed as `unknown` to avoid leaking the `Libsql.DatabaseConstructor`
+     * namespace from a non-re-exported module across the public API surface.
+     */
+    protected _driver(): unknown;
+    /**
+     * Skips a `ROLLBACK` for a transaction SQLite has already rolled back, and
+     * delegates every other statement to the `better-sqlite3` implementation.
+     * Knex calls it from outside the client (`executeQuery`), so it must stay
+     * reachable at runtime; `protected` is only a compile-time marker, as for
+     * `_driver`.
+     *
+     * Why: SQLite rolls a transaction back by itself on some errors
+     * (`SQLITE_FULL`, `SQLITE_IOERR`, `SQLITE_INTERRUPT`, `SQLITE_NOMEM` in the
+     * documented list, which SQLite reserves the right to extend) and documents
+     * that a following `ROLLBACK` then fails harmlessly
+     * (https://www.sqlite.org/lang_transaction.html, "Response To Errors Within
+     * A Transaction"). Knex does not treat it as harmless:
+     * `Transaction#query` replaces the error being rolled back with the failure
+     * of the `ROLLBACK`, so callers would see `ROLLBACK - cannot rollback - no
+     * transaction is active` instead of the `SQLITE_FULL` that caused it.
+     *
+     * Why not in each caller: every transaction in the archive goes through
+     * this dialect, and all of them lose the original error the same way.
+     *
+     * The skip needs all of: a Knex transaction client (`transacting`), a
+     * connection that reports `inTransaction === false`, and a rollback
+     * statement. Anything else is executed, so a stray `ROLLBACK` through the
+     * main client, or on a connection that does not expose the flag, stays
+     * visible as the programming error it is.
+     *
+     * `ROLLBACK TO SAVEPOINT` is skipped under the same conditions: the
+     * automatic rollback discards the savepoints with the transaction. A caller
+     * that catches the error of a nested transaction and carries on with the
+     * outer one is then running in autocommit mode, because the outer
+     * transaction no longer exists.
+     *
+     * The result mirrors what the `better-sqlite3` dialect records for a
+     * statement that returns no rows. Nothing ran, so there is no row id or
+     * change count to report, and `0` is fine even when `safeIntegers` makes
+     * real results `bigint`.
+     * @param connection - The raw `libsql` connection.
+     * @param query - The Knex query object.
+     */
+    protected _query(connection: unknown, query: DialectQuery): Promise<DialectQuery>;
+}
+export {};

package/lib/libsql-dialect.js

@@ -0,0 +1,116 @@
+// @ts-expect-error - Internal Knex subpath without published type declarations
+import Client_BetterSQLite3 from 'knex/lib/dialects/better-sqlite3/index.js';
+import libsql from 'libsql';
+import { dbLog } from './debug.js';
+/**
+ * Untyped reference to Knex's internal `better-sqlite3` dialect.
+ *
+ * Knex's published `exports` do not include this subpath, so TypeScript cannot
+ * resolve a declaration file for it. The dialect is a concrete subclass of
+ * `Knex.Client` at runtime, so the safe cast is intentional.
+ */
+const Base = Client_BetterSQLite3;
+/**
+ * Matches `ROLLBACK` and `ROLLBACK TO SAVEPOINT`. Capture group 1 is the
+ * statement kind, for logging.
+ *
+ * Same prefix test Knex's `Transaction#query` (`lib/execution/transaction.js`)
+ * uses to recognise a rollback statement. Keep the two in step when upgrading
+ * Knex.
+ */
+const ROLLBACK_PATTERN = /^(ROLLBACK(?:\s+TO\s+SAVEPOINT)?)\b/i;
+/**
+ * Knex dialect that reuses the `better-sqlite3` adapter logic but swaps the
+ * underlying driver for `libsql`. `libsql` ships pre-compiled binaries via the
+ * `@libsql/<platform>-<arch>` optionalDependencies pattern, so no postinstall
+ * download from GitHub Releases is required.
+ *
+ * A transaction that SQLite rolls back by itself (for example on a full disk)
+ * rejects with the original error, not with the failure of the `ROLLBACK`
+ * Knex sends afterwards.
+ * @example
+ * ```ts
+ * import knex from 'knex';
+ *
+ * import { LibsqlDialect } from './libsql-dialect.js';
+ *
+ * const db = knex({
+ *   client: LibsqlDialect,
+ *   connection: { filename: 'archive.sqlite' },
+ *   useNullAsDefault: true,
+ * });
+ *
+ * try {
+ *   await db.transaction(async (trx) => {
+ *     await trx.raw('INSERT INTO t (v) VALUES (?)', ['...']);
+ *   });
+ * } catch (error) {
+ *   // `SQLITE_FULL` when the disk filled up mid-transaction,
+ *   // not `ROLLBACK - cannot rollback - no transaction is active`.
+ *   console.error((error as { code?: string }).code);
+ * }
+ * ```
+ */
+export class LibsqlDialect extends Base {
+    /**
+     * Returns the `libsql` constructor as the SQLite driver.
+     *
+     * Typed as `unknown` to avoid leaking the `Libsql.DatabaseConstructor`
+     * namespace from a non-re-exported module across the public API surface.
+     */
+    _driver() {
+        return libsql;
+    }
+    /**
+     * Skips a `ROLLBACK` for a transaction SQLite has already rolled back, and
+     * delegates every other statement to the `better-sqlite3` implementation.
+     * Knex calls it from outside the client (`executeQuery`), so it must stay
+     * reachable at runtime; `protected` is only a compile-time marker, as for
+     * `_driver`.
+     *
+     * Why: SQLite rolls a transaction back by itself on some errors
+     * (`SQLITE_FULL`, `SQLITE_IOERR`, `SQLITE_INTERRUPT`, `SQLITE_NOMEM` in the
+     * documented list, which SQLite reserves the right to extend) and documents
+     * that a following `ROLLBACK` then fails harmlessly
+     * (https://www.sqlite.org/lang_transaction.html, "Response To Errors Within
+     * A Transaction"). Knex does not treat it as harmless:
+     * `Transaction#query` replaces the error being rolled back with the failure
+     * of the `ROLLBACK`, so callers would see `ROLLBACK - cannot rollback - no
+     * transaction is active` instead of the `SQLITE_FULL` that caused it.
+     *
+     * Why not in each caller: every transaction in the archive goes through
+     * this dialect, and all of them lose the original error the same way.
+     *
+     * The skip needs all of: a Knex transaction client (`transacting`), a
+     * connection that reports `inTransaction === false`, and a rollback
+     * statement. Anything else is executed, so a stray `ROLLBACK` through the
+     * main client, or on a connection that does not expose the flag, stays
+     * visible as the programming error it is.
+     *
+     * `ROLLBACK TO SAVEPOINT` is skipped under the same conditions: the
+     * automatic rollback discards the savepoints with the transaction. A caller
+     * that catches the error of a nested transaction and carries on with the
+     * outer one is then running in autocommit mode, because the outer
+     * transaction no longer exists.
+     *
+     * The result mirrors what the `better-sqlite3` dialect records for a
+     * statement that returns no rows. Nothing ran, so there is no row id or
+     * change count to report, and `0` is fine even when `safeIntegers` makes
+     * real results `bigint`.
+     * @param connection - The raw `libsql` connection.
+     * @param query - The Knex query object.
+     */
+    async _query(connection, query) {
+        // `connection` may be undefined: `super._query` then reports `No connection provided`.
+        const inTransaction = connection
+            ?.inTransaction;
+        const rollback = ROLLBACK_PATTERN.exec(query.sql ?? '');
+        if (this.transacting === true && inTransaction === false && rollback) {
+            dbLog('Skipping %s: SQLite has already rolled the transaction back', rollback[1]);
+            query.response = { changes: 0, lastInsertRowid: 0 };
+            query.context = { lastID: 0, changes: 0 };
+            return query;
+        }
+        return super._query(connection, query);
+    }
+}

package/lib/limited-page-ids.d.ts

@@ -0,0 +1,9 @@
+import type { Knex } from 'knex';
+/**
+ * Returns a Knex subquery builder that selects page IDs with pagination,
+ * ordered by `content_items.crawl_order` (nulls last), excluding
+ * redirected pages.
+ * @param limit - The maximum number of page IDs to return.
+ * @param offset - The number of page IDs to skip before returning results.
+ */
+export declare function limitedPageIds(limit: number, offset: number): (qb: Knex.QueryBuilder<Record<string, unknown>, unknown>) => Promise<void>;

package/lib/limited-page-ids.js

@@ -0,0 +1,18 @@
+/**
+ * Returns a Knex subquery builder that selects page IDs with pagination,
+ * ordered by `content_items.crawl_order` (nulls last), excluding
+ * redirected pages.
+ * @param limit - The maximum number of page IDs to return.
+ * @param offset - The number of page IDs to skip before returning results.
+ */
+export function limitedPageIds(limit, offset) {
+    return async (qb) => {
+        await qb
+            .select('id')
+            .from('content_items')
+            .orderByRaw('`crawl_order` ASC NULLS LAST')
+            .whereNull('redirect_dest_id')
+            .limit(limit)
+            .offset(offset);
+    };
+}

package/lib/meta/assert-compatible-version.d.ts

@@ -0,0 +1,60 @@
+import type { Knex } from 'knex';
+/**
+ * Minimum `info.version` this build accepts. Archives older than this must
+ * be upgraded by running the matching migration script before they can be
+ * opened (see {@link IncompatibleArchiveError}'s message for the mapping
+ * from archive version to script).
+ *
+ * This value tracks the **archive format cut**, not the npm `pkg.version`:
+ * a format-breaking change bumps it ahead of the corresponding release,
+ * and releases that do not cut the format leave it unchanged. The crawler
+ * writes this exact value into every new archive's `info.version`.
+ *
+ * History:
+ *
+ * - **pre-0.10**: HTML snapshots in `snapshot-html.zip` (#75), then
+ *   relocated to `page_html_blobs` (#84); pages table has flat `noindex`,
+ *   `og:type`-style columns derived from beholder 2.x's flat `Meta`.
+ * - **0.10.0**: `page_html_blobs` BLOB storage (#75/#84) +
+ *   nested-`Meta`-derived flat columns, `meta_extras` JSON, `page_tags` /
+ *   `page_jsonld` tables, denormalised aggregates (#85).
+ * - **0.13.0**: this build. Write-model refactor (epic #103) —
+ *   `content_items` / `page_meta` / `resource_items` / `anchor_edges` /
+ *   `resource_ref_edges` / `image_items` entity tables plus `url_refs` /
+ *   `text_refs` / `content_type_refs` / `json_refs` / `blob_refs` /
+ *   `header_flags` ref tables. Readers query the new tables exclusively;
+ *   pre-0.13 archives must be upgraded with `scripts/migrate-to-0.13.mjs`
+ *   which bumps `info.version` to `0.13.0` on completion.
+ * @example
+ * import { REQUIRED_FORMAT_VERSION } from '@nitpicker/archive/meta/assert-compatible-version';
+ *
+ * // Warn an operator before attempting to open an old archive:
+ * if (compareSemver(archiveInfo.version, REQUIRED_FORMAT_VERSION) < 0) {
+ *   console.error(`Archive needs migration to format ${REQUIRED_FORMAT_VERSION}`);
+ * }
+ */
+export declare const REQUIRED_FORMAT_VERSION = "0.13.0";
+/**
+ * Verifies that the archive's on-disk format is compatible with this build.
+ *
+ * Compares the archive's `info.version` (a semver string written by
+ * `setConfig` at archive-create time, or by the migration script) against
+ * {@link REQUIRED_FORMAT_VERSION}. Older archives throw
+ * {@link IncompatibleArchiveError} pointing the operator at the migration
+ * script.
+ *
+ * Called from `Database.#init` for both writer and read-only (stub viewer)
+ * connections so old `._nitpicker-*` stubs surface the error too. New
+ * archives where the `info` table does not yet exist are tolerated —
+ * `initSchema` will fill them in next.
+ *
+ * The check is intentionally version-string-only, not schema-shape-based:
+ * `info.version` is the single declared source of truth, and a v0.10
+ * `migrate-to-0.10.mjs` run bumps it explicitly so the assertion passes
+ * once migration completes.
+ * @param instance - The Knex query builder for the archive's libsql connection.
+ * @throws {IncompatibleArchiveError} when `info.version` is older than
+ *   {@link REQUIRED_FORMAT_VERSION}, or missing entirely on a non-empty
+ *   archive.
+ */
+export declare function assertCompatibleVersion(instance: Knex): Promise<void>;