@nitpicker/archive 0.0.0-stage → 0.25.0

package/lib/verify-migration/verify-migration.js

@@ -0,0 +1,120 @@
+import { checkAnchorEdgesCount } from './check-anchor-edges-count.js';
+import { checkAnchorEdgesSum } from './check-anchor-edges-sum.js';
+import { checkContentItemsCount } from './check-content-items-count.js';
+import { checkContentTypePreservation } from './check-content-type-preservation.js';
+import { checkImageItemsCount } from './check-image-items-count.js';
+import { checkPageMetaCount } from './check-page-meta-count.js';
+import { checkReaderParity } from './check-reader-parity.js';
+import { checkResourceItemsCount } from './check-resource-items-count.js';
+import { checkUrlRoundTrip } from './check-url-round-trip.js';
+import { MigrationVerificationError } from './types.js';
+/**
+ * Runs every 0.13 migration acceptance invariant against a populated archive and
+ * returns the verified row-count summary on success. The migration script
+ * (`scripts/migrate-to-0.13.mjs`) calls this **inside** the same
+ * `knex.transaction()` block that ran `populateEntityTables` so a thrown
+ * {@link MigrationVerificationError} rolls back the entire entity populate — ref
+ * tables from ref populate stay committed because they live in a separate, additive
+ * transaction and never lose facts on re-run.
+ *
+ * The returned {@link MigrationVerificationSummary} is echoed to stdout by the
+ * migration script so operators can eyeball migration outcomes (`did we lose
+ * rows silently within the invariant window?`) from batch pipeline logs
+ * without re-opening the archive. Callers must run this inside a `db.transaction()`
+ * for the counts and the JOIN-based checks (#7 / #8) to see a consistent
+ * snapshot of the archive.
+ *
+ * Checks run sequentially in issue #194's numbered order so error messages
+ * pinpoint the earliest broken invariant. Sequential (rather than parallel)
+ * execution keeps the failure diagnostics focused: fanning eight COUNTs out
+ * at once would return the first-completed rejection rather than the
+ * lowest-numbered one, which surprises operators reading migration logs.
+ * The individual COUNT queries are fast enough at any real archive scale
+ * for sequential wall-clock cost to be irrelevant.
+ *
+ * Non-{@link MigrationVerificationError} exceptions from a check (e.g. a
+ * transient libsql error or a schema-drift SqliteError) are wrapped in a
+ * {@link MigrationVerificationError} so the migration script's `catch` sees a
+ * uniform failure surface — otherwise operators grepping stderr for
+ * `migration verification failed` would miss driver-side errors.
+ * @param trx - Knex instance or transaction connected to the archive
+ *   **after** every 0.13 migration populate has run. Must be a transaction
+ *   in production so ref counts, JOIN samples, and error rollback see the
+ *   same snapshot; non-transactional callers (unit tests) work but do not
+ *   get snapshot isolation.
+ * @returns Row-count summary of the verified archive.
+ * @throws {MigrationVerificationError} on the first invariant that does not
+ *   hold; subsequent checks are skipped.
+ */
+export async function verifyMigration(trx) {
+    try {
+        await checkContentItemsCount(trx);
+        await checkPageMetaCount(trx);
+        await checkAnchorEdgesCount(trx);
+        await checkAnchorEdgesSum(trx);
+        await checkImageItemsCount(trx);
+        await checkResourceItemsCount(trx);
+        await checkContentTypePreservation(trx);
+        await checkUrlRoundTrip(trx);
+        // Reader-level parity between the legacy tables and the
+        // new entity tables (see `checkReaderParity` for the eight totals
+        // this catches that the row-count checks above miss).
+        await checkReaderParity(trx);
+    }
+    catch (error) {
+        if (error instanceof MigrationVerificationError) {
+            throw error;
+        }
+        throw new MigrationVerificationError({
+            check: 'runtime',
+            context: {
+                underlying_error: error instanceof Error ? error.message : String(error),
+            },
+        });
+    }
+    return collectSummary(trx);
+}
+/**
+ * Runs the six row counts that populate {@link MigrationVerificationSummary}.
+ * Separate from the invariant checks so those stay side-effect-free asserts,
+ * and this runs once at the end when all invariants have already held.
+ * @param trx - Same Knex handle passed to {@link verifyMigration}.
+ */
+async function collectSummary(trx) {
+    const [contentItems, pageMeta, anchorEdges, anchorEdgesSum, imageItems, resourceItems] = await Promise.all([
+        countOf(trx, 'content_items'),
+        countOf(trx, 'page_meta'),
+        countOf(trx, 'anchor_edges'),
+        sumOf(trx, 'anchor_edges', 'count'),
+        countOf(trx, 'image_items'),
+        countOf(trx, 'resource_items'),
+    ]);
+    return {
+        contentItems,
+        pageMeta,
+        anchorEdges,
+        anchorEdgesSum,
+        imageItems,
+        resourceItems,
+    };
+}
+/**
+ * `SELECT count(*) FROM <table>` returning a plain number.
+ * @param trx - Knex handle.
+ * @param table - Table to count.
+ */
+async function countOf(trx, table) {
+    const rows = await trx(table).count({ n: '*' });
+    return Number(rows[0].n);
+}
+/**
+ * `SELECT SUM(<column>) FROM <table>` returning a plain number (`null` sum
+ * on an empty table collapses to zero).
+ * @param trx - Knex handle.
+ * @param table - Table to sum over.
+ * @param column - Column to sum.
+ */
+async function sumOf(trx, table, column) {
+    const rows = await trx(table).sum({ n: column });
+    return Number(rows[0].n ?? 0);
+}

package/lib/with-bulk-write-pragmas.d.ts

@@ -0,0 +1,51 @@
+import type { Knex } from 'knex';
+/**
+ * Runs `work` with the connection tuned for one long bulk write (the viewer
+ * read model build: one transaction that writes every `viewer_*` table and
+ * then its indexes), then puts back the values those settings had on entry.
+ * See {@link BULK_WRITE_SETTINGS} for what is changed and why. No other
+ * setting is touched: `foreign_keys`, `wal_autocheckpoint` and anything the
+ * caller set stay as they were.
+ *
+ * Together with the per-statement turns of `releaseFinishedStatements`, on a
+ * multi-GB archive with a few thousand pages, these settings take the build's
+ * peak RSS above an idle process to about half of what the read-tuned
+ * settings give, with the build time in the same range. The measurement is
+ * reproducible with `scripts/bench-read-model-build.mjs --regions` (see
+ * `scripts/README.md`). The read-tuned 64 MiB cache and 256 MiB window
+ * (`applyConnectionPragmas`) stay for reads because larger values were
+ * measured to regress the viewer's queries; smaller ones are not re-measured
+ * for reads here, so they are applied only for the duration of the bulk write.
+ *
+ * What the caller must know:
+ *
+ * - PRAGMAs are per connection, and the archive's Knex pool holds a single
+ *   connection, so "the connection" is the one every query of `knex` runs on.
+ *   A pool of several connections would tune only the one that happened to
+ *   run the `PRAGMA` statements.
+ * - It rejects a transaction object: `mmap_size` only changes between
+ *   transactions, so inside one the setting would be accepted and silently
+ *   not applied.
+ * - The values are put back whether `work` resolved, threw, or the entry
+ *   settings themselves failed partway (then every setting whose entry value
+ *   was read is put back, the one that failed to change included). If `work` throws and putting them back throws too, both are
+ *   reported in one `AggregateError`, so the bulk write's own failure (for
+ *   example `SQLITE_FULL`) is not replaced. If `work` resolved and putting
+ *   them back throws, the call still resolves with `work`'s result and the
+ *   failure is reported with `process.emitWarning`: the bulk write is
+ *   committed, and the settings only affect the speed of later reads on this
+ *   connection, so failing the call would report a finished write as failed.
+ * @param knex - The archive connection the bulk write runs on, outside any
+ *   transaction.
+ * @param work - The bulk write.
+ * @returns What `work` resolved to.
+ * @throws {Error} When `knex` is a transaction, or when `work` throws.
+ * @example
+ * const knex = accessor.getKnex();
+ * await withBulkWritePragmas(knex, async () => {
+ *   await knex.transaction(async (trx) => {
+ *     // ... bulk inserts, then CREATE INDEX ...
+ *   });
+ * });
+ */
+export declare function withBulkWritePragmas<T>(knex: Knex, work: () => Promise<T>): Promise<T>;

package/lib/with-bulk-write-pragmas.js

@@ -0,0 +1,132 @@
+/**
+ * The connection settings for one long bulk write, applied in this order and
+ * put back in the reverse order. Each changes no output, only how much
+ * memory SQLite keeps during the write:
+ *
+ * - `mmap_size = 0`: the pages of the archive file the kernel maps in count
+ *   toward the process RSS, up to the whole 256 MiB window while the build
+ *   scans the write model. Plain reads use the page cache instead.
+ * - `cache_size = -16384` (16 MiB, from 64): the cache bounds both the dirty
+ *   pages held before they spill to the WAL and the sorter's in-memory part,
+ *   which SQLite sizes from `cache_size`.
+ *
+ * Why not `temp_store = FILE` too: libsql's SQLite is compiled with
+ * `TEMP_STORE=2`, so `CREATE INDEX` sorts and the temporary trees behind
+ * `ORDER BY` / `GROUP BY` stay in memory, and moving them to files lowers
+ * the build's peak RSS further. But the files go to the OS temp directory,
+ * not next to the archive: a small `/tmp` would stop a build that completes
+ * today with `SQLITE_FULL`, and a `tmpfs` `/tmp` keeps them in RAM where the
+ * RSS does not show them. Pointing them next to the archive needs
+ * `PRAGMA temp_store_directory`, which SQLite deprecates and which is
+ * process-wide and unsafe while another thread uses SQLite.
+ */
+const BULK_WRITE_SETTINGS = [
+    ['mmap_size', '0'],
+    ['cache_size', '-16384'],
+];
+/**
+ * Runs `work` with the connection tuned for one long bulk write (the viewer
+ * read model build: one transaction that writes every `viewer_*` table and
+ * then its indexes), then puts back the values those settings had on entry.
+ * See {@link BULK_WRITE_SETTINGS} for what is changed and why. No other
+ * setting is touched: `foreign_keys`, `wal_autocheckpoint` and anything the
+ * caller set stay as they were.
+ *
+ * Together with the per-statement turns of `releaseFinishedStatements`, on a
+ * multi-GB archive with a few thousand pages, these settings take the build's
+ * peak RSS above an idle process to about half of what the read-tuned
+ * settings give, with the build time in the same range. The measurement is
+ * reproducible with `scripts/bench-read-model-build.mjs --regions` (see
+ * `scripts/README.md`). The read-tuned 64 MiB cache and 256 MiB window
+ * (`applyConnectionPragmas`) stay for reads because larger values were
+ * measured to regress the viewer's queries; smaller ones are not re-measured
+ * for reads here, so they are applied only for the duration of the bulk write.
+ *
+ * What the caller must know:
+ *
+ * - PRAGMAs are per connection, and the archive's Knex pool holds a single
+ *   connection, so "the connection" is the one every query of `knex` runs on.
+ *   A pool of several connections would tune only the one that happened to
+ *   run the `PRAGMA` statements.
+ * - It rejects a transaction object: `mmap_size` only changes between
+ *   transactions, so inside one the setting would be accepted and silently
+ *   not applied.
+ * - The values are put back whether `work` resolved, threw, or the entry
+ *   settings themselves failed partway (then every setting whose entry value
+ *   was read is put back, the one that failed to change included). If `work` throws and putting them back throws too, both are
+ *   reported in one `AggregateError`, so the bulk write's own failure (for
+ *   example `SQLITE_FULL`) is not replaced. If `work` resolved and putting
+ *   them back throws, the call still resolves with `work`'s result and the
+ *   failure is reported with `process.emitWarning`: the bulk write is
+ *   committed, and the settings only affect the speed of later reads on this
+ *   connection, so failing the call would report a finished write as failed.
+ * @param knex - The archive connection the bulk write runs on, outside any
+ *   transaction.
+ * @param work - The bulk write.
+ * @returns What `work` resolved to.
+ * @throws {Error} When `knex` is a transaction, or when `work` throws.
+ * @example
+ * const knex = accessor.getKnex();
+ * await withBulkWritePragmas(knex, async () => {
+ *   await knex.transaction(async (trx) => {
+ *     // ... bulk inserts, then CREATE INDEX ...
+ *   });
+ * });
+ */
+export async function withBulkWritePragmas(knex, work) {
+    if (knex.isTransaction === true) {
+        throw new Error('withBulkWritePragmas must be called outside a transaction: mmap_size only changes between transactions');
+    }
+    const valuesOnEntry = [];
+    let result;
+    try {
+        for (const [name, value] of BULK_WRITE_SETTINGS) {
+            valuesOnEntry.push([name, await readPragmaValue(knex, name)]);
+            await knex.raw(`PRAGMA ${name} = ${value}`);
+        }
+        result = await work();
+    }
+    catch (workError) {
+        try {
+            await putBack(knex, valuesOnEntry);
+        }
+        catch (restoreError) {
+            throw new AggregateError([workError, restoreError], 'The bulk write failed AND putting the connection settings back afterwards also failed.');
+        }
+        throw workError;
+    }
+    try {
+        await putBack(knex, valuesOnEntry);
+    }
+    catch (restoreError) {
+        process.emitWarning(`The bulk write succeeded, but putting the connection settings back failed; later reads on this connection may be slower: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}`, { code: 'NITPICKER_BULK_WRITE_PRAGMAS_NOT_RESTORED' });
+    }
+    return result;
+}
+/**
+ * Reads one PRAGMA's current value as the text to set it back with.
+ * @param knex - The connection.
+ * @param name - The PRAGMA name.
+ * @returns The value.
+ * @throws {Error} When the driver returns no value for it.
+ */
+async function readPragmaValue(knex, name) {
+    const rows = await knex.raw(`PRAGMA ${name}`);
+    const value = Array.isArray(rows)
+        ? rows[0]?.[name]
+        : undefined;
+    if (typeof value !== 'number' && typeof value !== 'bigint') {
+        throw new TypeError(`Could not read the current value of PRAGMA ${name}`);
+    }
+    return String(value);
+}
+/**
+ * Sets each PRAGMA back to the value it had on entry, in the reverse order.
+ * @param knex - The connection.
+ * @param valuesOnEntry - The PRAGMAs changed so far, with their entry values.
+ */
+async function putBack(knex, valuesOnEntry) {
+    for (const [name, value] of valuesOnEntry.toReversed()) {
+        await knex.raw(`PRAGMA ${name} = ${value}`);
+    }
+}

package/lib/write-queue.d.ts

@@ -0,0 +1,32 @@
+/**
+ * A lightweight serial write queue that ensures asynchronous operations
+ * execute one at a time in FIFO order.
+ *
+ * Used to serialize concurrent Archive writes from crawler event handlers,
+ * preventing SQLite write-lock contention under high parallelism.
+ */
+export declare class WriteQueue {
+    #private;
+    /**
+     * The number of operations currently waiting or executing in the queue.
+     * @returns The current pending operation count.
+     */
+    get pending(): number;
+    /**
+     * Returns a promise that resolves when all currently enqueued operations
+     * have completed.
+     * @returns A promise that resolves when the queue is drained.
+     */
+    drain(): Promise<void>;
+    /**
+     * Enqueues an operation to run after all previously enqueued operations
+     * have completed. Operations are guaranteed to execute serially in the
+     * order they were enqueued. Both async and synchronous (throwing)
+     * operations are supported.
+     * @template T The resolved type of the operation's promise.
+     * @param operation - The function to execute. May return a `Promise` or a plain value.
+     * @returns A promise that resolves with the operation's result, or rejects
+     *          if the operation throws.
+     */
+    enqueue<T>(operation: () => Promise<T> | T): Promise<T>;
+}

package/lib/write-queue.js

@@ -0,0 +1,55 @@
+/**
+ * A lightweight serial write queue that ensures asynchronous operations
+ * execute one at a time in FIFO order.
+ *
+ * Used to serialize concurrent Archive writes from crawler event handlers,
+ * preventing SQLite write-lock contention under high parallelism.
+ */
+export class WriteQueue {
+    /** The tail of the promise chain used for serialization. */
+    #chain = Promise.resolve();
+    /** The number of operations currently waiting or executing in the queue. */
+    #pending = 0;
+    /**
+     * The number of operations currently waiting or executing in the queue.
+     * @returns The current pending operation count.
+     */
+    get pending() {
+        return this.#pending;
+    }
+    /**
+     * Returns a promise that resolves when all currently enqueued operations
+     * have completed.
+     * @returns A promise that resolves when the queue is drained.
+     */
+    async drain() {
+        await this.#chain;
+    }
+    /**
+     * Enqueues an operation to run after all previously enqueued operations
+     * have completed. Operations are guaranteed to execute serially in the
+     * order they were enqueued. Both async and synchronous (throwing)
+     * operations are supported.
+     * @template T The resolved type of the operation's promise.
+     * @param operation - The function to execute. May return a `Promise` or a plain value.
+     * @returns A promise that resolves with the operation's result, or rejects
+     *          if the operation throws.
+     */
+    enqueue(operation) {
+        this.#pending++;
+        return new Promise((resolve, reject) => {
+            this.#chain = this.#chain.then(async () => {
+                try {
+                    const result = await operation();
+                    resolve(result);
+                }
+                catch (error) {
+                    reject(error);
+                }
+                finally {
+                    this.#pending--;
+                }
+            });
+        });
+    }
+}