@eltjs/elt-sqlite 0.2.0 → 0.3.0

package/dist/sqlite-deduplicating-writer.d.ts

@@ -1,21 +1,23 @@
 import type { DatabaseSync, SQLInputValue } from 'node:sqlite';
 import type { CopyConfiguration, Deduplication, KeyValue } from '@eltjs/elt';
 import type { SQLiteColumn } from './sqlite-column.ts';
+import type { SQLiteDestination } from './sqlite-destination.ts';
+import type { SQLiteHistory } from './sqlite-history.ts';
 import type { SQLiteTable } from './sqlite-table.ts';
-import { SQLiteWriter } from './sqlite-writer.ts';
+import { SQLiteWriter, type Stamp } from './sqlite-writer.ts';
 export declare class SQLiteDeduplicatingWriter extends SQLiteWriter {
     #private;
     readonly deduplication: Deduplication;
     readonly keys: readonly SQLiteColumn[];
     readonly cursor?: SQLiteColumn;
-    constructor(configuration: CopyConfiguration, path: string, table: SQLiteTable);
+    constructor(configuration: CopyConfiguration, destination: SQLiteDestination, table: SQLiteTable);
     protected get replaces(): boolean;
     protected inspect(database: DatabaseSync): void;
     protected initialize(database: DatabaseSync, replacing: boolean): void;
-    protected replace(database: DatabaseSync): void;
     protected build(database: DatabaseSync, into: string): void;
     protected adopt(database: DatabaseSync): void;
-    protected merge(database: DatabaseSync, stage: string, loadedAt: string, into: string): void;
+    protected kept(stage: string): string;
+    protected merge(database: DatabaseSync, stage: string, { loadedAt, committedIn }: Stamp, into: string, history: SQLiteHistory | null): void;
     protected encode(record: unknown): SQLInputValue[];
     protected deletionKeys(key: Readonly<Record<string, KeyValue>>): readonly [readonly SQLiteColumn[], SQLInputValue[]];
 }

package/dist/sqlite-deduplicating-writer.js

@@ -1,12 +1,12 @@
 import { hash } from 'node:crypto';
 import { SQLiteColumns } from './sqlite-columns.js';
-import { SQLiteWriter, op, seq } from './sqlite-writer.js';
+import { SQLiteWriter, op, seq, version } from './sqlite-writer.js';
 export class SQLiteDeduplicatingWriter extends SQLiteWriter {
     deduplication;
     keys;
     cursor;
-    constructor(configuration, path, table) {
-        super(configuration, path, table);
+    constructor(configuration, destination, table) {
+        super(configuration, destination, table);
         this.deduplication = configuration.deduplication();
         const inferred = SQLiteColumns.fromSchema(configuration.stream.jsonSchema);
         const column = (field) => {
@@ -55,18 +55,14 @@ export class SQLiteDeduplicatingWriter extends SQLiteWriter {
                 .get())
             throw new TypeError('Existing rows repeat a deduplication key; replace them with overwrite_dedup before deduplicating incrementally');
     }
-    // A replacing load keeps none of the stored rows: its index is built once
-    // the commit has emptied the table.
+    // A replacing commit builds the index once the rows that leave are gone:
+    // under a narrower key, they can repeat a key.
     initialize(database, replacing) {
         if (replacing)
             this.dropDedupIndexes(database, this.#dedupIndex);
         else
             this.#index(database);
     }
-    replace(database) {
-        super.replace(database);
-        this.#index(database);
-    }
     // The target's key index, built only when it does not exist yet.
     #index(database) {
         this.dropDedupIndexes(database, this.#dedupIndex);
@@ -87,27 +83,55 @@ export class SQLiteDeduplicatingWriter extends SQLiteWriter {
         database.exec(`DROP INDEX IF EXISTS ${this.#provisionalIndex}`);
         this.#index(database);
     }
-    // The result of applying the staged operations one at a time: a staged
-    // DELETE removes its key, and only records after a key's last DELETE count.
-    // replace keeps the newest extraction, so a restated fact overwrites the
-    // loaded one; cursor_newer keeps the greatest cursor (the first on ties) and
-    // the guard that rejects out-of-order replay.
-    merge(database, stage, loadedAt, into) {
+    // The cursor of the guard that rejects out-of-order replay: cursor_newer's,
+    // when the copy has a cursor.
+    get #guard() {
+        return this.configuration.dedupPolicy === 'replace'
+            ? undefined
+            : this.cursor;
+    }
+    #same(left, right) {
+        return this.keys
+            .map(({ quotedName }) => `${left}.${quotedName} = ${right}.${quotedName}`)
+            .join(' AND ');
+    }
+    // The seq of each staged record that applying the staged operations one at
+    // a time keeps: only records after a key's last DELETE count. replace keeps
+    // the newest extraction, so a restated fact overwrites the loaded one;
+    // cursor_newer keeps the greatest cursor (the first on ties).
+    #accepted(stage) {
         const keys = this.keys.map((column) => column.quotedName);
-        const same = (left, right) => keys.map((key) => `${left}.${key} = ${right}.${key}`).join(' AND ');
-        database.exec(`DELETE FROM ${into} WHERE (${keys.join(', ')}) IN (SELECT ${keys.join(', ')} FROM ${stage} WHERE ${op} = 'D')`);
-        const { cursor } = this;
-        const guarded = this.configuration.dedupPolicy !== 'replace' && cursor !== undefined;
-        const order = guarded
-            ? `"staged".${cursor.quotedName} COLLATE BINARY DESC, "staged".${seq}`
-            : `"staged".${seq} DESC`;
+        const guard = this.#guard;
+        const order = guard === undefined
+            ? `"staged".${seq} DESC`
+            : `"staged".${guard.quotedName} COLLATE BINARY DESC, "staged".${seq}`;
         // Rank identifiers only: sorting whole mail bodies/files multiplies temporary disk usage.
+        return `WITH "deleted" AS (SELECT ${keys.join(', ')}, max(${seq}) AS "_elt_last" FROM ${stage} WHERE ${op} = 'D' GROUP BY ${keys.join(', ')}), "ranked" AS (SELECT "staged".${seq}, row_number() OVER (PARTITION BY ${keys.map((key) => `"staged".${key}`).join(', ')} ORDER BY ${order}) AS "_elt_rank" FROM ${stage} AS "staged" LEFT JOIN "deleted" ON ${this.#same('"deleted"', '"staged"')} WHERE "staged".${op} = 'R' AND ("deleted"."_elt_last" IS NULL OR "staged".${seq} > "deleted"."_elt_last")) SELECT ${seq} FROM "ranked" WHERE "_elt_rank" = 1`;
+    }
+    kept(stage) {
+        return `${seq} IN (${this.#accepted(stage)})`;
+    }
+    // A staged DELETE removes its key, and the records that #accepted keeps are
+    // upserted, with the guard, if any, rejecting out-of-order replay. A stored
+    // row with the values of the record that replaces it stays as it is, with
+    // its commit and its loaded_at; history keeps each stored row that a DELETE
+    // removes or a record with other values replaces.
+    merge(database, stage, { loadedAt, committedIn }, into, history) {
+        const keys = this.keys.map((column) => column.quotedName);
+        const deleted = `(${keys.join(', ')}) IN (SELECT ${keys.join(', ')} FROM ${stage} WHERE ${op} = 'D')`;
+        history?.keep(database, committedIn, `FROM ${into} AS "_elt_old" WHERE ${deleted}`);
+        database.exec(`DELETE FROM ${into} WHERE ${deleted}`);
+        const accepted = this.#accepted(stage);
+        const guard = this.#guard;
+        // Whether a staged record replaces the stored row: its values differ,
+        // and the guard, if any, lets it in.
+        const replaces = (record, row) => `${record}.${version} IS NOT ${row}.${version}${guard === undefined ? '' : ` AND ${record}.${guard.quotedName} COLLATE BINARY > ${row}.${guard.quotedName}`}`;
+        history?.keep(database, committedIn, `FROM ${stage} AS "_elt_new" JOIN ${into} AS "_elt_old" ON ${this.#same('"_elt_new"', '"_elt_old"')} WHERE "_elt_new".${seq} IN (${accepted}) AND ${replaces('"_elt_new"', '"_elt_old"')}`);
         const columns = this.table.columns.map((column) => column.quotedName);
         database
-            .prepare(`WITH "deleted" AS (SELECT ${keys.join(', ')}, max(${seq}) AS "_elt_last" FROM ${stage} WHERE ${op} = 'D' GROUP BY ${keys.join(', ')}), "ranked" AS (SELECT "staged".${seq}, row_number() OVER (PARTITION BY ${keys.map((key) => `"staged".${key}`).join(', ')} ORDER BY ${order}) AS "_elt_rank" FROM ${stage} AS "staged" LEFT JOIN "deleted" ON ${same('"deleted"', '"staged"')} WHERE "staged".${op} = 'R' AND ("deleted"."_elt_last" IS NULL OR "staged".${seq} > "deleted"."_elt_last")) ` +
-            `INSERT INTO ${into} AS "_elt_target" (${this.fields.join(', ')}) SELECT ${columns.join(', ')}, ? FROM ${stage} WHERE ${seq} IN (SELECT ${seq} FROM "ranked" WHERE "_elt_rank" = 1) ORDER BY ${seq} ` +
-            `ON CONFLICT (${keys.map((key) => `${key} COLLATE BINARY`).join(', ')}) DO UPDATE SET ${this.fields.map((field) => `${field} = excluded.${field}`).join(', ')}${guarded ? ` WHERE excluded.${cursor.quotedName} COLLATE BINARY > "_elt_target".${cursor.quotedName}` : ''}`)
-            .run(loadedAt);
+            .prepare(`INSERT INTO ${into} AS "_elt_target" (${this.written.join(', ')}) SELECT ${columns.join(', ')}, ?, ?, ${version} FROM ${stage} WHERE ${seq} IN (${accepted}) ORDER BY ${seq} ` +
+            `ON CONFLICT (${keys.map((key) => `${key} COLLATE BINARY`).join(', ')}) DO UPDATE SET ${this.written.map((field) => `${field} = excluded.${field}`).join(', ')} WHERE ${replaces('excluded', '"_elt_target"')}`)
+            .run(loadedAt, committedIn);
     }
     encode(record) {
         this.deduplication.key(record);

package/dist/sqlite-destination.d.ts

@@ -1,5 +1,6 @@
 import type { CopyConfiguration, Load, PipelineClock } from '@eltjs/elt';
 import { Destination } from '@eltjs/elt';
+import { type SQLiteReading } from './sqlite-as-of.ts';
 import type { SQLiteColumn } from './sqlite-column.ts';
 import { SQLiteColumns } from './sqlite-columns.ts';
 import { SQLiteTable } from './sqlite-table.ts';
@@ -7,13 +8,22 @@ import type { SQLiteWriter } from './sqlite-writer.ts';
 export declare class SQLiteDestination extends Destination<SQLiteTable> {
     readonly supportedDestinationSyncModes: readonly ["overwrite", "append", "append_dedup", "overwrite_dedup"];
     readonly path: string;
-    constructor({ path }: {
+    readonly retention: Temporal.Duration;
+    constructor({ path, retention, }: {
         path: string;
+        retention?: Temporal.Duration;
     });
     location(target: SQLiteTable): string;
-    load({ clock, }: {
+    load({ clock, connection, }: {
         clock: PipelineClock;
+        connection: string;
     }): Promise<Load<SQLiteTable>>;
+    positions(after?: string): readonly string[] | {
+        readonly expired: string;
+    };
+    reading(position?: string): SQLiteReading | {
+        readonly expired: string;
+    };
     table(name: string, configure?: (columns: SQLiteColumns) => readonly SQLiteColumn[]): SQLiteTable;
     createWriter(configuration: CopyConfiguration, target: SQLiteTable): SQLiteWriter;
 }

package/dist/sqlite-destination.js

@@ -1,9 +1,10 @@
 import { resolve } from 'node:path';
 import { Destination } from '@eltjs/elt';
-import { openDestination } from './open-destination.js';
 import { SQLiteAppendWriter } from './sqlite-append-writer.js';
+import { readAsOf, reader } from './sqlite-as-of.js';
 import { SQLiteColumns } from './sqlite-columns.js';
 import { SQLiteDeduplicatingWriter } from './sqlite-deduplicating-writer.js';
+import { SQLiteLoad, positionsAfter } from './sqlite-loads.js';
 import { SQLiteOverwriteWriter } from './sqlite-overwrite-writer.js';
 import { SQLiteTable } from './sqlite-table.js';
 export class SQLiteDestination extends Destination {
@@ -14,11 +15,17 @@ export class SQLiteDestination extends Destination {
         'overwrite_dedup',
     ]);
     path;
-    constructor({ path }) {
+    // How long the history keeps a position readable after its commit; Delta
+    // Lake keeps its log for the same 30 days by default.
+    retention;
+    constructor({ path, retention = Temporal.Duration.from({ days: 30 }), }) {
         super();
         if (!path)
             throw new TypeError('SQLite requires a database path');
+        if (retention.sign < 0)
+            throw new RangeError(`Retention ${retention} must not be negative`);
         this.path = path === ':memory:' ? path : resolve(path);
+        this.retention = retention;
         Object.freeze(this);
     }
     location(target) {
@@ -26,22 +33,34 @@ export class SQLiteDestination extends Destination {
     }
     // Each stream's commit is its own short transaction, so no transaction
     // stays open while the sources read.
-    async load({ clock, }) {
-        const database = openDestination(this.path);
-        const loadedAt = new Date(clock.Date.now()).toISOString();
+    async load({ clock, connection, }) {
+        const load = SQLiteLoad.open(this.path, { connector: connection }, { clock, retention: this.retention });
         return {
-            prepare: async (configuration, target, binding) => this.createWriter(configuration, target).prepare(database, binding, loadedAt),
-            [Symbol.asyncDispose]: async () => {
-                try {
-                    if (database.isTransaction)
-                        database.exec('ROLLBACK');
-                }
-                finally {
-                    database.close();
-                }
-            },
+            prepare: async (configuration, target, binding) => this.createWriter(configuration, target).prepare(load, binding),
+            [Symbol.asyncDispose]: async () => load[Symbol.dispose](),
         };
     }
+    // The positions of the file after a position, ascending, or all of them;
+    // each one, `${epoch}:${number}`, names a commit where no pass is half
+    // read. A position of another epoch, or past the newest commit, is
+    // expired, and so is one older than the retention.
+    positions(after) {
+        if (this.path === ':memory:')
+            throw new TypeError('Positions require a SQLite destination file');
+        using database = reader(this.path);
+        return positionsAfter(database, after);
+    }
+    // The file as of a position, or as of its newest position: each target,
+    // its reader view and each view over them show the rows that the commit of
+    // the position left, and the sync history refuses to be read. At the
+    // newest commit the reading reads the tables as they are. It opens the
+    // file read-only, in one read transaction. A position of another epoch,
+    // past the newest commit, or older than the retention, is expired.
+    reading(position) {
+        if (this.path === ':memory:')
+            throw new TypeError('Readings require a SQLite destination file');
+        return readAsOf(this.path, position);
+    }
     table(name, configure) {
         return new SQLiteTable(name, configure?.(new SQLiteColumns()));
     }
@@ -55,11 +74,11 @@ export class SQLiteDestination extends Destination {
         switch (configuration.destinationSyncMode) {
             case 'append_dedup':
             case 'overwrite_dedup':
-                return new SQLiteDeduplicatingWriter(configuration, this.path, table);
+                return new SQLiteDeduplicatingWriter(configuration, this, table);
             case 'append':
-                return new SQLiteAppendWriter(configuration, this.path, table);
+                return new SQLiteAppendWriter(configuration, this, table);
             case 'overwrite':
-                return new SQLiteOverwriteWriter(configuration, this.path, table);
+                return new SQLiteOverwriteWriter(configuration, this, table);
             default:
                 throw new TypeError(`Destination does not support ${configuration.destinationSyncMode}`);
         }

package/dist/sqlite-file-store.d.ts

@@ -7,10 +7,13 @@ export declare class SQLiteFileStore {
     static readonly chunkSize: number;
     readonly name: string;
     constructor(database: DatabaseSync, table: SQLiteTable, column: SQLiteColumn);
-    static tableName(table: SQLiteTable, column: Pick<SQLiteColumn, 'name'>): string;
+    static tableName(table: Pick<SQLiteTable, 'location'>, column: Pick<SQLiteColumn, 'name'>): string;
     get published(): boolean;
     stage(): void;
-    save(content: FileContent): Promise<number>;
+    save(content: FileContent): Promise<{
+        readonly id: number;
+        readonly sha256: Uint8Array;
+    }>;
     publish(): void;
     discard(): void;
     unstage(): void;

package/dist/sqlite-file-store.js

@@ -1,24 +1,28 @@
-import { hash } from 'node:crypto';
+import { createHash, hash } from 'node:crypto';
+import { Declaration } from '@zukhruf/sqlite-drift';
 import { identifiers } from './sqlite-identifiers.js';
 const quote = (name) => `"${name.replaceAll('"', '""')}"`;
 const tablePrefix = '_elt_files_';
 const chunks = '("file" INTEGER NOT NULL, "n" INTEGER NOT NULL, "bytes" BLOB NOT NULL, PRIMARY KEY ("file", "n")) STRICT';
+// The highest id that each store gave a file it published. Ids only go up: a
+// row of the history keeps the id of its file for a reading as of a position,
+// so no other file may take that id, even after the chunks of the file, or the
+// whole store, are gone.
+const ids = new Declaration([
+    'CREATE TABLE "_elt_file_ids" ("store" TEXT PRIMARY KEY, "last" INTEGER NOT NULL) STRICT',
+], { rows: 'replace' });
 // A file column's bytes, in ordered chunks: SQLite caps one BLOB at 1e9 bytes
 // and a whole-file BLOB would hold the file in memory. The row keeps the
-// file's id; triggers drop a row's chunks in the same transaction as the row.
-// A load stages chunks in a connection-private TEMP table and publishes them
-// inside a commit, so a load holds no transaction while it reads files.
+// file's id. A load stages chunks in a connection-private TEMP table and
+// publishes them inside a commit, so a load holds no transaction while it
+// reads files.
 export class SQLiteFileStore {
     static chunkSize = 4 * 1024 * 1024;
     name;
     #database;
-    #table;
-    #column;
     #staged;
     constructor(database, table, column) {
         this.#database = database;
-        this.#table = table;
-        this.#column = column;
         this.name = SQLiteFileStore.tableName(table, column);
         this.#staged = quote(`${tablePrefix}stage_${this.name.slice(tablePrefix.length)}`);
     }
@@ -34,6 +38,14 @@ export class SQLiteFileStore {
             .prepare(`SELECT 1 FROM sqlite_schema WHERE "type" = 'table' AND lower("name") = ?`)
             .get(location) !== undefined);
     }
+    #lastGivenId() {
+        if (!this.#exists('_elt_file_ids'))
+            return 0;
+        const row = this.#database
+            .prepare('SELECT "last" FROM "_elt_file_ids" WHERE "store" = ?')
+            .get(this.name);
+        return Number(row?.last ?? 0);
+    }
     get published() {
         return this.#exists(this.name);
     }
@@ -41,33 +53,37 @@ export class SQLiteFileStore {
         this.#database.exec(`DROP TABLE IF EXISTS temp.${this.#staged}`);
         this.#database.exec(`CREATE TEMP TABLE ${this.#staged} ${chunks}`);
     }
-    // An empty file still stores one empty chunk, so its id stays reserved. Ids
-    // continue after both the published and the staged chunks.
+    // Stages the file's chunks under an id above each id the store gave and each
+    // staged id. An empty file still stores one empty chunk, so its id stays
+    // reserved. Gives the id and the SHA-256 of the bytes, which names the
+    // content.
     async save(content) {
-        const highest = (table) => Number(this.#database
-            .prepare(`SELECT coalesce(max("file"), 0) AS "file" FROM ${table}`)
+        const given = this.#lastGivenId();
+        const staged = Number(this.#database
+            .prepare(`SELECT coalesce(max("file"), 0) AS "file" FROM temp.${this.#staged}`)
             .get()?.file);
-        const file = Math.max(this.published ? highest(quote(this.name)) : 0, highest(`temp.${this.#staged}`)) + 1;
+        const id = Math.max(given, staged) + 1;
         const insert = this.#database.prepare(`INSERT INTO temp.${this.#staged} ("file", "n", "bytes") VALUES (?, ?, ?)`);
+        const sha256 = createHash('sha256');
         let n = 0;
-        for await (const chunk of content.chunks(SQLiteFileStore.chunkSize))
-            insert.run(file, n++, chunk);
+        for await (const chunk of content.chunks(SQLiteFileStore.chunkSize)) {
+            sha256.update(chunk);
+            insert.run(id, n++, chunk);
+        }
         if (n === 0)
-            insert.run(file, 0, new Uint8Array());
-        return file;
+            insert.run(id, 0, new Uint8Array());
+        return { id, sha256: sha256.digest() };
     }
-    // Moves the staged chunks into the store, inside a commit, and attaches the
-    // triggers to the target once it exists. A reload swaps in a table without
-    // them, which its next commit attaches again.
+    // Moves the staged chunks into the store, and records the highest id they
+    // take, inside a commit.
     publish() {
         const table = quote(this.name);
-        const column = this.#column.quotedName;
         this.#database.exec(`CREATE TABLE IF NOT EXISTS ${table} ${chunks}`);
-        if (this.#exists(this.#table.location)) {
-            this.#database.exec(`CREATE TRIGGER IF NOT EXISTS ${quote(`${this.name}_delete`)} AFTER DELETE ON ${this.#table.quotedName} BEGIN DELETE FROM ${table} WHERE "file" = old.${column}; END`);
-            this.#database.exec(`CREATE TRIGGER IF NOT EXISTS ${quote(`${this.name}_update`)} AFTER UPDATE OF ${column} ON ${this.#table.quotedName} WHEN old.${column} IS NOT new.${column} BEGIN DELETE FROM ${table} WHERE "file" = old.${column}; END`);
-        }
         this.#database.exec(`INSERT INTO ${table} SELECT * FROM temp.${this.#staged}`);
+        ids.apply(this.#database);
+        this.#database
+            .prepare(`INSERT INTO "_elt_file_ids" ("store", "last") SELECT ?, "file" FROM (SELECT max("file") AS "file" FROM temp.${this.#staged}) WHERE "file" IS NOT NULL ON CONFLICT ("store") DO UPDATE SET "last" = excluded."last"`)
+            .run(this.name);
         this.discard();
     }
     discard() {

package/dist/sqlite-history.d.ts

@@ -0,0 +1,14 @@
+import type { DatabaseSync, SQLInputValue } from 'node:sqlite';
+import type { SQLiteTable } from './sqlite-table.ts';
+export declare const histories = "_elt_histories";
+export declare class SQLiteHistory {
+    #private;
+    readonly name: string;
+    constructor(table: Pick<SQLiteTable, 'name' | 'location'>);
+    create(database: DatabaseSync, written: readonly string[]): void;
+    keep(database: DatabaseSync, replacedIn: number, rows: string, params?: readonly SQLInputValue[]): void;
+}
+export declare function expireHistory(database: DatabaseSync, through: number): void;
+export declare function rowHash(columns: readonly {
+    readonly name: string;
+}[], values: readonly SQLInputValue[]): string;

package/dist/sqlite-history.js

@@ -0,0 +1,130 @@
+import { createHash, hash } from 'node:crypto';
+import { Declaration } from '@zukhruf/sqlite-drift';
+import { SQLiteFileStore } from './sqlite-file-store.js';
+import { identifiers } from './sqlite-identifiers.js';
+const quote = (name) => `"${name.replaceAll('"', '""')}"`;
+export const histories = '_elt_histories';
+// Which table keeps the replaced rows of which target, by the target's
+// location, so a reader finds the history of each target.
+const registry = new Declaration([
+    `CREATE TABLE ${quote(histories)} ("target" TEXT PRIMARY KEY, "history" TEXT NOT NULL) STRICT`,
+], { rows: 'replace' });
+// The rows of a target that commits replaced or removed, each with the
+// number of the commit that did, as a SQL Server temporal table keeps them
+// beside its current table. Its columns declare no type, so each value keeps
+// the type it had in the target whatever the target declares later. It takes
+// each column of the target that it does not have, and never drops one.
+export class SQLiteHistory {
+    name;
+    #table;
+    constructor(table) {
+        this.#table = table;
+        this.name = `_elt_history_${hash('sha256', table.location)}`;
+    }
+    // Made with the target, with written, the quoted columns that each row of
+    // the target holds, and recorded as its history. SQLite cannot add a
+    // column named rowid, _rowid_ or oid to a table, so the history has the
+    // target's columns from the start.
+    create(database, written) {
+        database.exec(`CREATE TABLE IF NOT EXISTS ${quote(this.name)} ("replaced_in" INTEGER NOT NULL, ${written.join(', ')})`);
+        database.exec(`CREATE INDEX IF NOT EXISTS ${quote(`${this.name}_replaced_in`)} ON ${quote(this.name)} ("replaced_in")`);
+        registry.apply(database);
+        database
+            .prepare(`INSERT INTO ${quote(histories)} ("target", "history") VALUES (?, ?) ON CONFLICT ("target") DO UPDATE SET "history" = excluded."history"`)
+            .run(this.#table.location, this.name);
+    }
+    // Copies the rows of the target that rows selects into the history, as
+    // replaced in commit replacedIn. rows is the FROM clause of a SELECT in
+    // which "_elt_old" names the target.
+    keep(database, replacedIn, rows, params = []) {
+        const columns = names(database, this.#table.name);
+        const kept = new Set(names(database, this.name).map((name) => identifiers.key(name)));
+        for (const column of columns)
+            if (!kept.has(identifiers.key(column)))
+                database.exec(`ALTER TABLE ${quote(this.name)} ADD COLUMN ${quote(column)}`);
+        database
+            .prepare(`INSERT INTO ${quote(this.name)} (${columns.map(quote).join(', ')}, "replaced_in") SELECT ${columns.map((column) => `"_elt_old".${quote(column)}`).join(', ')}, ? ${rows}`)
+            .run(replacedIn, ...params);
+    }
+}
+// Deletes, in each history, the rows that only a reading before commit
+// through reads: each row that a commit up to it replaced. The stored files
+// that only those rows refer to go in the same transaction, so the chunks of
+// the file stay the files that a row refers to. The rows of a reload's hidden
+// target refer to no such file: each file id is given once, and a hidden row
+// reaches a history only after its reload ends and drops the hidden target.
+export function expireHistory(database, through) {
+    const exists = (name) => database
+        .prepare(`SELECT 1 FROM sqlite_schema WHERE "type" = 'table' AND lower("name") = lower(?)`)
+        .get(name) !== undefined;
+    if (!exists(histories))
+        return;
+    const pairs = database
+        .prepare(`SELECT "target", "history" FROM ${quote(histories)}`)
+        .all()
+        .map((row) => ({
+        target: String(row.target),
+        history: String(row.history),
+    }));
+    for (const { target, history } of pairs) {
+        if (!exists(history))
+            continue;
+        const current = new Set(exists(target)
+            ? names(database, target).map((name) => identifiers.key(name))
+            : []);
+        for (const column of names(database, history)) {
+            const chunks = SQLiteFileStore.tableName({ location: target }, { name: column });
+            if (!exists(chunks))
+                continue;
+            const value = quote(column);
+            const kept = [
+                ...(current.has(identifiers.key(column))
+                    ? [`SELECT ${value} FROM ${quote(target)} WHERE ${value} IS NOT NULL`]
+                    : []),
+                `SELECT ${value} FROM ${quote(history)} WHERE "replaced_in" > ? AND ${value} IS NOT NULL`,
+            ];
+            database
+                .prepare(`DELETE FROM ${quote(chunks)} WHERE "file" IN (SELECT ${value} FROM ${quote(history)} WHERE "replaced_in" <= ?) AND "file" NOT IN (${kept.join(' UNION ')})`)
+                .run(through, through);
+        }
+        database
+            .prepare(`DELETE FROM ${quote(history)} WHERE "replaced_in" <= ?`)
+            .run(through);
+    }
+}
+function names(database, table) {
+    return database
+        .prepare(`PRAGMA main.table_info(${quote(table)})`)
+        .all()
+        .map(({ name }) => String(name));
+}
+// The SHA-256 of the values of a staged row, which names its version: a
+// commit that finds the stored row with the same hash leaves it as it is.
+// Each value that is not null counts with its column's name and its type, so
+// a column the stream adds, null in each stored row, makes no new version. A
+// stored file counts as the SHA-256 of its bytes, not as its id.
+export function rowHash(columns, values) {
+    const digest = createHash('sha256');
+    const add = (tag, data) => {
+        const bytes = typeof data === 'string'
+            ? Buffer.from(data)
+            : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+        digest.update(`${tag}${bytes.byteLength}:`);
+        digest.update(bytes);
+    };
+    columns.forEach((column, index) => {
+        const value = values[index];
+        if (value === null || value === undefined)
+            return;
+        add('c', column.name);
+        if (typeof value === 'string')
+            add('t', value);
+        else if (typeof value === 'bigint')
+            add('n', String(value));
+        else if (typeof value === 'number')
+            add('n', Number.isInteger(value) ? String(BigInt(value)) : String(value));
+        else
+            add('b', value);
+    });
+    return digest.digest('hex');
+}

package/dist/sqlite-identifiers.js

@@ -3,7 +3,15 @@ import { Identifiers } from '@eltjs/elt';
 // folds A-Z only, so `É` and `é` stay two names.
 class SQLiteIdentifiers extends Identifiers {
     maxBytes = Infinity;
-    ownColumns = ['loaded_at'];
+    // _rowid_ reads the id SQLite gives each row, which a difference needs, as
+    // long as no column takes the name.
+    ownColumns = [
+        'loaded_at',
+        'committed_in',
+        '_elt_row_hash',
+        'replaced_in',
+        '_rowid_',
+    ];
     key(name) {
         return name.replaceAll(/[A-Z]/g, (letter) => letter.toLowerCase());
     }

package/dist/sqlite-loads.d.ts

@@ -0,0 +1,38 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { PipelineClock } from '@eltjs/elt';
+export type LoadOwner = {
+    readonly connector: string;
+} | {
+    readonly dataset: string;
+};
+export declare class SQLiteLoad implements Disposable {
+    #private;
+    readonly database: DatabaseSync;
+    readonly loadedAt: string;
+    private constructor();
+    static open(path: string, owner: LoadOwner, retaining: Retaining): SQLiteLoad;
+    static once(database: DatabaseSync, connection: string, retaining: Retaining): number;
+    commit(): number;
+    [Symbol.dispose](): void;
+}
+export type Retaining = {
+    readonly clock: PipelineClock;
+    readonly retention: Temporal.Duration;
+};
+export type Numbering = {
+    readonly epoch: string;
+    readonly newest: number;
+    readonly minValid: number;
+    readonly ends: readonly number[];
+};
+export type Position = {
+    readonly epoch: string;
+    readonly number: number;
+};
+export declare function positionsAfter(database: DatabaseSync | null, after: string | undefined): readonly string[] | {
+    readonly expired: string;
+};
+export declare function expiry(numbered: Numbering | null, { epoch, number }: Position): string | undefined;
+export declare function readNumbering(database: DatabaseSync, schema?: string): Numbering | null;
+export declare function committedAt(database: DatabaseSync, number: number): string | undefined;
+export declare function parsePosition(position: string): Position;