@eltjs/elt-sqlite 0.1.3 → 0.3.0

package/dist/sqlite-as-of.js

@@ -0,0 +1,183 @@
+import { constants } from 'node:sqlite';
+import { UnavailableError, open } from '@zukhruf/sqlite-open';
+import { busyTimeout } from './open-destination.js';
+import { histories } from './sqlite-history.js';
+import { identifiers } from './sqlite-identifiers.js';
+import { committedAt, expiry, parsePosition, readNumbering, } from './sqlite-loads.js';
+import { attempts, checks, coverage } from './sqlite-sync-history-schema.js';
+const quote = (name) => `"${name.replaceAll('"', '""')}"`;
+// A reader of the file, or null while no file is at the path.
+export function reader(path) {
+    try {
+        return open(path, { readOnly: true, timeout: busyTimeout });
+    }
+    catch (error) {
+        if (error instanceof UnavailableError)
+            return null;
+        throw error;
+    }
+}
+// Reads the file as of the position, or as of its newest position. One read
+// transaction holds the snapshot that the numbering and every later read
+// see, so a commit after the reading begins shows in none of them.
+export function readAsOf(path, position) {
+    const asked = position === undefined ? undefined : parsePosition(position);
+    const database = reader(path);
+    if (database === null)
+        return { expired: `No load has committed to ${path}` };
+    using stack = new DisposableStack();
+    stack.use(database);
+    database.exec('BEGIN');
+    const numbered = readNumbering(database);
+    if (asked !== undefined) {
+        const expired = expiry(numbered, asked);
+        if (expired !== undefined)
+            return { expired };
+        refuseInsidePass(numbered, asked, path);
+    }
+    const number = asked?.number ?? numbered?.ends.at(-1);
+    const at = number === undefined ? undefined : committedAt(database, number);
+    if (numbered === null || number === undefined || at === undefined)
+        return {
+            expired: position === undefined
+                ? `${path} has no position yet`
+                : `Position ${position} names no commit of ${path}`,
+        };
+    if (number < numbered.newest)
+        shadow(database, number);
+    stack.move();
+    return {
+        sql: database,
+        position: `${numbered.epoch}:${number}`,
+        at: Temporal.Instant.from(at),
+        [Symbol.dispose]: () => database.close(),
+    };
+}
+// No later load can straddle a position, so one that a reader was given
+// stays one; a commit that is not a position stays inside its pass.
+function refuseInsidePass(numbered, { epoch, number }, file) {
+    if (!numbered?.ends.includes(number))
+        throw new TypeError(`${epoch}:${number} is not a position of ${file}: a reading of a commit inside a pass would show a part of that pass. Read a position that positions() lists.`);
+}
+// Shadows, for this connection, each name a reader queries with a TEMP view
+// of what it showed as of commit number. A TEMP view takes the place of a
+// table or view of main for an unqualified name, and SQLite resolves the
+// names of a view when it is read, so the views can be made in any order.
+function shadow(database, number) {
+    const targets = versioned(database, 'main');
+    for (const version of targets)
+        database.exec(`CREATE TEMP VIEW ${quote(version.target)} AS ${selectAsOf(database, version, number, 'main')}`);
+    shadowSyncHistory(database);
+    const views = storedViews(database);
+    for (const { sql } of views)
+        database.exec(sql.replace(/^CREATE VIEW /, 'CREATE TEMP VIEW '));
+    refuseQualified(database, views.map(({ name }) => name), new Set(targets.map(({ target }) => identifiers.key(target))));
+}
+/**
+ * The SELECT that reads one target of the file as of a position, for a reader
+ * that copies the target, such as into a database of its own: the current
+ * rows that the commit of the position had made, and the history rows that it
+ * had made and a later commit replaced. `schema` names the file where
+ * `database` attached it, main by default. Run the SELECT in the read
+ * transaction that this checks the position in, since a later commit can
+ * expire the position. A position of another epoch, past the newest commit,
+ * or older than the retention, is expired.
+ */
+export function asOfSelect(database, target, position, { schema = 'main' } = {}) {
+    const asked = parsePosition(position);
+    const numbered = readNumbering(database, schema);
+    const expired = expiry(numbered, asked);
+    if (expired !== undefined)
+        return { expired };
+    refuseInsidePass(numbered, asked, schema === 'main' ? 'the file' : `the file attached as ${schema}`);
+    const version = versioned(database, schema).find((version) => identifiers.key(version.target) === identifiers.key(target));
+    if (version === undefined)
+        throw new TypeError(`${quote(target)} is not a target of ${schema}`);
+    return selectAsOf(database, version, asked.number, schema);
+}
+// A target as of commit number: its current rows that the commit had made,
+// and the history rows that the commit had made and a later commit replaced.
+// A column that the history lacks reads as NULL there.
+function selectAsOf(database, { target, history }, number, schema) {
+    const columns = columnsOf(database, quote(target), schema);
+    const kept = new Set(columnsOf(database, quote(history), schema).map((column) => identifiers.key(column)));
+    const current = columns.map(quote).join(', ');
+    const replaced = columns
+        .map((column) => kept.has(identifiers.key(column))
+        ? quote(column)
+        : `NULL AS ${quote(column)}`)
+        .join(', ');
+    const file = quote(schema);
+    return `SELECT ${current} FROM ${file}.${quote(target)} WHERE "committed_in" <= ${number} UNION ALL SELECT ${replaced} FROM ${file}.${quote(history)} WHERE "committed_in" <= ${number} AND "replaced_in" > ${number}`;
+}
+// The sync history keeps no versions, so each of its tables reads as a view
+// that fails, and so does each view over them.
+function shadowSyncHistory(database) {
+    database.function('_elt_unversioned', { deterministic: false }, () => {
+        throw new Error('The sync history cannot be read as of a position: it keeps no row versions');
+    });
+    for (const table of [attempts, coverage, checks]) {
+        const columns = columnsOf(database, table);
+        if (columns.length > 0)
+            database.exec(`CREATE TEMP VIEW ${table} AS SELECT ${columns.map((column) => `NULL AS ${quote(column)}`).join(', ')} WHERE "_elt_unversioned"()`);
+    }
+}
+function storedViews(database) {
+    return database
+        .prepare(String.raw `SELECT "name", "sql" FROM main.sqlite_schema WHERE "type" = 'view' AND "name" NOT LIKE 'sqlite\_%' ESCAPE '\'`)
+        .all()
+        .map(({ name, sql }) => ({ name: String(name), sql: String(sql) }));
+}
+// Refuses a stored view that reads a versioned target as main."target": the
+// name skips the TEMP view of the target, so the view would show the target
+// as it is now. SQLite tells which view reads each table while it prepares a
+// statement, so each view is prepared once with an authorizer that watches.
+function refuseQualified(database, views, targets) {
+    let found;
+    database.setAuthorizer((action, table, _column, schema, view) => {
+        if (action === constants.SQLITE_READ &&
+            schema === 'main' &&
+            table !== null &&
+            targets.has(identifiers.key(table)) &&
+            identifiers.key(view ?? '') !== identifiers.key(table))
+            found ??= { view: view ?? '', target: table };
+        return constants.SQLITE_OK;
+    });
+    try {
+        for (const view of views) {
+            try {
+                database.prepare(`SELECT * FROM temp.${quote(view)}`);
+            }
+            catch {
+                // A view that fails to prepare fails when it is read, too.
+            }
+            if (found !== undefined)
+                throw new TypeError(`View ${quote(found.view)} reads ${quote(found.target)} as main.${quote(found.target)}, which shows it as it is now, not as of a position; name the table without main.`);
+        }
+    }
+    finally {
+        database.setAuthorizer(null);
+    }
+}
+// Each target with its history, as the file attached as schema records them.
+function versioned(database, schema) {
+    const file = quote(schema);
+    if (database
+        .prepare(`SELECT 1 FROM ${file}.sqlite_schema WHERE "type" = 'table' AND "name" = ?`)
+        .get(histories) === undefined)
+        return [];
+    return database
+        .prepare(`SELECT t."name" AS "target", h."history" FROM ${file}.${quote(histories)} AS h JOIN ${file}.sqlite_schema AS t ON t."type" = 'table' AND lower(t."name") = h."target" JOIN ${file}.sqlite_schema AS k ON k."type" = 'table' AND k."name" = h."history"`)
+        .all()
+        .map(({ target, history }) => ({
+        target: String(target),
+        history: String(history),
+    }));
+}
+// The columns of a table of schema, named as SQL quotes it.
+function columnsOf(database, table, schema = 'main') {
+    return database
+        .prepare(`PRAGMA ${quote(schema)}.table_info(${table})`)
+        .all()
+        .map(({ name }) => String(name));
+}

package/dist/sqlite-deduplicating-writer.d.ts

@@ -1,21 +1,23 @@
 import type { DatabaseSync, SQLInputValue } from 'node:sqlite';
 import type { CopyConfiguration, Deduplication, KeyValue } from '@eltjs/elt';
 import type { SQLiteColumn } from './sqlite-column.ts';
+import type { SQLiteDestination } from './sqlite-destination.ts';
+import type { SQLiteHistory } from './sqlite-history.ts';
 import type { SQLiteTable } from './sqlite-table.ts';
-import { SQLiteWriter } from './sqlite-writer.ts';
+import { SQLiteWriter, type Stamp } from './sqlite-writer.ts';
 export declare class SQLiteDeduplicatingWriter extends SQLiteWriter {
     #private;
     readonly deduplication: Deduplication;
     readonly keys: readonly SQLiteColumn[];
     readonly cursor?: SQLiteColumn;
-    constructor(configuration: CopyConfiguration, path: string, table: SQLiteTable);
+    constructor(configuration: CopyConfiguration, destination: SQLiteDestination, table: SQLiteTable);
     protected get replaces(): boolean;
     protected inspect(database: DatabaseSync): void;
     protected initialize(database: DatabaseSync, replacing: boolean): void;
-    protected replace(database: DatabaseSync): void;
     protected build(database: DatabaseSync, into: string): void;
     protected adopt(database: DatabaseSync): void;
-    protected merge(database: DatabaseSync, stage: string, loadedAt: string, into: string): void;
+    protected kept(stage: string): string;
+    protected merge(database: DatabaseSync, stage: string, { loadedAt, committedIn }: Stamp, into: string, history: SQLiteHistory | null): void;
     protected encode(record: unknown): SQLInputValue[];
     protected deletionKeys(key: Readonly<Record<string, KeyValue>>): readonly [readonly SQLiteColumn[], SQLInputValue[]];
 }

package/dist/sqlite-deduplicating-writer.js

@@ -1,12 +1,12 @@
 import { hash } from 'node:crypto';
 import { SQLiteColumns } from './sqlite-columns.js';
-import { SQLiteWriter, op, seq } from './sqlite-writer.js';
+import { SQLiteWriter, op, seq, version } from './sqlite-writer.js';
 export class SQLiteDeduplicatingWriter extends SQLiteWriter {
     deduplication;
     keys;
     cursor;
-    constructor(configuration, path, table) {
-        super(configuration, path, table);
+    constructor(configuration, destination, table) {
+        super(configuration, destination, table);
         this.deduplication = configuration.deduplication();
         const inferred = SQLiteColumns.fromSchema(configuration.stream.jsonSchema);
         const column = (field) => {
@@ -55,18 +55,14 @@ export class SQLiteDeduplicatingWriter extends SQLiteWriter {
                 .get())
             throw new TypeError('Existing rows repeat a deduplication key; replace them with overwrite_dedup before deduplicating incrementally');
     }
-    // A replacing load keeps none of the stored rows: its index is built once
-    // the commit has emptied the table.
+    // A replacing commit builds the index once the rows that leave are gone:
+    // under a narrower key, they can repeat a key.
     initialize(database, replacing) {
         if (replacing)
             this.dropDedupIndexes(database, this.#dedupIndex);
         else
             this.#index(database);
     }
-    replace(database) {
-        super.replace(database);
-        this.#index(database);
-    }
     // The target's key index, built only when it does not exist yet.
     #index(database) {
         this.dropDedupIndexes(database, this.#dedupIndex);
@@ -87,27 +83,55 @@ export class SQLiteDeduplicatingWriter extends SQLiteWriter {
         database.exec(`DROP INDEX IF EXISTS ${this.#provisionalIndex}`);
         this.#index(database);
     }
-    // The result of applying the staged operations one at a time: a staged
-    // DELETE removes its key, and only records after a key's last DELETE count.
-    // replace keeps the newest extraction, so a restated fact overwrites the
-    // loaded one; cursor_newer keeps the greatest cursor (the first on ties) and
-    // the guard that rejects out-of-order replay.
-    merge(database, stage, loadedAt, into) {
+    // The cursor of the guard that rejects out-of-order replay: cursor_newer's,
+    // when the copy has a cursor.
+    get #guard() {
+        return this.configuration.dedupPolicy === 'replace'
+            ? undefined
+            : this.cursor;
+    }
+    #same(left, right) {
+        return this.keys
+            .map(({ quotedName }) => `${left}.${quotedName} = ${right}.${quotedName}`)
+            .join(' AND ');
+    }
+    // The seq of each staged record that applying the staged operations one at
+    // a time keeps: only records after a key's last DELETE count. replace keeps
+    // the newest extraction, so a restated fact overwrites the loaded one;
+    // cursor_newer keeps the greatest cursor (the first on ties).
+    #accepted(stage) {
         const keys = this.keys.map((column) => column.quotedName);
-        const same = (left, right) => keys.map((key) => `${left}.${key} = ${right}.${key}`).join(' AND ');
-        database.exec(`DELETE FROM ${into} WHERE (${keys.join(', ')}) IN (SELECT ${keys.join(', ')} FROM ${stage} WHERE ${op} = 'D')`);
-        const { cursor } = this;
-        const guarded = this.configuration.dedupPolicy !== 'replace' && cursor !== undefined;
-        const order = guarded
-            ? `"staged".${cursor.quotedName} COLLATE BINARY DESC, "staged".${seq}`
-            : `"staged".${seq} DESC`;
+        const guard = this.#guard;
+        const order = guard === undefined
+            ? `"staged".${seq} DESC`
+            : `"staged".${guard.quotedName} COLLATE BINARY DESC, "staged".${seq}`;
         // Rank identifiers only: sorting whole mail bodies/files multiplies temporary disk usage.
+        return `WITH "deleted" AS (SELECT ${keys.join(', ')}, max(${seq}) AS "_elt_last" FROM ${stage} WHERE ${op} = 'D' GROUP BY ${keys.join(', ')}), "ranked" AS (SELECT "staged".${seq}, row_number() OVER (PARTITION BY ${keys.map((key) => `"staged".${key}`).join(', ')} ORDER BY ${order}) AS "_elt_rank" FROM ${stage} AS "staged" LEFT JOIN "deleted" ON ${this.#same('"deleted"', '"staged"')} WHERE "staged".${op} = 'R' AND ("deleted"."_elt_last" IS NULL OR "staged".${seq} > "deleted"."_elt_last")) SELECT ${seq} FROM "ranked" WHERE "_elt_rank" = 1`;
+    }
+    kept(stage) {
+        return `${seq} IN (${this.#accepted(stage)})`;
+    }
+    // A staged DELETE removes its key, and the records that #accepted keeps are
+    // upserted, with the guard, if any, rejecting out-of-order replay. A stored
+    // row with the values of the record that replaces it stays as it is, with
+    // its commit and its loaded_at; history keeps each stored row that a DELETE
+    // removes or a record with other values replaces.
+    merge(database, stage, { loadedAt, committedIn }, into, history) {
+        const keys = this.keys.map((column) => column.quotedName);
+        const deleted = `(${keys.join(', ')}) IN (SELECT ${keys.join(', ')} FROM ${stage} WHERE ${op} = 'D')`;
+        history?.keep(database, committedIn, `FROM ${into} AS "_elt_old" WHERE ${deleted}`);
+        database.exec(`DELETE FROM ${into} WHERE ${deleted}`);
+        const accepted = this.#accepted(stage);
+        const guard = this.#guard;
+        // Whether a staged record replaces the stored row: its values differ,
+        // and the guard, if any, lets it in.
+        const replaces = (record, row) => `${record}.${version} IS NOT ${row}.${version}${guard === undefined ? '' : ` AND ${record}.${guard.quotedName} COLLATE BINARY > ${row}.${guard.quotedName}`}`;
+        history?.keep(database, committedIn, `FROM ${stage} AS "_elt_new" JOIN ${into} AS "_elt_old" ON ${this.#same('"_elt_new"', '"_elt_old"')} WHERE "_elt_new".${seq} IN (${accepted}) AND ${replaces('"_elt_new"', '"_elt_old"')}`);
         const columns = this.table.columns.map((column) => column.quotedName);
         database
-            .prepare(`WITH "deleted" AS (SELECT ${keys.join(', ')}, max(${seq}) AS "_elt_last" FROM ${stage} WHERE ${op} = 'D' GROUP BY ${keys.join(', ')}), "ranked" AS (SELECT "staged".${seq}, row_number() OVER (PARTITION BY ${keys.map((key) => `"staged".${key}`).join(', ')} ORDER BY ${order}) AS "_elt_rank" FROM ${stage} AS "staged" LEFT JOIN "deleted" ON ${same('"deleted"', '"staged"')} WHERE "staged".${op} = 'R' AND ("deleted"."_elt_last" IS NULL OR "staged".${seq} > "deleted"."_elt_last")) ` +
-            `INSERT INTO ${into} AS "_elt_target" (${this.fields.join(', ')}) SELECT ${columns.join(', ')}, ? FROM ${stage} WHERE ${seq} IN (SELECT ${seq} FROM "ranked" WHERE "_elt_rank" = 1) ORDER BY ${seq} ` +
-            `ON CONFLICT (${keys.map((key) => `${key} COLLATE BINARY`).join(', ')}) DO UPDATE SET ${this.fields.map((field) => `${field} = excluded.${field}`).join(', ')}${guarded ? ` WHERE excluded.${cursor.quotedName} COLLATE BINARY > "_elt_target".${cursor.quotedName}` : ''}`)
-            .run(loadedAt);
+            .prepare(`INSERT INTO ${into} AS "_elt_target" (${this.written.join(', ')}) SELECT ${columns.join(', ')}, ?, ?, ${version} FROM ${stage} WHERE ${seq} IN (${accepted}) ORDER BY ${seq} ` +
+            `ON CONFLICT (${keys.map((key) => `${key} COLLATE BINARY`).join(', ')}) DO UPDATE SET ${this.written.map((field) => `${field} = excluded.${field}`).join(', ')} WHERE ${replaces('excluded', '"_elt_target"')}`)
+            .run(loadedAt, committedIn);
     }
     encode(record) {
         this.deduplication.key(record);

package/dist/sqlite-destination.d.ts

@@ -1,5 +1,6 @@
 import type { CopyConfiguration, Load, PipelineClock } from '@eltjs/elt';
 import { Destination } from '@eltjs/elt';
+import { type SQLiteReading } from './sqlite-as-of.ts';
 import type { SQLiteColumn } from './sqlite-column.ts';
 import { SQLiteColumns } from './sqlite-columns.ts';
 import { SQLiteTable } from './sqlite-table.ts';
@@ -7,13 +8,22 @@ import type { SQLiteWriter } from './sqlite-writer.ts';
 export declare class SQLiteDestination extends Destination<SQLiteTable> {
     readonly supportedDestinationSyncModes: readonly ["overwrite", "append", "append_dedup", "overwrite_dedup"];
     readonly path: string;
-    constructor({ path }: {
+    readonly retention: Temporal.Duration;
+    constructor({ path, retention, }: {
         path: string;
+        retention?: Temporal.Duration;
     });
     location(target: SQLiteTable): string;
-    load({ clock, }: {
+    load({ clock, connection, }: {
         clock: PipelineClock;
+        connection: string;
     }): Promise<Load<SQLiteTable>>;
+    positions(after?: string): readonly string[] | {
+        readonly expired: string;
+    };
+    reading(position?: string): SQLiteReading | {
+        readonly expired: string;
+    };
     table(name: string, configure?: (columns: SQLiteColumns) => readonly SQLiteColumn[]): SQLiteTable;
     createWriter(configuration: CopyConfiguration, target: SQLiteTable): SQLiteWriter;
 }

package/dist/sqlite-destination.js

@@ -1,9 +1,10 @@
 import { resolve } from 'node:path';
 import { Destination } from '@eltjs/elt';
-import { openDestination } from './open-destination.js';
 import { SQLiteAppendWriter } from './sqlite-append-writer.js';
+import { readAsOf, reader } from './sqlite-as-of.js';
 import { SQLiteColumns } from './sqlite-columns.js';
 import { SQLiteDeduplicatingWriter } from './sqlite-deduplicating-writer.js';
+import { SQLiteLoad, positionsAfter } from './sqlite-loads.js';
 import { SQLiteOverwriteWriter } from './sqlite-overwrite-writer.js';
 import { SQLiteTable } from './sqlite-table.js';
 export class SQLiteDestination extends Destination {
@@ -14,11 +15,17 @@ export class SQLiteDestination extends Destination {
         'overwrite_dedup',
     ]);
     path;
-    constructor({ path }) {
+    // How long the history keeps a position readable after its commit; Delta
+    // Lake keeps its log for the same 30 days by default.
+    retention;
+    constructor({ path, retention = Temporal.Duration.from({ days: 30 }), }) {
         super();
         if (!path)
             throw new TypeError('SQLite requires a database path');
+        if (retention.sign < 0)
+            throw new RangeError(`Retention ${retention} must not be negative`);
         this.path = path === ':memory:' ? path : resolve(path);
+        this.retention = retention;
         Object.freeze(this);
     }
     location(target) {
@@ -26,22 +33,34 @@ export class SQLiteDestination extends Destination {
     }
     // Each stream's commit is its own short transaction, so no transaction
     // stays open while the sources read.
-    async load({ clock, }) {
-        const database = openDestination(this.path);
-        const loadedAt = new Date(clock.Date.now()).toISOString();
+    async load({ clock, connection, }) {
+        const load = SQLiteLoad.open(this.path, { connector: connection }, { clock, retention: this.retention });
         return {
-            prepare: async (configuration, target, binding) => this.createWriter(configuration, target).prepare(database, binding, loadedAt),
-            [Symbol.asyncDispose]: async () => {
-                try {
-                    if (database.isTransaction)
-                        database.exec('ROLLBACK');
-                }
-                finally {
-                    database.close();
-                }
-            },
+            prepare: async (configuration, target, binding) => this.createWriter(configuration, target).prepare(load, binding),
+            [Symbol.asyncDispose]: async () => load[Symbol.dispose](),
         };
     }
+    // The positions of the file after a position, ascending, or all of them;
+    // each one, `${epoch}:${number}`, names a commit where no pass is half
+    // read. A position of another epoch, or past the newest commit, is
+    // expired, and so is one older than the retention.
+    positions(after) {
+        if (this.path === ':memory:')
+            throw new TypeError('Positions require a SQLite destination file');
+        using database = reader(this.path);
+        return positionsAfter(database, after);
+    }
+    // The file as of a position, or as of its newest position: each target,
+    // its reader view and each view over them show the rows that the commit of
+    // the position left, and the sync history refuses to be read. At the
+    // newest commit the reading reads the tables as they are. It opens the
+    // file read-only, in one read transaction. A position of another epoch,
+    // past the newest commit, or older than the retention, is expired.
+    reading(position) {
+        if (this.path === ':memory:')
+            throw new TypeError('Readings require a SQLite destination file');
+        return readAsOf(this.path, position);
+    }
     table(name, configure) {
         return new SQLiteTable(name, configure?.(new SQLiteColumns()));
     }
@@ -55,11 +74,11 @@ export class SQLiteDestination extends Destination {
         switch (configuration.destinationSyncMode) {
             case 'append_dedup':
             case 'overwrite_dedup':
-                return new SQLiteDeduplicatingWriter(configuration, this.path, table);
+                return new SQLiteDeduplicatingWriter(configuration, this, table);
             case 'append':
-                return new SQLiteAppendWriter(configuration, this.path, table);
+                return new SQLiteAppendWriter(configuration, this, table);
             case 'overwrite':
-                return new SQLiteOverwriteWriter(configuration, this.path, table);
+                return new SQLiteOverwriteWriter(configuration, this, table);
             default:
                 throw new TypeError(`Destination does not support ${configuration.destinationSyncMode}`);
         }

package/dist/sqlite-file-store.d.ts

@@ -7,10 +7,13 @@ export declare class SQLiteFileStore {
     static readonly chunkSize: number;
     readonly name: string;
     constructor(database: DatabaseSync, table: SQLiteTable, column: SQLiteColumn);
-    static tableName(table: SQLiteTable, column: Pick<SQLiteColumn, 'name'>): string;
+    static tableName(table: Pick<SQLiteTable, 'location'>, column: Pick<SQLiteColumn, 'name'>): string;
     get published(): boolean;
     stage(): void;
-    save(content: FileContent): Promise<number>;
+    save(content: FileContent): Promise<{
+        readonly id: number;
+        readonly sha256: Uint8Array;
+    }>;
     publish(): void;
     discard(): void;
     unstage(): void;

package/dist/sqlite-file-store.js

@@ -1,24 +1,28 @@
-import { hash } from 'node:crypto';
+import { createHash, hash } from 'node:crypto';
+import { Declaration } from '@zukhruf/sqlite-drift';
 import { identifiers } from './sqlite-identifiers.js';
 const quote = (name) => `"${name.replaceAll('"', '""')}"`;
 const tablePrefix = '_elt_files_';
 const chunks = '("file" INTEGER NOT NULL, "n" INTEGER NOT NULL, "bytes" BLOB NOT NULL, PRIMARY KEY ("file", "n")) STRICT';
+// The highest id that each store gave a file it published. Ids only go up: a
+// row of the history keeps the id of its file for a reading as of a position,
+// so no other file may take that id, even after the chunks of the file, or the
+// whole store, are gone.
+const ids = new Declaration([
+    'CREATE TABLE "_elt_file_ids" ("store" TEXT PRIMARY KEY, "last" INTEGER NOT NULL) STRICT',
+], { rows: 'replace' });
 // A file column's bytes, in ordered chunks: SQLite caps one BLOB at 1e9 bytes
 // and a whole-file BLOB would hold the file in memory. The row keeps the
-// file's id; triggers drop a row's chunks in the same transaction as the row.
-// A load stages chunks in a connection-private TEMP table and publishes them
-// inside a commit, so a load holds no transaction while it reads files.
+// file's id. A load stages chunks in a connection-private TEMP table and
+// publishes them inside a commit, so a load holds no transaction while it
+// reads files.
 export class SQLiteFileStore {
     static chunkSize = 4 * 1024 * 1024;
     name;
     #database;
-    #table;
-    #column;
     #staged;
     constructor(database, table, column) {
         this.#database = database;
-        this.#table = table;
-        this.#column = column;
         this.name = SQLiteFileStore.tableName(table, column);
         this.#staged = quote(`${tablePrefix}stage_${this.name.slice(tablePrefix.length)}`);
     }
@@ -34,6 +38,14 @@ export class SQLiteFileStore {
             .prepare(`SELECT 1 FROM sqlite_schema WHERE "type" = 'table' AND lower("name") = ?`)
             .get(location) !== undefined);
     }
+    #lastGivenId() {
+        if (!this.#exists('_elt_file_ids'))
+            return 0;
+        const row = this.#database
+            .prepare('SELECT "last" FROM "_elt_file_ids" WHERE "store" = ?')
+            .get(this.name);
+        return Number(row?.last ?? 0);
+    }
     get published() {
         return this.#exists(this.name);
     }
@@ -41,33 +53,37 @@ export class SQLiteFileStore {
         this.#database.exec(`DROP TABLE IF EXISTS temp.${this.#staged}`);
         this.#database.exec(`CREATE TEMP TABLE ${this.#staged} ${chunks}`);
     }
-    // An empty file still stores one empty chunk, so its id stays reserved. Ids
-    // continue after both the published and the staged chunks.
+    // Stages the file's chunks under an id above each id the store gave and each
+    // staged id. An empty file still stores one empty chunk, so its id stays
+    // reserved. Gives the id and the SHA-256 of the bytes, which names the
+    // content.
     async save(content) {
-        const highest = (table) => Number(this.#database
-            .prepare(`SELECT coalesce(max("file"), 0) AS "file" FROM ${table}`)
+        const given = this.#lastGivenId();
+        const staged = Number(this.#database
+            .prepare(`SELECT coalesce(max("file"), 0) AS "file" FROM temp.${this.#staged}`)
             .get()?.file);
-        const file = Math.max(this.published ? highest(quote(this.name)) : 0, highest(`temp.${this.#staged}`)) + 1;
+        const id = Math.max(given, staged) + 1;
         const insert = this.#database.prepare(`INSERT INTO temp.${this.#staged} ("file", "n", "bytes") VALUES (?, ?, ?)`);
+        const sha256 = createHash('sha256');
         let n = 0;
-        for await (const chunk of content.chunks(SQLiteFileStore.chunkSize))
-            insert.run(file, n++, chunk);
+        for await (const chunk of content.chunks(SQLiteFileStore.chunkSize)) {
+            sha256.update(chunk);
+            insert.run(id, n++, chunk);
+        }
         if (n === 0)
-            insert.run(file, 0, new Uint8Array());
-        return file;
+            insert.run(id, 0, new Uint8Array());
+        return { id, sha256: sha256.digest() };
     }
-    // Moves the staged chunks into the store, inside a commit, and attaches the
-    // triggers to the target once it exists. A reload swaps in a table without
-    // them, which its next commit attaches again.
+    // Moves the staged chunks into the store, and records the highest id they
+    // take, inside a commit.
     publish() {
         const table = quote(this.name);
-        const column = this.#column.quotedName;
         this.#database.exec(`CREATE TABLE IF NOT EXISTS ${table} ${chunks}`);
-        if (this.#exists(this.#table.location)) {
-            this.#database.exec(`CREATE TRIGGER IF NOT EXISTS ${quote(`${this.name}_delete`)} AFTER DELETE ON ${this.#table.quotedName} BEGIN DELETE FROM ${table} WHERE "file" = old.${column}; END`);
-            this.#database.exec(`CREATE TRIGGER IF NOT EXISTS ${quote(`${this.name}_update`)} AFTER UPDATE OF ${column} ON ${this.#table.quotedName} WHEN old.${column} IS NOT new.${column} BEGIN DELETE FROM ${table} WHERE "file" = old.${column}; END`);
-        }
         this.#database.exec(`INSERT INTO ${table} SELECT * FROM temp.${this.#staged}`);
+        ids.apply(this.#database);
+        this.#database
+            .prepare(`INSERT INTO "_elt_file_ids" ("store", "last") SELECT ?, "file" FROM (SELECT max("file") AS "file" FROM temp.${this.#staged}) WHERE "file" IS NOT NULL ON CONFLICT ("store") DO UPDATE SET "last" = excluded."last"`)
+            .run(this.name);
         this.discard();
     }
     discard() {

package/dist/sqlite-history.d.ts

@@ -0,0 +1,14 @@
+import type { DatabaseSync, SQLInputValue } from 'node:sqlite';
+import type { SQLiteTable } from './sqlite-table.ts';
+export declare const histories = "_elt_histories";
+export declare class SQLiteHistory {
+    #private;
+    readonly name: string;
+    constructor(table: Pick<SQLiteTable, 'name' | 'location'>);
+    create(database: DatabaseSync, written: readonly string[]): void;
+    keep(database: DatabaseSync, replacedIn: number, rows: string, params?: readonly SQLInputValue[]): void;
+}
+export declare function expireHistory(database: DatabaseSync, through: number): void;
+export declare function rowHash(columns: readonly {
+    readonly name: string;
+}[], values: readonly SQLInputValue[]): string;