@eltjs/elt-sqlite 0.0.0-stage → 0.1.1

package/dist/computed-dataset.js

@@ -0,0 +1,230 @@
+import { createHash } from 'node:crypto';
+import { Declaration } from '@zukhruf/sqlite-drift';
+import { open } from '@zukhruf/sqlite-open';
+import { transaction } from '@zukhruf/sqlite-transaction';
+import { DerivedDataset, } from '@eltjs/elt';
+import { busyTimeout, openDestination } from './open-destination.js';
+import { SQLiteDestination } from './sqlite-destination.js';
+import { publishSQLiteViews } from './sqlite-views.js';
+import { View } from './view.js';
+const quote = (name) => `"${name.replaceAll('"', '""')}"`;
+// The columns the dataset keeps on every output row beside the rule's own.
+const engineColumns = {
+    input_hash: 'SHA-256 of the input the row was made from; the row is made again when its input changes.',
+    rule: 'What made the row, such as the parser of its format or the model; the row is made again when that changes. unsupported: the rule does not handle the input.',
+    status: 'computed: the rule made the row. empty: the rule found nothing to keep, such as a photo without words. failed: the rule failed, error says why, and the refresh after the passes tries again, at most once for each pass.',
+    error: 'Why the rule failed. NULL unless status is failed.',
+};
+// Data computed on write from what the pipeline's copies load: a rule makes
+// one output row for each input, such as the text of each stored file, as
+// pgai's vectorizer keeps an embedding beside each row. The pipeline refreshes
+// it as its copies commit; each refresh makes the rows whose input or rule
+// changed and deletes the rows whose input is gone, and a refresh after the
+// passes also tries failed rows again; the passes try each input at most
+// once, so a failing rule is asked once a pass.
+// A rule's failure on one input is that row's; it never fails the load.
+// Readers query the view name, over the table raw_<name>, which a refresh
+// replaces when its declaration changes, since it is rebuilt from the inputs.
+export class ComputedDataset extends DerivedDataset {
+    name;
+    #into;
+    #description;
+    #from;
+    #rule;
+    #table;
+    #declaration;
+    // Every column of an output row, in table order.
+    #columns;
+    constructor({ into, name, description, from, compute, }) {
+        super();
+        this.name = name;
+        this.#into = into;
+        this.#description = description;
+        this.#from = Object.freeze([...from]);
+        this.#rule = compute;
+        this.#table = `raw_${name}`;
+        this.#columns = Object.freeze([
+            compute.key.name,
+            ...Object.keys(engineColumns),
+            ...Object.keys(compute.output),
+        ]);
+        const ruleColumns = Object.entries(compute.output).map(([column, { type }]) => `${quote(column)} ${type === 'blob' ? 'BLOB' : 'TEXT'}`);
+        this.#declaration = new Declaration([
+            `CREATE TABLE ${quote(this.#table)} (${[
+                `${quote(compute.key.name)} TEXT PRIMARY KEY`,
+                '"input_hash" TEXT NOT NULL',
+                '"rule" TEXT NOT NULL',
+                `"status" TEXT NOT NULL CHECK ("status" IN ('computed', 'empty', 'failed'))`,
+                '"error" TEXT',
+                ...ruleColumns,
+            ].join(', ')}) STRICT`,
+        ], { rows: 'replace' });
+        Object.freeze(this);
+    }
+    validate(connections) {
+        for (const target of this.#from)
+            if (!(target instanceof View) &&
+                this.#source(connections, target) === undefined)
+                throw new TypeError(`Computed dataset ${this.name} reads ${target.name}, which no copy of the pipeline loads`);
+        const made = connections
+            .filter(({ destination }) => this.#same(destination))
+            .flatMap(({ steps }) => steps)
+            .flatMap(({ to }) => [to.name, to.readerView])
+            .filter((name) => name !== undefined)
+            .map((name) => name.toLowerCase());
+        for (const name of [this.name, this.#table])
+            if (made.includes(name.toLowerCase()))
+                throw new TypeError(`Computed dataset ${this.name} needs ${name}, which a copy of the pipeline makes`);
+    }
+    async refresh({ connections, signal, trigger, }) {
+        const inputs = this.#inputs(connections);
+        using database = openDestination(this.#into.path);
+        const key = this.#rule.key.name;
+        const stored = transaction(database, () => {
+            this.#declaration.apply(database);
+            publishSQLiteViews(database, {
+                views: [
+                    {
+                        name: this.name,
+                        description: this.#description,
+                        columns: {
+                            [key]: this.#rule.key.description,
+                            ...engineColumns,
+                            ...Object.fromEntries(Object.entries(this.#rule.output).map(([column, { description }]) => [column, description])),
+                        },
+                        query: `SELECT ${this.#columns.map(quote).join(', ')} FROM ${quote(this.#table)}`,
+                    },
+                ],
+            });
+            const rows = new Map();
+            const remove = database.prepare(`DELETE FROM ${quote(this.#table)} WHERE ${quote(key)} = ?`);
+            for (const row of database
+                .prepare(`SELECT ${quote(key)} AS "key", "input_hash", "rule", "status" FROM ${quote(this.#table)}`)
+                .all()) {
+                const id = String(row.key);
+                if (!inputs.has(id))
+                    remove.run(id);
+                else
+                    rows.set(id, {
+                        inputHash: String(row.input_hash),
+                        rule: String(row.rule),
+                        status: String(row.status),
+                    });
+            }
+            return rows;
+        }, { mode: 'immediate' });
+        const write = database.prepare(`INSERT OR REPLACE INTO ${quote(this.#table)} (${this.#columns.map(quote).join(', ')}) VALUES (${this.#columns.map(() => '?').join(', ')})`);
+        const store = (id, inputHash, rule, status, error, values) => write.run(id, inputHash, rule, status, error, ...Object.keys(this.#rule.output).map((column) => values[column] ?? null));
+        const due = [];
+        for (const [id, input] of inputs) {
+            if (signal?.aborted)
+                return;
+            const inputHash = createHash('sha256')
+                .update(JSON.stringify(input))
+                .digest('hex');
+            const rule = await this.#rule.identity(input);
+            const before = stored.get(id);
+            // A rule that does not handle the input keeps what it made before.
+            if (rule === null) {
+                if (before?.status === 'computed')
+                    continue;
+                if (before?.rule === 'unsupported' && before.inputHash === inputHash)
+                    continue;
+                store(id, inputHash, 'unsupported', 'empty', null, {});
+                continue;
+            }
+            if (before !== undefined &&
+                before.inputHash === inputHash &&
+                before.rule === rule) {
+                // A failed input waits for the refresh after the passes.
+                if (before.status !== 'failed' || trigger.kind === 'commit')
+                    continue;
+            }
+            // The passes try each input, as it is now, at most once: the refresh
+            // after them leaves what a commit refresh of the same passes tried.
+            const attempt = `${id}\0${inputHash}\0${rule}`;
+            if (trigger.tried.has(attempt))
+                continue;
+            if (trigger.kind === 'commit')
+                trigger.tried.add(attempt);
+            due.push({ id, input, inputHash, rule });
+        }
+        // A pool of calls, each taking the next batch once its own committed, so
+        // a slow call holds only its place, as a worker pool keeps its other
+        // workers busy. A call that fails stops the pool taking more.
+        const failures = [];
+        let next = 0;
+        await Promise.all(Array.from({ length: this.#rule.concurrency }, async () => {
+            while (failures.length === 0 && signal?.aborted !== true) {
+                const batch = due.slice(next, next + this.#rule.batch);
+                next += this.#rule.batch;
+                if (batch.length === 0)
+                    return;
+                let results;
+                try {
+                    results = await this.#rule.compute(batch.map(({ input }) => input), signal);
+                }
+                catch (error) {
+                    failures.push(error);
+                    return;
+                }
+                // Each call commits on its own, so readers see rows as they land.
+                transaction(database, () => {
+                    batch.forEach(({ id, inputHash, rule }, index) => {
+                        const result = results[index];
+                        if (result === undefined || 'gone' in result)
+                            return;
+                        if ('value' in result)
+                            store(id, inputHash, rule, 'computed', null, result.value);
+                        else if ('error' in result)
+                            store(id, inputHash, rule, 'failed', result.error, {});
+                        else
+                            store(id, inputHash, rule, 'empty', null, {});
+                    });
+                }, { mode: 'immediate' });
+            }
+        }));
+        if (failures.length > 0)
+            throw failures[0];
+    }
+    // Each input by key, from the rows of each relation of from that exists
+    // yet: what the copies loaded, or what a view before this dataset gives. A
+    // key read twice keeps its first input.
+    #inputs(connections) {
+        const inputs = new Map();
+        for (const target of this.#from) {
+            const path = target instanceof View
+                ? target.into.path
+                : this.#source(connections, target);
+            if (path === undefined)
+                continue;
+            using database = open(path, { readOnly: true, timeout: busyTimeout });
+            const relation = target instanceof View
+                ? target.name
+                : (target.readerView ?? target.name);
+            if (database
+                .prepare('SELECT 1 FROM sqlite_schema WHERE lower("name") = lower(?)')
+                .get(relation) === undefined)
+                continue;
+            for (const row of database
+                .prepare(`SELECT * FROM ${quote(relation)}`)
+                .all()) {
+                const read = this.#rule.read(row);
+                if (read !== null && !inputs.has(read.key))
+                    inputs.set(read.key, read.input);
+            }
+        }
+        return inputs;
+    }
+    // The file of the copy of the pipeline that loads target.
+    #source(connections, target) {
+        const connection = connections.find(({ steps }) => steps.some(({ to }) => to === target));
+        return connection?.destination instanceof SQLiteDestination
+            ? connection.destination.path
+            : undefined;
+    }
+    #same(destination) {
+        return (destination instanceof SQLiteDestination &&
+            destination.path === this.#into.path);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+export { ComputedDataset } from './computed-dataset.ts';
+export { openDestination } from './open-destination.ts';
+export { installSQLiteCatalog } from './sqlite-catalog.ts';
+export { SQLiteCheckpointStore } from './sqlite-checkpoint-store.ts';
+export { SQLiteColumn } from './sqlite-column.ts';
+export { SQLiteColumns } from './sqlite-columns.ts';
+export { SQLiteDestination } from './sqlite-destination.ts';
+export { type PassState, type PassStatus, readPassStatus, type StreamPassStatus, } from './sqlite-passes.ts';
+export { SQLiteSyncHistory } from './sqlite-sync-history.ts';
+export { SQLiteTable } from './sqlite-table.ts';
+export { isReserved } from './reserved.ts';
+export { publishSQLiteViews, type SQLiteView } from './sqlite-views.ts';
+export { View } from './view.ts';

package/dist/index.js

@@ -0,0 +1,13 @@
+export { ComputedDataset } from './computed-dataset.js';
+export { openDestination } from './open-destination.js';
+export { installSQLiteCatalog } from './sqlite-catalog.js';
+export { SQLiteCheckpointStore } from './sqlite-checkpoint-store.js';
+export { SQLiteColumn } from './sqlite-column.js';
+export { SQLiteColumns } from './sqlite-columns.js';
+export { SQLiteDestination } from './sqlite-destination.js';
+export { readPassStatus, } from './sqlite-passes.js';
+export { SQLiteSyncHistory } from './sqlite-sync-history.js';
+export { SQLiteTable } from './sqlite-table.js';
+export { isReserved } from './reserved.js';
+export { publishSQLiteViews } from './sqlite-views.js';
+export { View } from './view.js';

package/dist/open-destination.d.ts

@@ -0,0 +1,3 @@
+import type { DatabaseSync } from 'node:sqlite';
+export declare const busyTimeout = 30000;
+export declare function openDestination(path: string): DatabaseSync;

package/dist/open-destination.js

@@ -0,0 +1,14 @@
+import { PersistentWriteAheadLog, open } from '@zukhruf/sqlite-open';
+// How long a connection waits for the commit of another connection, in
+// milliseconds.
+export const busyTimeout = 30_000;
+// Opens a connection that writes a SQLite file. Agents read the file with
+// sqlite3 -readonly from a folder that they cannot write, so the -wal and
+// -shm files must stay after the last close. The last connection that writes
+// the file and closes decides this, so every such connection opens here.
+export function openDestination(path) {
+    return open(path, {
+        journal: new PersistentWriteAheadLog(),
+        timeout: busyTimeout,
+    });
+}

package/dist/reserved.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Whether the destination and its sync history own a table, index or view of
+ * this name: each one whose name starts with `_elt_`, in any case, as SQLite
+ * compares names. No stream, view or computed dataset has such a name, so a
+ * reader that follows the data of a file skips these tables.
+ */
+export declare function isReserved(name: string): boolean;

package/dist/reserved.js

@@ -0,0 +1,9 @@
+/**
+ * Whether the destination and its sync history own a table, index or view of
+ * this name: each one whose name starts with `_elt_`, in any case, as SQLite
+ * compares names. No stream, view or computed dataset has such a name, so a
+ * reader that follows the data of a file skips these tables.
+ */
+export function isReserved(name) {
+    return /^_elt_/i.test(name);
+}

package/dist/sqlite-append-writer.d.ts

@@ -0,0 +1,8 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { CopyConfiguration } from '@eltjs/elt';
+import type { SQLiteTable } from './sqlite-table.ts';
+import { SQLiteWriter } from './sqlite-writer.ts';
+export declare class SQLiteAppendWriter extends SQLiteWriter {
+    constructor(configuration: CopyConfiguration, path: string, table: SQLiteTable);
+    protected merge(database: DatabaseSync, stage: string, loadedAt: string, into: string): void;
+}

package/dist/sqlite-append-writer.js

@@ -0,0 +1,10 @@
+import { SQLiteWriter } from './sqlite-writer.js';
+export class SQLiteAppendWriter extends SQLiteWriter {
+    constructor(configuration, path, table) {
+        super(configuration, path, table);
+        Object.freeze(this);
+    }
+    merge(database, stage, loadedAt, into) {
+        this.append(database, stage, loadedAt, into);
+    }
+}

package/dist/sqlite-catalog.d.ts

@@ -0,0 +1,3 @@
+export declare function installSQLiteCatalog({ path }: {
+    path: string;
+}): void;

package/dist/sqlite-catalog.js

@@ -0,0 +1,33 @@
+import { transaction } from '@zukhruf/sqlite-transaction';
+import { readerCatalog } from '@eltjs/elt';
+import { openDestination } from './open-destination.js';
+import { createDescriptions, descriptions } from './sqlite-descriptions.js';
+import { publishSQLiteViews } from './sqlite-views.js';
+// Lists what readers of one file can query: every described view and its
+// columns, from what loads and publications wrote there; safe to repeat.
+// Every column of a described view is described: loads refuse a reader view
+// with an undescribed column, and publications describe exactly their columns.
+export function installSQLiteCatalog({ path }) {
+    if (path === ':memory:')
+        throw new TypeError('A SQLite catalog requires a database file');
+    using database = openDestination(path);
+    transaction(database, () => {
+        createDescriptions(database);
+        publishSQLiteViews(database, {
+            views: [
+                {
+                    ...readerCatalog,
+                    query: `SELECT 'view' AS "kind", s."name" AS "name", NULL AS "data_type", d."description" AS "description"
+            FROM sqlite_schema s JOIN ${descriptions} d ON d."relation" = s."name" AND d."column" = ''
+            WHERE s."type" = 'view'
+            UNION ALL
+            SELECT 'column', s."name" || '.' || c."column", c."data_type", c."description"
+            FROM sqlite_schema s JOIN ${descriptions} d ON d."relation" = s."name" AND d."column" = ''
+            JOIN ${descriptions} c ON c."relation" = s."name" AND c."column" <> ''
+            WHERE s."type" = 'view'
+            ORDER BY 2`,
+                },
+            ],
+        });
+    }, { mode: 'immediate' });
+}

package/dist/sqlite-checkpoint-store.d.ts

@@ -0,0 +1,9 @@
+import { type CheckpointSession, CheckpointStore } from '@eltjs/elt';
+export declare class SQLiteCheckpointStore extends CheckpointStore {
+    #private;
+    readonly path: string;
+    constructor({ path }: {
+        path: string;
+    });
+    protected session<T>(ids: readonly string[], work: (session: CheckpointSession) => Promise<T>): Promise<T>;
+}

package/dist/sqlite-checkpoint-store.js

@@ -0,0 +1,84 @@
+import { chmodSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { Modes, Mutex } from '@zukhruf/mutex';
+import { SqliteStore } from '@zukhruf/mutex/sqlite';
+import { Declaration } from '@zukhruf/sqlite-drift';
+import { open } from '@zukhruf/sqlite-open';
+import { CheckpointStore } from '@eltjs/elt';
+import { busyTimeout } from './open-destination.js';
+const checkpoints = new Declaration([
+    'CREATE TABLE checkpoints (id TEXT PRIMARY KEY NOT NULL, binding TEXT NOT NULL, reloading INTEGER NOT NULL CHECK (reloading IN (0, 1)), state TEXT NOT NULL) STRICT',
+], { rows: 'replace' });
+export class SQLiteCheckpointStore extends CheckpointStore {
+    path;
+    // One key per replication, in a folder beside the state file. The kernel
+    // releases a key when the run holding it exits.
+    #locks;
+    constructor({ path }) {
+        super();
+        if (!path || path === ':memory:' || path.includes('\0'))
+            throw new TypeError('Checkpoints require a persistent SQLite file');
+        this.path = resolve(path);
+        // The lock never waits (skipIfBusy makes one attempt), and the write of
+        // its token retries only a busy file on Windows: real time, as no caller
+        // has another correct value.
+        this.#locks = new Mutex(new SqliteStore(`${this.path}.locks`, { clock: globalThis }));
+        Object.freeze(this);
+    }
+    // Holds each replication's key in turn, then works. A replication already
+    // running fails the run fast; others run in parallel.
+    async session(ids, work) {
+        const [id, ...rest] = ids;
+        if (id === undefined)
+            return this.#withFile(work);
+        const held = await this.#locks.acquire(id, () => this.session(rest, work), {
+            mode: Modes.skipIfBusy(),
+        });
+        if (!held.acquired)
+            throw new TypeError(`Checkpoint ${id} is in use by another run`);
+        return held.value;
+    }
+    // Each save commits on its own, so it is durable when it resolves.
+    async #withFile(work) {
+        // Runs of other replications write the same file.
+        using database = open(this.path, { timeout: busyTimeout });
+        // SQLite gives the -wal and -shm files the mode of the file when it makes
+        // them, at the first read after the open.
+        chmodSync(this.path, 0o600);
+        // A table another version declared otherwise is replaced with its
+        // checkpoints, so every replication starts over.
+        checkpoints.apply(database);
+        return await work({
+            read: async (id) => {
+                const saved = database
+                    .prepare('SELECT binding, reloading FROM checkpoints WHERE id = ?')
+                    .get(id);
+                return saved === undefined
+                    ? undefined
+                    : {
+                        binding: String(saved.binding),
+                        reloading: saved.reloading === 1,
+                    };
+            },
+            state: async (id) => {
+                const saved = database
+                    .prepare('SELECT state FROM checkpoints WHERE id = ?')
+                    .get(id);
+                return saved === undefined ? undefined : String(saved.state);
+            },
+            save: async (id, { binding, reloading, state }) => {
+                database
+                    .prepare('INSERT INTO checkpoints (id, binding, reloading, state) VALUES (?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET binding = excluded.binding, reloading = excluded.reloading, state = excluded.state')
+                    .run(id, binding, reloading ? 1 : 0, state);
+            },
+            reloaded: async (id) => {
+                database
+                    .prepare('UPDATE checkpoints SET reloading = 0 WHERE id = ?')
+                    .run(id);
+            },
+            remove: async (id) => {
+                database.prepare('DELETE FROM checkpoints WHERE id = ?').run(id);
+            },
+        });
+    }
+}

package/dist/sqlite-column.d.ts

@@ -0,0 +1,53 @@
+import type { SQLInputValue } from 'node:sqlite';
+import type { DeclaredFormat, DocumentParser, FileReference } from '@eltjs/elt';
+import { FileRead } from '@eltjs/elt';
+declare const storageTypes: {
+    readonly text: 'TEXT';
+    readonly integer: 'INTEGER';
+    readonly int64: 'INTEGER';
+    readonly real: 'REAL';
+    readonly decimal: 'TEXT';
+    readonly blob: 'BLOB';
+    readonly boolean: 'INTEGER';
+    readonly date: 'TEXT';
+    readonly timestamp: 'TEXT';
+    readonly local_timestamp: 'TEXT';
+    readonly local_time: 'TEXT';
+    readonly object: 'TEXT';
+};
+type Kind = keyof typeof storageTypes;
+type PlainKind = 'text' | 'integer' | 'real' | 'blob' | 'boolean' | 'date' | 'timestamp' | 'object';
+export type SQLiteColumnType = PlainKind | DeclaredFormat;
+export declare function canonical(kind: Kind, name: string, format?: DeclaredFormat): string;
+export declare class SQLiteColumn {
+    #private;
+    readonly field: string;
+    readonly name: string;
+    readonly type: SQLiteColumnType;
+    readonly kind: Kind;
+    readonly isPrimaryKey: boolean;
+    readonly nullable: boolean;
+    readonly optional: boolean;
+    readonly array: boolean;
+    readonly fileRead?: FileRead;
+    constructor(field: string, type: SQLiteColumnType, options: {
+        nullable: boolean;
+        optional: boolean;
+        primaryKey: boolean;
+        array?: boolean;
+        fileRead?: FileRead;
+        name?: string;
+    });
+    primaryKey(): SQLiteColumn;
+    notNull(): SQLiteColumn;
+    from(file: FileReference): SQLiteColumn;
+    parse(parser: DocumentParser): SQLiteColumn;
+    named(name: string): SQLiteColumn;
+    get quotedName(): string;
+    get storesFile(): boolean;
+    get storageType(): string;
+    get dataType(): string;
+    get definition(): string;
+    encode(record: unknown): SQLInputValue;
+}
+export {};