@eltjs/elt-sqlite 0.0.0-stage → 0.1.0

package/dist/sqlite-sync-history.js

@@ -0,0 +1,119 @@
+import { transaction } from '@zukhruf/sqlite-transaction';
+import { SyncHistory, copyStatus, errorMessage, passError, passFailureType, passStatus, recordedFailure, } from '@eltjs/elt';
+import { openDestination } from './open-destination.js';
+import { installSQLiteCatalog } from './sqlite-catalog.js';
+import { SQLiteDestination } from './sqlite-destination.js';
+import { attempts, checks, coverage, syncHistoryTables, syncHistoryViews, } from './sqlite-sync-history-schema.js';
+import { publishSQLiteViews } from './sqlite-views.js';
+// Records every pass in the file its connection loads, beside the data, and
+// shows readers that history, and the catalog, through views there.
+export class SQLiteSyncHistory extends SyncHistory {
+    #soleWriter;
+    // soleWriter: the caller knows that no other process runs a pass of its
+    // connections while this history is in use, such as the leader of a
+    // connection's flight.
+    constructor(options) {
+        super();
+        this.#soleWriter = options?.soleWriter === true;
+        Object.freeze(this);
+    }
+    validate(connection) {
+        this.#path(connection);
+    }
+    async recordChecks(connection, streamChecks) {
+        write(this.#path(connection), (database) => {
+            const check = database.prepare(`INSERT INTO ${checks} ("connector", "stream", "checked_at") VALUES (?, ?, ?) ON CONFLICT DO UPDATE SET "checked_at" = excluded."checked_at"`);
+            for (const { stream, at } of streamChecks)
+                check.run(connection.name, stream, at.toISOString());
+        });
+    }
+    // Every selected stream is declared before reading, even if it produces no
+    // rows or the process dies: an unfinished attempt stays running, never success.
+    async begin(connection, copies, at) {
+        const path = this.#path(connection);
+        const id = write(path, (database) => {
+            const attempt = database
+                .prepare(`INSERT INTO ${attempts} ("connector", "source", "started_at") VALUES (?, ?, ?) RETURNING "id"`)
+                .get(connection.name, connection.source.identity, at.toISOString());
+            if (attempt === undefined)
+                throw new Error('Sync attempt was not recorded');
+            const id = Number(attempt.id);
+            const declare = database.prepare(`INSERT INTO ${coverage} ("attempt_id", "stream", "target_schema", "target_table", "sync_mode", "destination_sync_mode", "description", "selection") VALUES (?, ?, 'main', ?, ?, ?, ?, ?)`);
+            for (const { copy, coverage } of copies)
+                declare.run(id, copy.from.name, copy.to.name, copy.configuration.syncMode, copy.configuration.destinationSyncMode, coverage.description, JSON.stringify(coverage.selection));
+            return id;
+        });
+        return {
+            finish: async (outcomes, upstream, ended) => this.#finish(path, id, outcomes, upstream, ended),
+            fail: async (error, failureType, ended) => write(path, (database) => fail(database, id, error, failureType, ended)),
+        };
+    }
+    // Brings the history's tables to their declaration and publishes its views
+    // and the catalog in each connection's file. Tables another version
+    // declared otherwise are replaced, and their history with them. A sole
+    // writer then records every pass of its connections that began and never
+    // ended as failed: the process running it ended first, so no outcome will
+    // come, as Airbyte fails a connection's unfinished jobs before it starts the
+    // next. Any other caller leaves them running, since another process may
+    // still be running them.
+    async install(connections, at) {
+        for (const path of new Set(connections.map((each) => this.#path(each)))) {
+            write(path, (database) => {
+                syncHistoryTables.apply(database);
+                publishSQLiteViews(database, { views: syncHistoryViews });
+            });
+            installSQLiteCatalog({ path });
+        }
+        if (this.#soleWriter)
+            for (const connection of connections)
+                this.#failUnfinished(connection, at);
+    }
+    #failUnfinished(connection, at) {
+        const error = new Error('The process running this pass ended before the pass did');
+        write(this.#path(connection), (database) => {
+            const unfinished = database
+                .prepare(`SELECT "id" FROM ${attempts} WHERE "connector" = ? AND "status" = 'running'`)
+                .all(connection.name);
+            for (const { id } of unfinished)
+                fail(database, Number(id), error, 'system', at);
+        });
+    }
+    #path({ name, destination }) {
+        if (!(destination instanceof SQLiteDestination))
+            throw new TypeError(`Connection ${name}: SQLite sync history records SQLite destinations only`);
+        return persistent(destination);
+    }
+    #finish(path, id, outcomes, upstream, ended) {
+        write(path, (database) => {
+            const record = database.prepare(`UPDATE ${coverage} SET "status" = ?, "written_count" = ?, "deleted_count" = ?, "failures" = ? WHERE "attempt_id" = ? AND "stream" = ?`);
+            for (const outcome of outcomes)
+                record.run(copyStatus(outcome), outcome.count, outcome.deleted, JSON.stringify(outcome.failures.map(recordedFailure)), id, outcome.copy.from.name);
+            const bound = database.prepare(`UPDATE ${coverage} SET "upstream_current_as_of" = ? WHERE "attempt_id" = ? AND "stream" = ?`);
+            for (const [stream, at] of upstream)
+                bound.run(at.toISOString(), id, stream);
+            database
+                .prepare(`UPDATE ${attempts} SET "completed_at" = ?, "status" = ?, "error" = ?, "failure_type" = ? WHERE "id" = ?`)
+                .run(ended.toISOString(), passStatus(outcomes), passError(outcomes), passFailureType(outcomes), id);
+        });
+    }
+}
+// A pass that produced no outcomes, so what it committed is unknown.
+function fail(database, id, error, failureType, at) {
+    database
+        .prepare(`UPDATE ${coverage} SET "status" = 'failed', "failures" = ? WHERE "attempt_id" = ?`)
+        .run(JSON.stringify([
+        recordedFailure({ partition: null, error, failureType }),
+    ]), id);
+    database
+        .prepare(`UPDATE ${attempts} SET "status" = 'failed', "completed_at" = ?, "error" = ?, "failure_type" = ? WHERE "id" = ?`)
+        .run(at.toISOString(), errorMessage(error), failureType, id);
+}
+function persistent({ path }) {
+    if (path === ':memory:')
+        throw new TypeError('SQLite sync history requires a destination file');
+    return path;
+}
+function write(path, work) {
+    using database = openDestination(path);
+    return transaction(database, () => work(database), { mode: 'immediate' });
+}

package/dist/sqlite-table.d.ts

@@ -0,0 +1,15 @@
+import type { Stream } from '@eltjs/elt';
+import { Target } from '@eltjs/elt';
+import { SQLiteColumn } from './sqlite-column.ts';
+export declare class SQLiteTable extends Target {
+    readonly name: string;
+    readonly columns: readonly SQLiteColumn[];
+    readonly readerView: string | undefined;
+    constructor(name: string, columns?: readonly SQLiteColumn[], readerView?: string);
+    withReaderView(name: string): SQLiteTable;
+    resolve(stream: Stream): SQLiteTable;
+    get location(): string;
+    get quotedName(): string;
+    get createTableSQL(): string;
+    definition(name: string): string;
+}

package/dist/sqlite-table.js

@@ -0,0 +1,83 @@
+import { Target } from '@eltjs/elt';
+import { SQLiteColumn, canonical } from './sqlite-column.js';
+import { SQLiteColumns } from './sqlite-columns.js';
+import { identifiers } from './sqlite-identifiers.js';
+// A reusable target definition. Empty columns mean infer from the Copy's
+// stream. Its own name, its view's and its columns' follow the destination's
+// naming rule, so a field SQLite would take for another still loads.
+export class SQLiteTable extends Target {
+    name;
+    columns;
+    // The name readers query: a documented view of exactly this table.
+    readerView;
+    constructor(name, columns, readerView) {
+        if (typeof name !== 'string' ||
+            (readerView !== undefined && typeof readerView !== 'string'))
+            throw new TypeError('A table and its view are named by strings');
+        const table = identifiers.relation(name);
+        const view = readerView === undefined ? undefined : identifiers.relation(readerView);
+        if (view !== undefined && identifiers.key(view) === identifiers.key(table))
+            throw new TypeError('A reader view needs a name of its own');
+        if (columns !== undefined &&
+            (!Array.isArray(columns) ||
+                columns.length === 0 ||
+                !columns.every((column) => column instanceof SQLiteColumn)))
+            throw new TypeError('A table requires at least one SQLite column');
+        const fields = (columns ?? []).map((column) => column.field);
+        if (new Set(fields).size !== fields.length)
+            throw new TypeError('Two columns hold one field');
+        if ((columns ?? []).filter((column) => column.isPrimaryKey).length > 1)
+            throw new TypeError('Only one primary-key column is supported');
+        const fileReads = (columns ?? []).flatMap((column) => column.fileRead === undefined ? [] : [column.fileRead]);
+        for (const column of columns ?? []) {
+            if (column.fileRead === undefined)
+                continue;
+            if (column.fileRead.outputType === 'bytes'
+                ? column.kind !== 'blob'
+                : column.kind !== 'text')
+                throw new TypeError('Original files require a BLOB column; parsed text and stored references require a TEXT column');
+        }
+        super(fileReads);
+        this.name = table;
+        this.columns = Object.freeze(identifiers
+            .columns(columns ?? [])
+            .map(([column, stored]) => column.named(stored)));
+        this.readerView = view;
+        Object.freeze(this);
+    }
+    // Loads also keep a view of this table under another name, created with the
+    // table and described by the same descriptions, which every column then needs.
+    withReaderView(name) {
+        return new SQLiteTable(this.name, this.columns.length === 0 ? undefined : this.columns, name);
+    }
+    resolve(stream) {
+        if (this.columns.length === 0)
+            return new SQLiteTable(this.name, SQLiteColumns.fromSchema(stream.jsonSchema), this.readerView);
+        const { properties } = stream.jsonSchema;
+        // Explicit columns of a stream that declares no properties stand alone.
+        if (properties === undefined)
+            return this;
+        for (const column of this.columns)
+            if (column.fileRead === undefined &&
+                !Object.hasOwn(properties, column.field))
+                throw new TypeError(`Stream ${stream.name} does not describe field ${column.field}`);
+        return this;
+    }
+    // SQLite takes names that differ only in ASCII case for one, so one table
+    // has one location.
+    get location() {
+        return identifiers.key(this.name);
+    }
+    get quotedName() {
+        return `"${this.name.replaceAll('"', '""')}"`;
+    }
+    get createTableSQL() {
+        return `CREATE TABLE IF NOT EXISTS ${this.definition(this.quotedName)}`;
+    }
+    // A table named name with these columns, as CREATE TABLE spells it.
+    definition(name) {
+        if (this.columns.length === 0)
+            throw new TypeError('Resolve inferred columns before creating a table');
+        return `${name} (${this.columns.map((column) => column.definition).join(', ')}, "loaded_at" TEXT NOT NULL${canonical('timestamp', '"loaded_at"')}) STRICT`;
+    }
+}

package/dist/sqlite-views.d.ts

@@ -0,0 +1,16 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { ReaderRelation } from '@eltjs/elt';
+export type SQLiteView = ReaderRelation & {
+    readonly query: string;
+};
+export declare function viewDefinition(view: SQLiteView): string;
+/**
+ * Publish ordinary views and their descriptions on a caller-owned connection,
+ * inside its transaction when one is open. Supply trusted SQL and list
+ * dependencies before dependents. Only a named view whose stored definition
+ * differs is replaced, so a publication that changes nothing leaves the schema
+ * as it was.
+ */
+export declare function publishSQLiteViews(database: DatabaseSync, { views }: {
+    views: readonly SQLiteView[];
+}): void;

package/dist/sqlite-views.js

@@ -0,0 +1,62 @@
+import { Declaration } from '@zukhruf/sqlite-drift';
+import { transaction } from '@zukhruf/sqlite-transaction';
+import { isReserved } from './reserved.js';
+import { describe } from './sqlite-descriptions.js';
+const quote = (name) => `"${name.replaceAll('"', '""')}"`;
+export function viewDefinition(view) {
+    return `CREATE VIEW ${quote(view.name)} AS ${view.query}`;
+}
+function text(value, what) {
+    if (typeof value !== 'string' ||
+        !value.trim() ||
+        value.includes('\0') ||
+        !value.isWellFormed())
+        throw new TypeError(`Invalid ${what}`);
+}
+/**
+ * Publish ordinary views and their descriptions on a caller-owned connection,
+ * inside its transaction when one is open. Supply trusted SQL and list
+ * dependencies before dependents. Only a named view whose stored definition
+ * differs is replaced, so a publication that changes nothing leaves the schema
+ * as it was.
+ */
+export function publishSQLiteViews(database, { views }) {
+    const names = new Set();
+    for (const view of views) {
+        text(view.name, 'view name');
+        if (isReserved(view.name))
+            throw new TypeError('View names starting with _elt_ are reserved');
+        if (names.has(view.name.toLowerCase()))
+            throw new TypeError('Duplicate view names');
+        names.add(view.name.toLowerCase());
+        text(view.query, 'view query');
+        text(view.description, 'view description');
+        for (const [column, description] of Object.entries(view.columns)) {
+            text(column, 'column name');
+            text(description, 'column description');
+        }
+    }
+    if (views.length === 0)
+        return;
+    // A failed definition restores every replaced view, even if the caller
+    // catches the error and continues its transaction. With no transaction of
+    // the caller, it begins deferred.
+    transaction(database, () => {
+        // A declaration refuses an element of more than one statement, so a
+        // query cannot append DDL.
+        new Declaration(views.map(viewDefinition), { rows: 'replace' }).apply(database);
+        for (const view of views) {
+            const columns = database
+                .prepare('SELECT "name" FROM pragma_table_info(?)')
+                .all(view.name)
+                .map(({ name }) => String(name));
+            if (columns.length !== Object.keys(view.columns).length ||
+                columns.some((name) => !Object.hasOwn(view.columns, name)))
+                throw new TypeError(`View ${quote(view.name)} must describe exactly its output columns: ${columns.join(', ')}`);
+            describe(database, view.name, view.description, Object.fromEntries(Object.entries(view.columns).map(([column, description]) => [
+                column,
+                { description },
+            ])));
+        }
+    }, { mode: 'deferred' });
+}

package/dist/sqlite-writer.d.ts

@@ -0,0 +1,38 @@
+import type { DatabaseSync, SQLInputValue } from 'node:sqlite';
+import type { CopyConfiguration, FieldValues, KeyValue, Stage } from '@eltjs/elt';
+import { Writer } from '@eltjs/elt';
+import type { SQLiteColumn } from './sqlite-column.ts';
+import type { SQLiteTable } from './sqlite-table.ts';
+export declare const seq = "\"_elt_seq\"";
+export declare const op = "\"_elt_op\"";
+export declare abstract class SQLiteWriter extends Writer {
+    #private;
+    readonly configuration: CopyConfiguration;
+    readonly path: string;
+    readonly table: SQLiteTable;
+    constructor(configuration: CopyConfiguration, path: string, table: SQLiteTable);
+    protected initialize(database: DatabaseSync, _replacing: boolean): void;
+    protected dropDedupIndexes(database: DatabaseSync, kept: string | null): void;
+    protected inspect(_database: DatabaseSync): void;
+    protected abstract merge(database: DatabaseSync, stage: string, loadedAt: string, into: string): void;
+    protected build(_database: DatabaseSync, _into: string): void;
+    protected adopt(_database: DatabaseSync): void;
+    protected append(database: DatabaseSync, stage: string, loadedAt: string, into: string): void;
+    protected get replaces(): boolean;
+    protected replace(database: DatabaseSync): void;
+    private get hash();
+    protected get fields(): string[];
+    protected encode(record: unknown): SQLInputValue[];
+    protected deletionKeys(_key: Readonly<Record<string, KeyValue>>): readonly [readonly SQLiteColumn[], SQLInputValue[]];
+    private writers;
+    private own;
+    private exists;
+    private values;
+    clear(writer: string, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
+    private describe;
+    private installReaderView;
+    prepare(database: DatabaseSync, { writer, reloading }: {
+        writer: string;
+        reloading: boolean;
+    }, loadedAt: string): Stage;
+}