@eltjs/source-microsoft-sql-server 0.0.0-stage → 0.1.0

package/dist/rowversion-stream.js

@@ -0,0 +1,70 @@
+import { SqlServerStream, pageSize } from './sql-server-stream.js';
+function parse(state) {
+    if (state === null)
+        return null;
+    if (typeof state === 'object' &&
+        'above' in state &&
+        typeof state.above === 'string' &&
+        'fields' in state &&
+        Array.isArray(state.fields) &&
+        state.fields.every((field) => typeof field === 'string'))
+        return { above: state.above, fields: state.fields };
+    throw new TypeError(`Unknown rowversion state ${JSON.stringify(state)}`);
+}
+// A keyed table with a rowversion column and no Change Tracking the login
+// can read: each read takes the rows inserted or updated since the last,
+// in rowversion order, up to the oldest write still open. A deleted row is
+// never seen, so it stays loaded. Adding a column bumps no rowversion, so a
+// table that gained one is read again from its first row.
+export class RowversionStream extends SqlServerStream {
+    constructor(table) {
+        super(table, {
+            supportedSyncModes: ['full_refresh', 'incremental'],
+            sourceDefinedCursor: true,
+            strategy: `Read by its rowversion column ${table.rowversion?.name}: inserts and updates apply; deletions are not seen.`,
+        });
+    }
+    gauges(watch) {
+        return [watch.rowversion];
+    }
+    coverage() {
+        return {
+            description: `Every row of ${this.table.quoted} the login can read, inserted or updated since the last read`,
+            selection: { table: this.table.quoted },
+        };
+    }
+    async *extract(configuration, state, session) {
+        if (configuration.syncMode === 'full_refresh') {
+            yield* this.everyRow(session);
+            return;
+        }
+        const column = this.#column();
+        const below = await session.minActiveRowversion();
+        const saved = parse(state);
+        let above = saved === null || this.gained(saved.fields) ? '0' : saved.above;
+        const fields = this.fieldNames;
+        for (;;) {
+            let count = 0;
+            for await (const row of session.rowversions(this.table, above, below, pageSize)) {
+                yield this.message(row);
+                above = String(row.value(column));
+                count += 1;
+            }
+            if (count === 0)
+                return;
+            yield {
+                type: 'STATE',
+                stream: this.stream.name,
+                state: { above, fields },
+            };
+            if (count < pageSize)
+                return;
+        }
+    }
+    #column() {
+        const column = this.table.rowversion;
+        if (column === undefined)
+            throw new TypeError(`${this.table.quoted} has no rowversion column`);
+        return column;
+    }
+}

package/dist/snapshot-stream.d.ts

@@ -0,0 +1,12 @@
+import { type CopyConfiguration, type ExtractionCoverage, type SourceMessage } from '@eltjs/elt';
+import type { SqlServerSession, SqlServerTable } from '@eltjs/sdk-microsoft-sql-server';
+import { SqlServerStream } from './sql-server-stream.ts';
+import type { WatchGauges } from './watch-gauges.ts';
+import type { WholeTableGauges } from './whole-table-gauges.ts';
+export declare class SnapshotStream extends SqlServerStream {
+    #private;
+    constructor(table: SqlServerTable, whole: WholeTableGauges);
+    gauges(watch: WatchGauges): readonly import("@eltjs/elt").Gauge<import("@zukhruf/async").Value>[];
+    coverage(): ExtractionCoverage;
+    extract(configuration: CopyConfiguration, state: unknown, session: SqlServerSession, signal: AbortSignal | undefined): AsyncIterable<SourceMessage>;
+}

package/dist/snapshot-stream.js

@@ -0,0 +1,38 @@
+import { ClosedWorld } from '@zukhruf/snapshot-differential';
+import { diffSnapshot, } from '@eltjs/elt';
+import { SqlServerStream } from './sql-server-stream.js';
+// A keyed table with no Change Tracking or rowversion the login can read:
+// every read takes every row, and an incremental copy compares the read with
+// the snapshot in its checkpoint, so it writes only the rows that are new or
+// changed and deletes the rows that are gone. The read lists every row, so a
+// key it does not list was deleted.
+export class SnapshotStream extends SqlServerStream {
+    #whole;
+    constructor(table, whole) {
+        const hint = table.changeTracking === 'denied'
+            ? ` Granting the login VIEW CHANGE TRACKING ON ${table.quoted} would read only changes and deletions.`
+            : '';
+        super(table, {
+            supportedSyncModes: ['full_refresh', 'incremental'],
+            sourceDefinedCursor: true,
+            emitsDeletes: true,
+            strategy: `Read in full on every sync; new and changed rows and deletions apply.${hint}${whole.hint}`,
+        });
+        this.#whole = whole;
+    }
+    gauges(watch) {
+        return this.#whole.gauges(this.table, watch);
+    }
+    coverage() {
+        return {
+            description: `Every row of ${this.table.quoted} the login can read`,
+            selection: { table: this.table.quoted },
+        };
+    }
+    extract(configuration, state, session, signal) {
+        return diffSnapshot(configuration, this.records(session), state, {
+            policy: new ClosedWorld(),
+            signal,
+        });
+    }
+}

package/dist/sql-server-copies.d.ts

@@ -0,0 +1,10 @@
+import { Copy, type Target } from '@eltjs/elt';
+import type { SqlServerSource } from './sql-server-source.ts';
+/**
+ * Every table as a copy into `target(stream)`: an incremental copy that
+ * applies changes by key for a stream that reads changes, and a full
+ * refresh that replaces the target for one that reads every row. Each copy
+ * is named after its source and stream, so its checkpoint follows the table
+ * and two databases that hold a table of one name share a pipeline.
+ */
+export declare function sqlServerCopies<T extends Target>(source: SqlServerSource, target: (stream: string) => T): Copy<T>[];

package/dist/sql-server-copies.js

@@ -0,0 +1,25 @@
+import { Copy } from '@eltjs/elt';
+/**
+ * Every table as a copy into `target(stream)`: an incremental copy that
+ * applies changes by key for a stream that reads changes, and a full
+ * refresh that replaces the target for one that reads every row. Each copy
+ * is named after its source and stream, so its checkpoint follows the table
+ * and two databases that hold a table of one name share a pipeline.
+ */
+export function sqlServerCopies(source, target) {
+    return source.streams.map((stream) => {
+        const mode = stream.sourceDefinedCursor
+            ? {
+                syncMode: 'incremental',
+                destinationSyncMode: 'append_dedup',
+            }
+            : {
+                syncMode: 'full_refresh',
+                destinationSyncMode: 'overwrite',
+            };
+        return new Copy(stream, target(stream.name), {
+            id: `${source.identity}/${stream.name}`,
+            ...mode,
+        });
+    });
+}

package/dist/sql-server-fields.d.ts

@@ -0,0 +1,11 @@
+import type { KeyValue, Properties } from '@eltjs/elt';
+import { type SqlServerRow, type SqlServerTable, type SqlServerValue } from '@eltjs/sdk-microsoft-sql-server';
+export declare function sentence(text: string): string;
+export declare class TableFields {
+    #private;
+    constructor(table: SqlServerTable);
+    get properties(): Properties;
+    get primaryKey(): string[];
+    record(row: SqlServerRow): Record<string, unknown>;
+    key(values: readonly SqlServerValue[]): Record<string, KeyValue>;
+}

package/dist/sql-server-fields.js

@@ -0,0 +1,172 @@
+import { OffsetTimestamp, VariantValue, } from '@eltjs/sdk-microsoft-sql-server';
+// The JSON Schema of a column's values, before nullability and description.
+function scalar(column) {
+    const { value } = column.codec;
+    switch (value.kind) {
+        case 'text':
+        case 'spatial':
+        case 'variant':
+            return { type: 'string' };
+        case 'integer':
+            return { type: 'integer' };
+        case 'int64':
+            return { type: 'string', format: 'int64' };
+        case 'boolean':
+            return { type: 'boolean' };
+        case 'number':
+            return { type: 'number' };
+        case 'decimal':
+            return {
+                type: 'string',
+                format: 'decimal',
+                precision: value.precision,
+                scale: value.scale,
+            };
+        case 'date':
+            return { type: 'string', format: 'date' };
+        case 'local-timestamp':
+            return {
+                type: 'string',
+                format: 'date-time-local',
+                precision: value.precision,
+            };
+        case 'timestamp':
+            return {
+                type: 'string',
+                format: 'date-time',
+                precision: value.precision,
+            };
+        case 'local-time':
+            return {
+                type: 'string',
+                format: 'time-local',
+                precision: value.precision,
+            };
+        case 'bytes':
+            return { type: 'string', contentEncoding: 'base64' };
+    }
+}
+// A description from SQL Server ends its sentence, so the notes after it
+// read as sentences of their own.
+export function sentence(text) {
+    return /[.!?]$/.test(text.trimEnd()) ? text.trimEnd() : `${text.trimEnd()}.`;
+}
+function described(column, sibling) {
+    const type = column.type === 'timestamp'
+        ? 'rowversion, which SQL Server raises on every insert and update of the row'
+        : column.type;
+    const own = column.description === null
+        ? `SQL Server column ${column.name} (${type}).`
+        : sentence(column.description);
+    const notes = [
+        column.codec.value.kind === 'timestamp'
+            ? `The UTC instant; ${sibling} holds the offset it was written with.`
+            : undefined,
+        column.codec.value.kind === 'spatial'
+            ? 'Extended well-known text: SRID=<id>; then the shape with its Z and M.'
+            : undefined,
+        column.masked
+            ? 'Dynamic data masking applies: a login without UNMASK reads masked values.'
+            : undefined,
+    ];
+    return [own, ...notes.filter((note) => note !== undefined)].join(' ');
+}
+function field(schema, nullable, description) {
+    return {
+        ...schema,
+        type: nullable ? [schema.type, 'null'] : schema.type,
+        description,
+    };
+}
+// What a column adds beside itself: the offset of a datetimeoffset, and the
+// base type of a sql_variant.
+function suffix(column) {
+    switch (column.codec.value.kind) {
+        case 'timestamp':
+            return '_offset';
+        case 'variant':
+            return '_type';
+        default:
+            return undefined;
+    }
+}
+// A table's record fields: one per readable column, plus the field a
+// datetimeoffset or sql_variant column adds beside itself, named once for
+// the table. A column may already hold <column>_offset, so the sibling takes
+// the next free name: <column>_offset_1, then _2. Names compare without case,
+// as SQL Server's default collation compares them.
+export class TableFields {
+    #table;
+    #siblings;
+    constructor(table) {
+        this.#table = table;
+        const taken = new Set(table.columns.map(({ name }) => name.toLowerCase()));
+        const siblings = new Map();
+        for (const column of table.columns) {
+            const added = suffix(column);
+            if (added === undefined)
+                continue;
+            const base = `${column.name}${added}`;
+            let name = base;
+            for (let next = 1; taken.has(name.toLowerCase()); next += 1)
+                name = `${base}_${next}`;
+            taken.add(name.toLowerCase());
+            siblings.set(column, name);
+        }
+        this.#siblings = siblings;
+        Object.freeze(this);
+    }
+    get properties() {
+        const fields = {};
+        for (const column of this.#table.columns) {
+            const sibling = this.#siblings.get(column);
+            fields[column.name] = field(scalar(column), column.nullable, described(column, sibling));
+            if (sibling === undefined)
+                continue;
+            fields[sibling] =
+                column.codec.value.kind === 'timestamp'
+                    ? field({ type: 'integer' }, column.nullable, `The offset from UTC, in minutes, ${column.name} was written with.`)
+                    : field({ type: 'string' }, column.nullable, `The SQL Server type of the value ${column.name} holds.`);
+        }
+        return fields;
+    }
+    // The stream's key: the table's primary key, with the base type of a
+    // sql_variant key column, since its text alone can repeat across types.
+    get primaryKey() {
+        return this.#table.primaryKey.flatMap((column) => {
+            const type = column.codec.value.kind === 'variant'
+                ? this.#siblings.get(column)
+                : undefined;
+            return type === undefined ? [column.name] : [column.name, type];
+        });
+    }
+    record(row) {
+        return Object.assign({}, ...this.#table.columns.map((column, index) => this.#plain(column, row.values[index] ?? null)));
+    }
+    // A deleted row's key fields, from its primary key values in key order.
+    key(values) {
+        const fields = {};
+        for (const [index, column] of this.#table.primaryKey.entries()) {
+            const value = values[index] ?? null;
+            for (const [name, part] of Object.entries(this.#plain(column, value))) {
+                if (typeof part !== 'string' &&
+                    typeof part !== 'number' &&
+                    typeof part !== 'boolean')
+                    throw new TypeError(`${this.#table.quoted} key ${column.name} has no value`);
+                if (name === column.name || column.codec.value.kind === 'variant')
+                    fields[name] = part;
+            }
+        }
+        return fields;
+    }
+    #plain(column, value) {
+        const sibling = this.#siblings.get(column);
+        if (sibling === undefined)
+            return { [column.name]: value };
+        if (value instanceof OffsetTimestamp)
+            return { [column.name]: value.instant, [sibling]: value.offset };
+        if (value instanceof VariantValue)
+            return { [column.name]: value.text, [sibling]: value.type };
+        return { [column.name]: null, [sibling]: null };
+    }
+}

package/dist/sql-server-source.d.ts

@@ -0,0 +1,32 @@
+import { type Gauge, Group } from '@zukhruf/async';
+import { Catalog, type CopyConfiguration, type ExtractionCoverage, type Partition, Source, type SourceMessage, type Stream } from '@eltjs/elt';
+import type { SqlServerDatabase, SqlServerSession } from '@eltjs/sdk-microsoft-sql-server';
+import { SqlServerLoginError, SqlServerPermissionError } from '@eltjs/sdk-microsoft-sql-server';
+import { WatchGauges } from './watch-gauges.ts';
+declare class StreamGroup extends Group implements AsyncDisposable {
+    #private;
+    constructor(members: readonly Gauge[], gauges: WatchGauges);
+    [Symbol.asyncDispose](): Promise<void>;
+}
+export declare class SqlServerSource extends Source<SqlServerSession> {
+    #private;
+    readonly identity: string;
+    protected readonly catalog: Catalog;
+    static discover(database: SqlServerDatabase, options?: {
+        readonly schemas?: readonly string[];
+    }): Promise<SqlServerSource>;
+    private constructor();
+    get streams(): readonly Stream[];
+    protected open(): Promise<SqlServerSession>;
+    protected readonly configErrors: (typeof SqlServerLoginError | typeof SqlServerPermissionError)[];
+    coverage(stream: Stream): ExtractionCoverage;
+    currentAsOf(): Promise<null>;
+    changeSignals(streams: readonly Stream[]): {
+        gauge: StreamGroup;
+        streams: Stream[];
+        interval: number;
+        timeout: number;
+    }[];
+    protected extract(configuration: CopyConfiguration, state: unknown, _partition: Partition | null, session: SqlServerSession, signal: AbortSignal | undefined): AsyncIterable<SourceMessage>;
+}
+export {};

package/dist/sql-server-source.js

@@ -0,0 +1,113 @@
+import { Group } from '@zukhruf/async';
+import { Catalog, Source, } from '@eltjs/elt';
+import { SqlServerLoginError, SqlServerPermissionError, } from '@eltjs/sdk-microsoft-sql-server';
+import { ChangeTrackingStream } from './change-tracking-stream.js';
+import { FullRefreshStream } from './full-refresh-stream.js';
+import { RowversionStream } from './rowversion-stream.js';
+import { SnapshotStream } from './snapshot-stream.js';
+import { WatchGauges } from './watch-gauges.js';
+import { WholeTableGauges } from './whole-table-gauges.js';
+// How often a watch asks the database whether anything changed.
+const pollIntervalMs = 30_000;
+// How long the queries of one check can take. Each query took 160 to 300 ms
+// on Azure SQL Database, and a check runs at most four at once, one on each
+// connection of the session. A check that takes a third of the interval is
+// stuck: its gauges fail, as a system failure, and the next check reads them
+// again.
+const checkTimeoutMs = 10_000;
+// The gauges of one stream: a check wakes the stream when one changed. The
+// streams of a watch share its session, which the first group that the
+// pipeline disposes closes.
+class StreamGroup extends Group {
+    #gauges;
+    constructor(members, gauges) {
+        super(members);
+        this.#gauges = gauges;
+    }
+    [Symbol.asyncDispose]() {
+        return this.#gauges[Symbol.asyncDispose]();
+    }
+}
+// Change Tracking when the login can read it, else the rowversion, else a
+// snapshot of the whole table, all by key; a table with no key in full.
+function streamFor(table, whole) {
+    if (table.primaryKey.length === 0)
+        return new FullRefreshStream(table, whole);
+    if (table.changeTracking === 'readable')
+        return new ChangeTrackingStream(table);
+    if (table.rowversion !== undefined)
+        return new RowversionStream(table);
+    return new SnapshotStream(table, whole);
+}
+// Every table one SQL Server database holds that the login can read, one
+// stream each, named <schema>.<table>. The streams are the database's own
+// tables, so a source is discovered, and a host discovers it again on every
+// run: a table whose columns changed arrives as a changed stream, which
+// starts its copy over.
+export class SqlServerSource extends Source {
+    identity;
+    catalog;
+    #database;
+    #streams;
+    static async discover(database, options = {}) {
+        await using session = await database.open();
+        // A table whose every column is denied has nothing to read.
+        const tables = (await session.tables()).filter(({ schema, columns }) => columns.length > 0 &&
+            (options.schemas === undefined || options.schemas.includes(schema)));
+        // A schema that keeps nothing is misspelled, cased unlike the database, or
+        // not granted; loading less in silence would hide it.
+        for (const schema of options.schemas ?? [])
+            if (!tables.some((table) => table.schema === schema))
+                throw new TypeError(`The schemas to keep name ${schema}, which holds no table this login can read in ${database.location}`);
+        const whole = await WholeTableGauges.discover(session);
+        return new SqlServerSource(database, tables.map((table) => streamFor(table, whole)));
+    }
+    constructor(database, streams) {
+        super();
+        this.identity = `sql-server:${database.name}`;
+        this.#database = database;
+        this.#streams = new Map(streams.map((stream) => [stream.stream.name, stream]));
+        this.catalog = new Catalog(streams.map(({ stream }) => stream));
+        Object.freeze(this);
+    }
+    // The streams the database's tables became, by name.
+    get streams() {
+        return this.catalog.streams;
+    }
+    open() {
+        return this.#database.open();
+    }
+    // A server that cannot be reached now is a system failure: a later
+    // attempt, or the next check of a watch, can succeed.
+    configErrors = [
+        SqlServerLoginError,
+        SqlServerPermissionError,
+    ];
+    coverage(stream) {
+        return this.#reader(stream).coverage();
+    }
+    async currentAsOf() {
+        return null;
+    }
+    // Checks the gauges of the streams every 30 s. A refused login fails
+    // every stream as config, and the pipeline reads its gauges no more in
+    // this watch.
+    changeSignals(streams) {
+        const gauges = new WatchGauges(this.#database);
+        return streams.map((stream) => ({
+            gauge: new StreamGroup(this.#reader(stream).gauges(gauges), gauges),
+            streams: [stream],
+            interval: pollIntervalMs,
+            timeout: checkTimeoutMs,
+        }));
+    }
+    extract(configuration, state, _partition, session, signal) {
+        return this.#reader(configuration.stream).extract(configuration, state, session, signal);
+    }
+    #reader(stream) {
+        const reader = this.#streams.get(stream.name);
+        if (reader === undefined)
+            throw new TypeError(`SQL Server has no stream ${stream.name}`);
+        return reader;
+    }
+}

package/dist/sql-server-stream.d.ts

@@ -0,0 +1,27 @@
+import type { Gauge } from '@zukhruf/async';
+import { type CopyConfiguration, type ExtractionCoverage, type SourceMessage, Stream, type SyncMode } from '@eltjs/elt';
+import type { SqlServerRow, SqlServerSession, SqlServerTable } from '@eltjs/sdk-microsoft-sql-server';
+import { TableFields } from './sql-server-fields.ts';
+import type { WatchGauges } from './watch-gauges.ts';
+export declare const pageSize = 10000;
+type SyncTraits = {
+    readonly supportedSyncModes: readonly SyncMode[];
+    readonly sourceDefinedCursor?: true;
+    readonly emitsDeletes?: true;
+    readonly strategy: string;
+};
+export declare abstract class SqlServerStream {
+    readonly table: SqlServerTable;
+    readonly fields: TableFields;
+    readonly stream: Stream;
+    constructor(table: SqlServerTable, traits: SyncTraits);
+    abstract coverage(): ExtractionCoverage;
+    abstract gauges(watch: WatchGauges): readonly Gauge[];
+    protected get fieldNames(): readonly string[];
+    protected gained(read: readonly string[]): boolean;
+    abstract extract(configuration: CopyConfiguration, state: unknown, session: SqlServerSession, signal: AbortSignal | undefined): AsyncIterable<SourceMessage>;
+    protected message(row: SqlServerRow): SourceMessage;
+    protected records(session: SqlServerSession): AsyncGenerator<Record<string, unknown>>;
+    protected everyRow(session: SqlServerSession): AsyncGenerator<SourceMessage>;
+}
+export {};

package/dist/sql-server-stream.js

@@ -0,0 +1,65 @@
+import { Stream, validateRecords, } from '@eltjs/elt';
+import { TableFields, sentence } from './sql-server-fields.js';
+// Rows per page of a resumable read: each page ends in a checkpoint, so a
+// read that fails resumes after the last page it committed.
+export const pageSize = 10_000;
+// One table as a stream. Each way of reading a table is its own subclass:
+// the base names the stream after the table, describes its columns, and
+// turns rows into records; a subclass declares how it syncs and reads.
+export class SqlServerStream {
+    table;
+    fields;
+    stream;
+    constructor(table, traits) {
+        this.table = table;
+        this.fields = new TableFields(table);
+        const { properties } = this.fields;
+        this.stream = new Stream({
+            name: `${table.schema}.${table.name}`,
+            jsonSchema: {
+                type: 'object',
+                description: [
+                    table.description === null
+                        ? `SQL Server table ${table.quoted}.`
+                        : sentence(table.description),
+                    traits.strategy,
+                    table.unreadable.length > 0
+                        ? `The login cannot read ${table.unreadable.join(', ')}, which are left out.`
+                        : undefined,
+                ]
+                    .filter((part) => part !== undefined)
+                    .join(' '),
+                properties,
+                required: Object.keys(properties),
+            },
+            primaryKey: this.fields.primaryKey,
+            supportedSyncModes: traits.supportedSyncModes,
+            sourceDefinedCursor: traits.sourceDefinedCursor,
+            emitsDeletes: traits.emitsDeletes,
+        });
+    }
+    // The fields a read carries. A cursor's state records them, since a field
+    // the table gains later reaches only rows changed after it unless every row
+    // is read again: elt keeps a stream's checkpoint and rows through a change
+    // of its fields, and leaves filling them in to the source.
+    get fieldNames() {
+        return Object.keys(this.fields.properties);
+    }
+    // Whether this stream carries a field a state was read without.
+    gained(read) {
+        return this.fieldNames.some((field) => !read.includes(field));
+    }
+    message(row) {
+        const [data] = validateRecords(this.stream, [this.fields.record(row)], 'SQL Server');
+        return { stream: this.stream.name, data };
+    }
+    // Every row as it is now, in one read.
+    async *records(session) {
+        for await (const row of session.rows(this.table))
+            yield* validateRecords(this.stream, [this.fields.record(row)], 'SQL Server');
+    }
+    async *everyRow(session) {
+        for await (const data of this.records(session))
+            yield { stream: this.stream.name, data };
+    }
+}

package/dist/watch-gauges.d.ts

@@ -0,0 +1,13 @@
+import { Collector, type Gauge, Period } from '@zukhruf/async';
+import type { SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
+export declare class WatchGauges implements AsyncDisposable {
+    #private;
+    readonly statistics: Collector<string>;
+    readonly rowCounts: Collector<string>;
+    readonly lastWrites: Collector<string>;
+    readonly changeTracking: Gauge;
+    readonly rowversion: Gauge;
+    readonly hour: Period;
+    constructor(database: SqlServerDatabase);
+    [Symbol.asyncDispose](): Promise<void>;
+}