@eltjs/elt-sqlserver 0.0.0-stage → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +485 -2
  3. package/dist/computed-dataset.d.ts +31 -0
  4. package/dist/computed-dataset.js +380 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +10 -0
  7. package/dist/sql-server-append-writer.d.ts +8 -0
  8. package/dist/sql-server-append-writer.js +13 -0
  9. package/dist/sql-server-as-of.d.ts +14 -0
  10. package/dist/sql-server-as-of.js +44 -0
  11. package/dist/sql-server-checkpoint-store.d.ts +10 -0
  12. package/dist/sql-server-checkpoint-store.js +101 -0
  13. package/dist/sql-server-column.d.ts +53 -0
  14. package/dist/sql-server-column.js +295 -0
  15. package/dist/sql-server-columns.d.ts +15 -0
  16. package/dist/sql-server-columns.js +49 -0
  17. package/dist/sql-server-connection.d.ts +6 -0
  18. package/dist/sql-server-connection.js +56 -0
  19. package/dist/sql-server-deduplicating-writer.d.ts +18 -0
  20. package/dist/sql-server-deduplicating-writer.js +102 -0
  21. package/dist/sql-server-dependents.d.ts +9 -0
  22. package/dist/sql-server-dependents.js +54 -0
  23. package/dist/sql-server-derived-status.d.ts +9 -0
  24. package/dist/sql-server-derived-status.js +60 -0
  25. package/dist/sql-server-descriptions.d.ts +6 -0
  26. package/dist/sql-server-descriptions.js +53 -0
  27. package/dist/sql-server-destination.d.ts +32 -0
  28. package/dist/sql-server-destination.js +95 -0
  29. package/dist/sql-server-failure-type.d.ts +2 -0
  30. package/dist/sql-server-failure-type.js +25 -0
  31. package/dist/sql-server-file-store.d.ts +21 -0
  32. package/dist/sql-server-file-store.js +91 -0
  33. package/dist/sql-server-fit.d.ts +20 -0
  34. package/dist/sql-server-fit.js +52 -0
  35. package/dist/sql-server-history.d.ts +15 -0
  36. package/dist/sql-server-history.js +148 -0
  37. package/dist/sql-server-identifiers.d.ts +10 -0
  38. package/dist/sql-server-identifiers.js +27 -0
  39. package/dist/sql-server-indexes.d.ts +23 -0
  40. package/dist/sql-server-indexes.js +121 -0
  41. package/dist/sql-server-load.d.ts +19 -0
  42. package/dist/sql-server-load.js +58 -0
  43. package/dist/sql-server-numbering.d.ts +32 -0
  44. package/dist/sql-server-numbering.js +255 -0
  45. package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
  46. package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
  47. package/dist/sql-server-overwrite-writer.d.ts +4 -0
  48. package/dist/sql-server-overwrite-writer.js +9 -0
  49. package/dist/sql-server-parts-table.d.ts +24 -0
  50. package/dist/sql-server-parts-table.js +167 -0
  51. package/dist/sql-server-passes.d.ts +6 -0
  52. package/dist/sql-server-passes.js +71 -0
  53. package/dist/sql-server-reader-view.d.ts +12 -0
  54. package/dist/sql-server-reader-view.js +71 -0
  55. package/dist/sql-server-rule-columns.d.ts +12 -0
  56. package/dist/sql-server-rule-columns.js +69 -0
  57. package/dist/sql-server-schemas.d.ts +4 -0
  58. package/dist/sql-server-schemas.js +8 -0
  59. package/dist/sql-server-setup-error.d.ts +3 -0
  60. package/dist/sql-server-setup-error.js +6 -0
  61. package/dist/sql-server-sync-history-schema.d.ts +7 -0
  62. package/dist/sql-server-sync-history-schema.js +120 -0
  63. package/dist/sql-server-sync-history.d.ts +14 -0
  64. package/dist/sql-server-sync-history.js +165 -0
  65. package/dist/sql-server-table.d.ts +10 -0
  66. package/dist/sql-server-table.js +40 -0
  67. package/dist/sql-server-views.d.ts +7 -0
  68. package/dist/sql-server-views.js +70 -0
  69. package/dist/sql-server-writer.d.ts +40 -0
  70. package/dist/sql-server-writer.js +914 -0
  71. package/dist/view.d.ts +23 -0
  72. package/dist/view.js +82 -0
  73. package/package.json +48 -4
@@ -0,0 +1,165 @@
1
+ import { hash } from 'node:crypto';
2
+ import { SyncHistory, copyStatus, errorMessage, passError, passFailureType, passStatus, recordedFailure, } from '@eltjs/elt';
3
+ import { SqlServerDatabase, } from '@eltjs/sdk-microsoft-sql-server';
4
+ import { connect, takeLock, transaction, } from './sql-server-connection.js';
5
+ import { SqlServerDestination } from './sql-server-destination.js';
6
+ import { databaseCopy, grantReading, installNumbering, } from './sql-server-numbering.js';
7
+ import { warehouse } from './sql-server-schemas.js';
8
+ import { attempts, catalog, checks, coverage, syncHistoryTables, syncHistoryViews, } from './sql-server-sync-history-schema.js';
9
+ import { publishViews } from './sql-server-views.js';
10
+ // Records every pass in the _warehouse schema, and shows readers that history
11
+ // and the catalog through views in marts, as in elt-postgresql. readers are
12
+ // the database users or roles that read the database as of a position with
13
+ // their own session: each install grants them the numbering, and nothing
14
+ // else of _warehouse.
15
+ export class SqlServerSyncHistory extends SyncHistory {
16
+ #database;
17
+ #readers;
18
+ constructor({ login, readers = [], }) {
19
+ super();
20
+ this.#database = new SqlServerDatabase(login);
21
+ this.#readers = Object.freeze(readers.map((reader) => {
22
+ if (typeof reader !== 'string' || !reader || reader.length > 128)
23
+ throw new TypeError(`Invalid reader: ${JSON.stringify(reader)}`);
24
+ return reader;
25
+ }));
26
+ Object.freeze(this);
27
+ }
28
+ validate({ name, destination }) {
29
+ if (!(destination instanceof SqlServerDestination))
30
+ throw new TypeError(`Connection ${name}: SQL Server sync history records SQL Server destinations only`);
31
+ }
32
+ async recordChecks(connection, streamChecks) {
33
+ await using session = await connect(this.#database);
34
+ await transaction(session, async () => {
35
+ for (const { stream, at } of streamChecks)
36
+ await session.query(`UPDATE ${checks} SET [checked_at] = CAST(@at AS datetimeoffset(3)) WHERE [connector] = @connector AND [stream] = @stream;
37
+ IF @@ROWCOUNT = 0
38
+ INSERT INTO ${checks} ([connector], [stream], [checked_at]) VALUES (@connector, @stream, CAST(@at AS datetimeoffset(3)));`, { connector: connection.name, stream, at: at.toISOString() });
39
+ });
40
+ }
41
+ // Every selected stream is declared before reading, even if it produces no
42
+ // rows or the process dies. The pass's own session holds an application
43
+ // lock on its attempt until the pipeline disposes the pass, and the server
44
+ // lets go of the locks of a session that ends, also of a killed process.
45
+ // So an attempt of the connection that runs while no session holds its lock
46
+ // will get no outcome, and the next attempt records it as failed (ADR
47
+ // 0009).
48
+ async begin(connection, copies, at) {
49
+ const session = await connect(this.#database);
50
+ try {
51
+ const id = await transaction(session, async () => {
52
+ await failEnded(session, connection.name, at);
53
+ const [attempt] = await session.query(`INSERT INTO ${attempts} ([connector], [source], [started_at]) OUTPUT CONVERT(nvarchar(20), inserted.[id]) AS [id] VALUES (@connector, @source, CAST(@at AS datetimeoffset(3)))`, {
54
+ connector: connection.name,
55
+ source: connection.source.identity,
56
+ at: at.toISOString(),
57
+ });
58
+ if (typeof attempt?.id !== 'string')
59
+ throw new Error('Sync attempt was not recorded');
60
+ // A lock of the session outlives the transaction, so no reader sees
61
+ // the attempt before its lock is held.
62
+ await session.query(`DECLARE @result int;
63
+ EXEC @result = sp_getapplock @Resource = @resource, @LockMode = N'Exclusive', @LockOwner = N'Session', @LockTimeout = 0;
64
+ IF @result < 0 THROW 50000, N'The lock of the attempt was not granted', 1;`, { resource: attemptLock(attempt.id) });
65
+ for (const { copy, coverage: covered } of copies)
66
+ await session.query(`INSERT INTO ${coverage} ([attempt_id], [stream], [target_table], [sync_mode], [destination_sync_mode], [description], [selection])
67
+ VALUES (CAST(@attempt AS bigint), @stream, @target, @syncMode, @destinationSyncMode, @description, CAST(@selection AS json))`, {
68
+ attempt: attempt.id,
69
+ stream: copy.from.name,
70
+ target: copy.to.name,
71
+ syncMode: copy.configuration.syncMode,
72
+ destinationSyncMode: copy.configuration.destinationSyncMode,
73
+ description: covered.description,
74
+ selection: JSON.stringify(covered.selection),
75
+ });
76
+ return attempt.id;
77
+ });
78
+ return {
79
+ finish: async (outcomes, upstream, ended) => {
80
+ await transaction(session, () => finish(session, id, outcomes, upstream, ended));
81
+ },
82
+ fail: async (error, failureType, ended) => {
83
+ await transaction(session, () => fail(session, [id], error, failureType, ended));
84
+ },
85
+ [Symbol.asyncDispose]: () => session[Symbol.asyncDispose](),
86
+ };
87
+ }
88
+ catch (error) {
89
+ await session[Symbol.asyncDispose]();
90
+ throw error;
91
+ }
92
+ }
93
+ // Makes the history's tables, publishes its views and the catalog, and
94
+ // grants each reader the numbering; safe to repeat.
95
+ async install() {
96
+ await using session = await connect(this.#database);
97
+ if (this.#readers.length > 0)
98
+ await installNumbering(session, await databaseCopy(session));
99
+ await transaction(session, async () => {
100
+ // The numbering of the destinations makes _warehouse too, under the
101
+ // same lock.
102
+ await takeLock(session, warehouse);
103
+ await session.query(syncHistoryTables);
104
+ await publishViews(session, [...syncHistoryViews, catalog]);
105
+ for (const reader of this.#readers)
106
+ await grantReading(session, reader);
107
+ });
108
+ }
109
+ }
110
+ // The application lock of an attempt, named by its hash, as sp_getapplock
111
+ // keeps 255 characters of a name.
112
+ const attemptLock = (id) => `elt-attempt-${hash('sha256', `elt:sync-attempt:${id}`).slice(0, 32)}`;
113
+ // Records as failed each attempt of connector that runs while no session
114
+ // holds its lock: the session that ran it ended first, so no outcome will
115
+ // come. Only the server knows which sessions live, so a try of each lock is
116
+ // the test, and the lock it takes ends with the transaction.
117
+ async function failEnded(session, connector, at) {
118
+ const running = await session.query(`SELECT CONVERT(nvarchar(20), [id]) AS [id] FROM ${attempts} WHERE [connector] = @connector AND [status] = N'running'`, { connector });
119
+ const ended = [];
120
+ for (const { id } of running) {
121
+ const [lock] = await session.query(`DECLARE @result int;
122
+ EXEC @result = sp_getapplock @Resource = @resource, @LockMode = N'Exclusive', @LockOwner = N'Transaction', @LockTimeout = 0;
123
+ SELECT @result AS [result];`, { resource: attemptLock(String(id)) });
124
+ if (typeof lock?.result === 'number' && lock.result >= 0)
125
+ ended.push(String(id));
126
+ }
127
+ if (ended.length > 0)
128
+ await fail(session, ended, new Error('The session running this pass ended before the pass did: its process stopped, or lost the database'), 'system', at);
129
+ }
130
+ // Attempts that produced no outcomes, so what they committed is unknown.
131
+ async function fail(session, ids, error, failureType, at) {
132
+ await session.query(`UPDATE ${coverage} SET [status] = N'failed', [failures] = CAST(@failures AS json)
133
+ WHERE [attempt_id] IN (SELECT CAST([value] AS bigint) FROM OPENJSON(@ids));
134
+ UPDATE ${attempts} SET [status] = N'failed', [completed_at] = CAST(@at AS datetimeoffset(3)), [error] = @error, [failure_type] = @failureType
135
+ WHERE [id] IN (SELECT CAST([value] AS bigint) FROM OPENJSON(@ids));`, {
136
+ ids: JSON.stringify(ids),
137
+ failures: JSON.stringify([
138
+ recordedFailure({ partition: null, error, failureType }),
139
+ ]),
140
+ at: at.toISOString(),
141
+ error: errorMessage(error),
142
+ failureType,
143
+ });
144
+ }
145
+ async function finish(session, id, outcomes, upstream, ended) {
146
+ for (const outcome of outcomes)
147
+ await session.query(`UPDATE ${coverage} SET [status] = @status, [written_count] = CAST(@written AS bigint), [deleted_count] = CAST(@deleted AS bigint), [failures] = CAST(@failures AS json)
148
+ WHERE [attempt_id] = CAST(@attempt AS bigint) AND [stream] = @stream`, {
149
+ status: copyStatus(outcome),
150
+ written: String(outcome.count),
151
+ deleted: String(outcome.deleted),
152
+ failures: JSON.stringify(outcome.failures.map(recordedFailure)),
153
+ attempt: id,
154
+ stream: outcome.copy.from.name,
155
+ });
156
+ for (const [stream, at] of upstream)
157
+ await session.query(`UPDATE ${coverage} SET [upstream_current_as_of] = CAST(@at AS datetimeoffset(3)) WHERE [attempt_id] = CAST(@attempt AS bigint) AND [stream] = @stream`, { at: at.toISOString(), attempt: id, stream });
158
+ await session.query(`UPDATE ${attempts} SET [completed_at] = CAST(@ended AS datetimeoffset(3)), [status] = @status, [error] = @error, [failure_type] = @failureType WHERE [id] = CAST(@attempt AS bigint)`, {
159
+ ended: ended.toISOString(),
160
+ status: passStatus(outcomes),
161
+ error: passError(outcomes),
162
+ failureType: passFailureType(outcomes),
163
+ attempt: id,
164
+ });
165
+ }
@@ -0,0 +1,10 @@
1
+ import { type Stream, Target } from '@eltjs/elt';
2
+ import { SqlServerColumn } from './sql-server-column.ts';
3
+ export declare class SqlServerTable extends Target {
4
+ readonly name: string;
5
+ readonly raw: string;
6
+ readonly columns: readonly SqlServerColumn[];
7
+ constructor(name: string, columns?: readonly SqlServerColumn[]);
8
+ resolve(stream: Stream): SqlServerTable;
9
+ get quotedRaw(): string;
10
+ }
@@ -0,0 +1,40 @@
1
+ import { Target, requireDescribed, targetFileReads, } from '@eltjs/elt';
2
+ import { SqlServerColumn } from './sql-server-column.js';
3
+ import { SqlServerColumns } from './sql-server-columns.js';
4
+ import { identifiers, quote, rawTable } from './sql-server-identifiers.js';
5
+ // A reusable target definition. Empty columns mean infer from the Copy's
6
+ // stream. The host names only the reader view in marts; the destination
7
+ // keeps the rows in the raw table in _warehouse. Both names and the columns'
8
+ // follow the destination's naming rule, so a field SQL Server cannot hold as
9
+ // it is still loads.
10
+ export class SqlServerTable extends Target {
11
+ // The name readers query: the view in marts over the raw table.
12
+ name;
13
+ // The table in _warehouse that the commits of the target write.
14
+ raw;
15
+ columns;
16
+ constructor(name, columns) {
17
+ const view = identifiers.reader(name);
18
+ if (columns !== undefined &&
19
+ (!Array.isArray(columns) ||
20
+ columns.length === 0 ||
21
+ !columns.every((column) => column instanceof SqlServerColumn)))
22
+ throw new TypeError('A table requires at least one SQL Server column');
23
+ super(targetFileReads(columns ?? []));
24
+ this.name = view;
25
+ this.raw = rawTable(view);
26
+ this.columns = Object.freeze(identifiers
27
+ .columns(columns ?? [])
28
+ .map(([column, stored]) => column.named(stored)));
29
+ Object.freeze(this);
30
+ }
31
+ resolve(stream) {
32
+ if (this.columns.length === 0)
33
+ return new SqlServerTable(this.name, SqlServerColumns.fromSchema(stream.jsonSchema));
34
+ requireDescribed(stream, this.columns);
35
+ return this;
36
+ }
37
+ get quotedRaw() {
38
+ return quote(this.raw);
39
+ }
40
+ }
@@ -0,0 +1,7 @@
1
+ import type { ReaderRelation } from '@eltjs/elt';
2
+ import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
3
+ export type SqlServerView = ReaderRelation & {
4
+ readonly query: string;
5
+ };
6
+ export declare function publishViews(connection: SqlServerConnection, views: readonly SqlServerView[]): Promise<void>;
7
+ export declare function withdrawView(connection: SqlServerConnection, name: string): Promise<void>;
@@ -0,0 +1,70 @@
1
+ import { takeLock } from './sql-server-connection.js';
2
+ import { refreshDependents } from './sql-server-dependents.js';
3
+ import { TargetDescriptions } from './sql-server-descriptions.js';
4
+ import { quote } from './sql-server-identifiers.js';
5
+ import { inMarts, marts } from './sql-server-schemas.js';
6
+ // Publishes views and their descriptions in marts, inside the caller's
7
+ // transaction, under the lock of marts, which it makes when the database has
8
+ // none. A view whose stored definition is the text its query makes stays as
9
+ // it is; the others are made again in place, so each keeps its object and
10
+ // its grants, and the views over each one that are not in views read its
11
+ // new columns (ADR 0006). A view made again must have exactly the columns
12
+ // that it describes; the descriptions of the columns a view does not have
13
+ // yet, such as a view over * before its table takes a new column, wait for
14
+ // a later publication. Supply trusted SQL, and list dependencies before
15
+ // dependents.
16
+ export async function publishViews(connection, views) {
17
+ await takeLock(connection, marts);
18
+ await connection.query(`IF SCHEMA_ID(N'${marts}') IS NULL EXEC (N'CREATE SCHEMA ${quote(marts)}')`);
19
+ for (const view of views) {
20
+ const relation = inMarts(view.name);
21
+ const changed = !(await stored(connection, relation, view.query));
22
+ // CREATE VIEW must be the only statement of its batch.
23
+ if (changed)
24
+ await refreshDependents(connection, {
25
+ roots: [relation],
26
+ views: views.map(({ name }) => inMarts(name)),
27
+ changed: relation,
28
+ }, async () => {
29
+ await connection.query(`CREATE OR ALTER VIEW ${relation} AS ${view.query}`);
30
+ });
31
+ const columns = (await connection.query('SELECT [name] FROM sys.columns WHERE [object_id] = OBJECT_ID(@view) ORDER BY [column_id]', { view: relation })).map(({ name }) => String(name));
32
+ if (changed &&
33
+ (columns.length !== Object.keys(view.columns).length ||
34
+ columns.some((name) => !Object.hasOwn(view.columns, name))))
35
+ throw new TypeError(`View ${relation} must describe exactly its output columns: ${columns.join(', ')}`);
36
+ await new TargetDescriptions(view.description, Object.fromEntries(Object.entries(view.columns).filter(([name]) => columns.includes(name)))).write(connection, 'VIEW', marts, view.name);
37
+ }
38
+ }
39
+ // Whether the view in marts has the definition that query makes. SQL Server
40
+ // keeps the text of the statement that made the view, with CREATE OR ALTER
41
+ // VIEW kept as CREATE VIEW (probe of 2026-11-20), so the comparison skips
42
+ // the words before the name.
43
+ async function stored(connection, relation, query) {
44
+ const [view] = await connection.query("SELECT OBJECT_DEFINITION(OBJECT_ID(@view, N'V')) AS [definition]", { view: relation });
45
+ return (typeof view?.definition === 'string' &&
46
+ view.definition.replace(/^CREATE\s+(?:OR\s+ALTER\s+)?VIEW\s+/i, '') ===
47
+ `${relation} AS ${query}`);
48
+ }
49
+ // Removes a view and each view over it, deepest first, inside the caller's
50
+ // transaction under the lock of marts, as elt-postgresql drops a view with
51
+ // CASCADE and dbt drops a model's relation: a view of the pipeline over it
52
+ // waits or is made again by its own refresh, and a view that someone made
53
+ // over it is gone. SQL Server keeps the dependency of a view by name, so a
54
+ // view over a dropped view would stay, and fail each read with error 4413.
55
+ export async function withdrawView(connection, name) {
56
+ await takeLock(connection, marts);
57
+ const readers = await connection.query(`WITH [readers] AS (
58
+ SELECT d.[referencing_id] AS [id], 1 AS [depth] FROM sys.sql_expression_dependencies AS d
59
+ WHERE d.[referenced_id] = OBJECT_ID(@view, N'V')
60
+ UNION ALL
61
+ SELECT d.[referencing_id], r.[depth] + 1 FROM [readers] AS r
62
+ JOIN sys.sql_expression_dependencies AS d ON d.[referenced_id] = r.[id]
63
+ )
64
+ SELECT QUOTENAME(OBJECT_SCHEMA_NAME([id])) + N'.' + QUOTENAME(OBJECT_NAME([id])) AS [view]
65
+ FROM [readers] WHERE OBJECTPROPERTY([id], 'IsView') = 1
66
+ GROUP BY [id] ORDER BY MAX([depth]) DESC`, { view: inMarts(name) });
67
+ for (const { view } of readers)
68
+ await connection.query(`DROP VIEW ${String(view)}`);
69
+ await connection.query(`DROP VIEW IF EXISTS ${inMarts(name)}`);
70
+ }
@@ -0,0 +1,40 @@
1
+ import { type Clearing, type CopyConfiguration, type FieldValues, type KeyValue, type Stage, Writer } from '@eltjs/elt';
2
+ import type { SqlServerConnection, SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
3
+ import type { EncodedValue, SqlServerColumn } from './sql-server-column.ts';
4
+ import { TargetHistory } from './sql-server-history.ts';
5
+ import type { SqlServerLoad } from './sql-server-load.ts';
6
+ import type { SqlServerTable } from './sql-server-table.ts';
7
+ export declare const seq = "[_elt_seq]";
8
+ export declare const op = "[_elt_op]";
9
+ export declare const version = "[_elt_row_hash]";
10
+ export declare const keyHash = "[_elt_key_hash]";
11
+ export declare abstract class SqlServerWriter extends Writer {
12
+ #private;
13
+ readonly configuration: CopyConfiguration;
14
+ readonly table: SqlServerTable;
15
+ abstract readonly keys: readonly SqlServerColumn[];
16
+ protected readonly history: TargetHistory;
17
+ constructor(configuration: CopyConfiguration, database: SqlServerDatabase, table: SqlServerTable, retention: Temporal.Duration);
18
+ get qualifiedName(): string;
19
+ protected inspect(_connection: SqlServerConnection): Promise<void>;
20
+ protected abstract merge(connection: SqlServerConnection, stage: string, written: Written, into: string, history: TargetHistory | null): Promise<boolean>;
21
+ protected get replaces(): boolean;
22
+ protected kept(_stage: string): string;
23
+ protected encode(record: unknown): EncodedValue[];
24
+ protected deletionRow(_key: Readonly<Record<string, KeyValue>>): EncodedValue[];
25
+ protected get written(): string[];
26
+ protected staged(alias: string): string[];
27
+ clear({ writer, clock, connection: connector }: Clearing, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
28
+ prepare(load: SqlServerLoad, { writer, reloading }: {
29
+ writer: string;
30
+ reloading: boolean;
31
+ }): Promise<Stage>;
32
+ }
33
+ export type Written = {
34
+ readonly loadedAt: string;
35
+ readonly committedIn: number;
36
+ };
37
+ export declare const writtenParameters: ({ loadedAt, committedIn }: Written) => {
38
+ loaded: string;
39
+ committed: string;
40
+ };