@eltjs/elt-sqlserver 0.0.0-stage → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +485 -2
- package/dist/computed-dataset.d.ts +31 -0
- package/dist/computed-dataset.js +380 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +10 -0
- package/dist/sql-server-append-writer.d.ts +8 -0
- package/dist/sql-server-append-writer.js +13 -0
- package/dist/sql-server-as-of.d.ts +14 -0
- package/dist/sql-server-as-of.js +44 -0
- package/dist/sql-server-checkpoint-store.d.ts +10 -0
- package/dist/sql-server-checkpoint-store.js +101 -0
- package/dist/sql-server-column.d.ts +53 -0
- package/dist/sql-server-column.js +295 -0
- package/dist/sql-server-columns.d.ts +15 -0
- package/dist/sql-server-columns.js +49 -0
- package/dist/sql-server-connection.d.ts +6 -0
- package/dist/sql-server-connection.js +56 -0
- package/dist/sql-server-deduplicating-writer.d.ts +18 -0
- package/dist/sql-server-deduplicating-writer.js +102 -0
- package/dist/sql-server-dependents.d.ts +9 -0
- package/dist/sql-server-dependents.js +54 -0
- package/dist/sql-server-derived-status.d.ts +9 -0
- package/dist/sql-server-derived-status.js +60 -0
- package/dist/sql-server-descriptions.d.ts +6 -0
- package/dist/sql-server-descriptions.js +53 -0
- package/dist/sql-server-destination.d.ts +32 -0
- package/dist/sql-server-destination.js +95 -0
- package/dist/sql-server-failure-type.d.ts +2 -0
- package/dist/sql-server-failure-type.js +25 -0
- package/dist/sql-server-file-store.d.ts +21 -0
- package/dist/sql-server-file-store.js +91 -0
- package/dist/sql-server-fit.d.ts +20 -0
- package/dist/sql-server-fit.js +52 -0
- package/dist/sql-server-history.d.ts +15 -0
- package/dist/sql-server-history.js +148 -0
- package/dist/sql-server-identifiers.d.ts +10 -0
- package/dist/sql-server-identifiers.js +27 -0
- package/dist/sql-server-indexes.d.ts +23 -0
- package/dist/sql-server-indexes.js +121 -0
- package/dist/sql-server-load.d.ts +19 -0
- package/dist/sql-server-load.js +58 -0
- package/dist/sql-server-numbering.d.ts +32 -0
- package/dist/sql-server-numbering.js +255 -0
- package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
- package/dist/sql-server-overwrite-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-writer.js +9 -0
- package/dist/sql-server-parts-table.d.ts +24 -0
- package/dist/sql-server-parts-table.js +167 -0
- package/dist/sql-server-passes.d.ts +6 -0
- package/dist/sql-server-passes.js +71 -0
- package/dist/sql-server-reader-view.d.ts +12 -0
- package/dist/sql-server-reader-view.js +71 -0
- package/dist/sql-server-rule-columns.d.ts +12 -0
- package/dist/sql-server-rule-columns.js +69 -0
- package/dist/sql-server-schemas.d.ts +4 -0
- package/dist/sql-server-schemas.js +8 -0
- package/dist/sql-server-setup-error.d.ts +3 -0
- package/dist/sql-server-setup-error.js +6 -0
- package/dist/sql-server-sync-history-schema.d.ts +7 -0
- package/dist/sql-server-sync-history-schema.js +120 -0
- package/dist/sql-server-sync-history.d.ts +14 -0
- package/dist/sql-server-sync-history.js +165 -0
- package/dist/sql-server-table.d.ts +10 -0
- package/dist/sql-server-table.js +40 -0
- package/dist/sql-server-views.d.ts +7 -0
- package/dist/sql-server-views.js +70 -0
- package/dist/sql-server-writer.d.ts +40 -0
- package/dist/sql-server-writer.js +914 -0
- package/dist/view.d.ts +23 -0
- package/dist/view.js +82 -0
- package/package.json +48 -4
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
import { FileRead, isCalendarDate, isTimestamp, } from '@eltjs/elt';
|
|
2
|
+
import { quote } from './sql-server-identifiers.js';
|
|
3
|
+
// The type of each kind, as SQL Server spells it in sys.columns.
|
|
4
|
+
const storageTypes = {
|
|
5
|
+
text: 'nvarchar(max)',
|
|
6
|
+
integer: 'bigint',
|
|
7
|
+
int64: 'bigint',
|
|
8
|
+
real: 'float',
|
|
9
|
+
decimal: 'nvarchar(max)',
|
|
10
|
+
boolean: 'bit',
|
|
11
|
+
date: 'date',
|
|
12
|
+
timestamp: 'datetimeoffset(7)',
|
|
13
|
+
local_timestamp: 'datetime2(7)',
|
|
14
|
+
local_time: 'time(7)',
|
|
15
|
+
blob: 'varbinary(max)',
|
|
16
|
+
// An object of scalar fields, only as an item of an array column.
|
|
17
|
+
object: 'json',
|
|
18
|
+
};
|
|
19
|
+
// The kind that stores each string format; base64 text loads as its bytes.
|
|
20
|
+
const formatKinds = {
|
|
21
|
+
'date-time': 'timestamp',
|
|
22
|
+
date: 'date',
|
|
23
|
+
'date-time-local': 'local_timestamp',
|
|
24
|
+
'time-local': 'local_time',
|
|
25
|
+
int64: 'int64',
|
|
26
|
+
decimal: 'decimal',
|
|
27
|
+
base64: 'blob',
|
|
28
|
+
};
|
|
29
|
+
const plainKinds = new Set([
|
|
30
|
+
'text',
|
|
31
|
+
'integer',
|
|
32
|
+
'real',
|
|
33
|
+
'blob',
|
|
34
|
+
'boolean',
|
|
35
|
+
'date',
|
|
36
|
+
'timestamp',
|
|
37
|
+
'object',
|
|
38
|
+
]);
|
|
39
|
+
// SQL Server keeps time to 100 nanoseconds, and a decimal to 38 digits.
|
|
40
|
+
const finestPrecision = 7;
|
|
41
|
+
const mostDigits = 38;
|
|
42
|
+
export class SqlServerColumn {
|
|
43
|
+
// The record field the column holds.
|
|
44
|
+
field;
|
|
45
|
+
// What SQL Server calls the column, which the table it belongs to decides.
|
|
46
|
+
name;
|
|
47
|
+
type;
|
|
48
|
+
kind;
|
|
49
|
+
isPrimaryKey;
|
|
50
|
+
nullable;
|
|
51
|
+
optional;
|
|
52
|
+
// An array of kind, stored as a JSON array.
|
|
53
|
+
array;
|
|
54
|
+
fileRead;
|
|
55
|
+
constructor(field, type, options) {
|
|
56
|
+
if (typeof field !== 'string')
|
|
57
|
+
throw new TypeError('A column holds a field named by a string');
|
|
58
|
+
if (typeof type === 'string' && !plainKinds.has(type))
|
|
59
|
+
throw new TypeError('Unsupported SQL Server column type');
|
|
60
|
+
this.type = type;
|
|
61
|
+
const kind = typeof type === 'string' ? type : formatKinds[type.name];
|
|
62
|
+
this.array = options.array ?? false;
|
|
63
|
+
if (this.array && (kind === 'blob' || options.primaryKey))
|
|
64
|
+
throw new TypeError('Array columns hold scalar values or objects and cannot be keys');
|
|
65
|
+
this.fileRead = options.fileRead;
|
|
66
|
+
if (this.fileRead !== undefined &&
|
|
67
|
+
(!(this.fileRead instanceof FileRead) || this.fileRead.name !== field))
|
|
68
|
+
throw new TypeError('Column file read must match its field');
|
|
69
|
+
this.field = field;
|
|
70
|
+
this.name = options.name ?? field;
|
|
71
|
+
this.kind = kind;
|
|
72
|
+
this.isPrimaryKey = options.primaryKey;
|
|
73
|
+
this.nullable = !options.primaryKey && options.nullable;
|
|
74
|
+
this.optional = !options.primaryKey && options.optional;
|
|
75
|
+
Object.freeze(this);
|
|
76
|
+
}
|
|
77
|
+
primaryKey() {
|
|
78
|
+
return new SqlServerColumn(this.field, this.type, {
|
|
79
|
+
nullable: false,
|
|
80
|
+
optional: false,
|
|
81
|
+
primaryKey: true,
|
|
82
|
+
array: this.array,
|
|
83
|
+
fileRead: this.fileRead,
|
|
84
|
+
name: this.name,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
notNull() {
|
|
88
|
+
return new SqlServerColumn(this.field, this.type, {
|
|
89
|
+
nullable: false,
|
|
90
|
+
optional: false,
|
|
91
|
+
primaryKey: this.isPrimaryKey,
|
|
92
|
+
array: this.array,
|
|
93
|
+
fileRead: this.fileRead,
|
|
94
|
+
name: this.name,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
from(file) {
|
|
98
|
+
if (this.array || (this.kind !== 'blob' && this.kind !== 'text'))
|
|
99
|
+
throw new TypeError('Files require a BLOB column, parsed TEXT or a stored TEXT reference');
|
|
100
|
+
return new SqlServerColumn(this.field, this.kind, {
|
|
101
|
+
nullable: this.nullable,
|
|
102
|
+
optional: this.optional,
|
|
103
|
+
primaryKey: this.isPrimaryKey,
|
|
104
|
+
fileRead: new FileRead(this.field, file, this.fileRead?.parser),
|
|
105
|
+
name: this.name,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
parse(parser) {
|
|
109
|
+
if (this.kind !== 'text')
|
|
110
|
+
throw new TypeError('Document parsing requires a TEXT column');
|
|
111
|
+
if (this.fileRead === undefined)
|
|
112
|
+
throw new TypeError('Select a source file before selecting a parser');
|
|
113
|
+
return new SqlServerColumn(this.field, this.kind, {
|
|
114
|
+
nullable: this.nullable,
|
|
115
|
+
optional: this.optional,
|
|
116
|
+
primaryKey: this.isPrimaryKey,
|
|
117
|
+
fileRead: new FileRead(this.field, this.fileRead.file, parser),
|
|
118
|
+
name: this.name,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
// This column as the table names it.
|
|
122
|
+
named(name) {
|
|
123
|
+
return new SqlServerColumn(this.field, this.type, {
|
|
124
|
+
nullable: this.nullable,
|
|
125
|
+
optional: this.optional,
|
|
126
|
+
primaryKey: this.isPrimaryKey,
|
|
127
|
+
array: this.array,
|
|
128
|
+
fileRead: this.fileRead,
|
|
129
|
+
name,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
get storesFile() {
|
|
133
|
+
return this.kind === 'blob' && this.fileRead !== undefined;
|
|
134
|
+
}
|
|
135
|
+
get quotedName() {
|
|
136
|
+
return quote(this.name);
|
|
137
|
+
}
|
|
138
|
+
// A time finer than 100 nanoseconds keeps its exact text, which sorts in
|
|
139
|
+
// time order, since its values share one width.
|
|
140
|
+
get #exactText() {
|
|
141
|
+
const { type } = this;
|
|
142
|
+
return (typeof type !== 'string' &&
|
|
143
|
+
(type.name === 'date-time' ||
|
|
144
|
+
type.name === 'date-time-local' ||
|
|
145
|
+
type.name === 'time-local') &&
|
|
146
|
+
type.precision > finestPrecision);
|
|
147
|
+
}
|
|
148
|
+
// The type of the column, with the precision and scale that its format
|
|
149
|
+
// declares, as sys.columns spells it.
|
|
150
|
+
get storageType() {
|
|
151
|
+
if (this.storesFile)
|
|
152
|
+
return 'uniqueidentifier';
|
|
153
|
+
if (this.array)
|
|
154
|
+
return 'json';
|
|
155
|
+
if (this.#exactText)
|
|
156
|
+
return 'nvarchar(max)';
|
|
157
|
+
const { type } = this;
|
|
158
|
+
if (typeof type === 'string')
|
|
159
|
+
return storageTypes[this.kind];
|
|
160
|
+
switch (type.name) {
|
|
161
|
+
case 'decimal':
|
|
162
|
+
return type.precision !== undefined && type.precision <= mostDigits
|
|
163
|
+
? `decimal(${type.precision},${type.scale ?? 0})`
|
|
164
|
+
: storageTypes.decimal;
|
|
165
|
+
case 'date-time':
|
|
166
|
+
return `datetimeoffset(${type.precision})`;
|
|
167
|
+
case 'date-time-local':
|
|
168
|
+
return `datetime2(${type.precision})`;
|
|
169
|
+
case 'time-local':
|
|
170
|
+
return `time(${type.precision})`;
|
|
171
|
+
default:
|
|
172
|
+
return storageTypes[this.kind];
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// The type that OPENJSON reads the value as from the JSON batch: the bytes
|
|
176
|
+
// of a blob travel as base64 text, which BASE64_DECODE takes only as
|
|
177
|
+
// varchar (ADR 0011), and an array as the JSON of its items.
|
|
178
|
+
get batchType() {
|
|
179
|
+
if (this.array)
|
|
180
|
+
return 'nvarchar(max)';
|
|
181
|
+
if (this.kind === 'blob' && !this.storesFile)
|
|
182
|
+
return 'varchar(max)';
|
|
183
|
+
return this.storageType;
|
|
184
|
+
}
|
|
185
|
+
// The value that the batch carries in its column alias, as the column
|
|
186
|
+
// stores it.
|
|
187
|
+
stored(alias) {
|
|
188
|
+
if (this.array)
|
|
189
|
+
return `CAST(${alias} AS json)`;
|
|
190
|
+
if (this.kind === 'blob' && !this.storesFile)
|
|
191
|
+
return `BASE64_DECODE(${alias})`;
|
|
192
|
+
return alias;
|
|
193
|
+
}
|
|
194
|
+
// A field the stream requires stays nullable in storage: rows loaded before
|
|
195
|
+
// it was added have no value for it. Records are still checked for it when
|
|
196
|
+
// they are written (encode).
|
|
197
|
+
get definition() {
|
|
198
|
+
const check = this.array
|
|
199
|
+
? ` CHECK (ISJSON(${this.quotedName}, ARRAY) = 1)`
|
|
200
|
+
: '';
|
|
201
|
+
return `${this.quotedName} ${this.storageType} ${this.isPrimaryKey ? 'NOT NULL' : 'NULL'}${check}`;
|
|
202
|
+
}
|
|
203
|
+
encode(record) {
|
|
204
|
+
if (record === null || typeof record !== 'object' || Array.isArray(record))
|
|
205
|
+
throw new TypeError(`Record is missing field ${JSON.stringify(this.field)}`);
|
|
206
|
+
if (!Object.hasOwn(record, this.field)) {
|
|
207
|
+
if (this.optional)
|
|
208
|
+
return null;
|
|
209
|
+
throw new TypeError(`Record is missing field ${JSON.stringify(this.field)}`);
|
|
210
|
+
}
|
|
211
|
+
const value = Reflect.get(record, this.field);
|
|
212
|
+
if (value === null && this.nullable)
|
|
213
|
+
return null;
|
|
214
|
+
if (this.array) {
|
|
215
|
+
if (Array.isArray(value)) {
|
|
216
|
+
const elements = value.map((element) => this.#element(element));
|
|
217
|
+
if (elements.every((element) => element !== undefined))
|
|
218
|
+
return elements;
|
|
219
|
+
}
|
|
220
|
+
throw new TypeError(`Field ${JSON.stringify(this.field)} requires an array of ${this.kind}${this.nullable ? ' or null' : ' (not null)'}`);
|
|
221
|
+
}
|
|
222
|
+
const encoded = this.#scalar(value);
|
|
223
|
+
if (encoded !== undefined)
|
|
224
|
+
return encoded;
|
|
225
|
+
throw new TypeError(`Field ${JSON.stringify(this.field)} requires ${this.kind}${this.nullable ? ' or null' : ' (not null)'}`);
|
|
226
|
+
}
|
|
227
|
+
// One element of an array in its batch encoding: an object item as its
|
|
228
|
+
// fields, any other as its scalar; undefined when the kind rejects it.
|
|
229
|
+
#element(value) {
|
|
230
|
+
if (this.kind !== 'object')
|
|
231
|
+
return this.#scalar(value);
|
|
232
|
+
return value !== null &&
|
|
233
|
+
typeof value === 'object' &&
|
|
234
|
+
!Array.isArray(value) &&
|
|
235
|
+
Object.values(value).every((field) => field === null ||
|
|
236
|
+
typeof field === 'string' ||
|
|
237
|
+
typeof field === 'boolean' ||
|
|
238
|
+
(typeof field === 'number' && Number.isFinite(field)))
|
|
239
|
+
? Object.fromEntries(Object.entries(value))
|
|
240
|
+
: undefined;
|
|
241
|
+
}
|
|
242
|
+
// One scalar in its batch encoding, or undefined when the kind rejects it.
|
|
243
|
+
#scalar(value) {
|
|
244
|
+
const { type } = this;
|
|
245
|
+
if (typeof type !== 'string') {
|
|
246
|
+
if (typeof value !== 'string' || !type.accepts(value))
|
|
247
|
+
return undefined;
|
|
248
|
+
return type.name === 'base64'
|
|
249
|
+
? Buffer.from(value, 'base64').toBase64()
|
|
250
|
+
: value;
|
|
251
|
+
}
|
|
252
|
+
switch (type) {
|
|
253
|
+
case 'text':
|
|
254
|
+
if (typeof value === 'string')
|
|
255
|
+
return value;
|
|
256
|
+
break;
|
|
257
|
+
case 'boolean':
|
|
258
|
+
if (typeof value === 'boolean')
|
|
259
|
+
return value;
|
|
260
|
+
break;
|
|
261
|
+
case 'integer':
|
|
262
|
+
if (typeof value === 'bigint' &&
|
|
263
|
+
value >= -(2n ** 63n) &&
|
|
264
|
+
value < 2n ** 63n)
|
|
265
|
+
return String(value);
|
|
266
|
+
if (typeof value === 'number' && Number.isSafeInteger(value))
|
|
267
|
+
return value;
|
|
268
|
+
break;
|
|
269
|
+
case 'real':
|
|
270
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
271
|
+
return value;
|
|
272
|
+
break;
|
|
273
|
+
case 'date':
|
|
274
|
+
if (isCalendarDate(value))
|
|
275
|
+
return value;
|
|
276
|
+
break;
|
|
277
|
+
case 'timestamp':
|
|
278
|
+
if (isTimestamp(value))
|
|
279
|
+
return value;
|
|
280
|
+
break;
|
|
281
|
+
case 'blob':
|
|
282
|
+
if (this.storesFile) {
|
|
283
|
+
if (typeof value === 'string' &&
|
|
284
|
+
/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(value))
|
|
285
|
+
return value;
|
|
286
|
+
}
|
|
287
|
+
else if (value instanceof Uint8Array)
|
|
288
|
+
return value.toBase64();
|
|
289
|
+
break;
|
|
290
|
+
// An object is only an array's item, which #element encodes.
|
|
291
|
+
case 'object':
|
|
292
|
+
}
|
|
293
|
+
return undefined;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type Stream } from '@eltjs/elt';
|
|
2
|
+
import { SqlServerColumn } from './sql-server-column.ts';
|
|
3
|
+
export declare class SqlServerColumns {
|
|
4
|
+
#private;
|
|
5
|
+
static fromSchema(schema: Stream['jsonSchema']): readonly SqlServerColumn[];
|
|
6
|
+
text(field: string): SqlServerColumn;
|
|
7
|
+
integer(field: string): SqlServerColumn;
|
|
8
|
+
int64(field: string): SqlServerColumn;
|
|
9
|
+
real(field: string): SqlServerColumn;
|
|
10
|
+
boolean(field: string): SqlServerColumn;
|
|
11
|
+
date(field: string): SqlServerColumn;
|
|
12
|
+
timestamp(field: string): SqlServerColumn;
|
|
13
|
+
localTimestamp(field: string, precision: number): SqlServerColumn;
|
|
14
|
+
blob(field: string): SqlServerColumn;
|
|
15
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { Int64Format, LocalTimestampFormat, schemaColumns, } from '@eltjs/elt';
|
|
2
|
+
import { SqlServerColumn, } from './sql-server-column.js';
|
|
3
|
+
// SQL Server column declarations are independent of a source or connection.
|
|
4
|
+
export class SqlServerColumns {
|
|
5
|
+
// Scalars and arrays of scalars, as in elt-postgresql: the string formats
|
|
6
|
+
// get their own types, so readers can do date arithmetic without parsing
|
|
7
|
+
// text, and an array is a JSON array of its items.
|
|
8
|
+
static fromSchema(schema) {
|
|
9
|
+
return schemaColumns(schema, 'SQL Server').map(({ field, type, ...options }) => new SqlServerColumn(field, type, { ...options, primaryKey: false }));
|
|
10
|
+
}
|
|
11
|
+
text(field) {
|
|
12
|
+
return this.#column(field, 'text');
|
|
13
|
+
}
|
|
14
|
+
integer(field) {
|
|
15
|
+
return this.#column(field, 'integer');
|
|
16
|
+
}
|
|
17
|
+
// A signed 64-bit integer, which a stream spells as decimal text (format
|
|
18
|
+
// int64), so a value past Number.MAX_SAFE_INTEGER keeps every digit.
|
|
19
|
+
int64(field) {
|
|
20
|
+
return this.#column(field, new Int64Format());
|
|
21
|
+
}
|
|
22
|
+
real(field) {
|
|
23
|
+
return this.#column(field, 'real');
|
|
24
|
+
}
|
|
25
|
+
boolean(field) {
|
|
26
|
+
return this.#column(field, 'boolean');
|
|
27
|
+
}
|
|
28
|
+
date(field) {
|
|
29
|
+
return this.#column(field, 'date');
|
|
30
|
+
}
|
|
31
|
+
timestamp(field) {
|
|
32
|
+
return this.#column(field, 'timestamp');
|
|
33
|
+
}
|
|
34
|
+
// A wall-clock date and time with no zone and precision digits of a second
|
|
35
|
+
// (format date-time-local), as SQL Server's datetime2 holds it.
|
|
36
|
+
localTimestamp(field, precision) {
|
|
37
|
+
return this.#column(field, new LocalTimestampFormat(precision));
|
|
38
|
+
}
|
|
39
|
+
blob(field) {
|
|
40
|
+
return this.#column(field, 'blob');
|
|
41
|
+
}
|
|
42
|
+
#column(field, type) {
|
|
43
|
+
return new SqlServerColumn(field, type, {
|
|
44
|
+
nullable: true,
|
|
45
|
+
optional: false,
|
|
46
|
+
primaryKey: false,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { SqlServerConnection, SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
|
|
2
|
+
export type SqlServerLogin = ConstructorParameters<typeof SqlServerDatabase>[0];
|
|
3
|
+
export declare function connect(database: SqlServerDatabase): Promise<SqlServerConnection>;
|
|
4
|
+
export declare function transaction<T>(connection: SqlServerConnection, work: () => Promise<T>): Promise<T>;
|
|
5
|
+
export declare function takeLock(connection: SqlServerConnection, name: string): Promise<void>;
|
|
6
|
+
export declare function makeSchema(connection: SqlServerConnection, schema: string): Promise<void>;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { quote } from './sql-server-identifiers.js';
|
|
2
|
+
// One session of the database for the destination: a statement that fails
|
|
3
|
+
// inside a transaction rolls the whole transaction back, so no commit keeps
|
|
4
|
+
// a part of its writes.
|
|
5
|
+
export async function connect(database) {
|
|
6
|
+
const connection = await database.connect();
|
|
7
|
+
try {
|
|
8
|
+
await connection.query('SET XACT_ABORT ON');
|
|
9
|
+
return connection;
|
|
10
|
+
}
|
|
11
|
+
catch (error) {
|
|
12
|
+
await connection[Symbol.asyncDispose]();
|
|
13
|
+
throw error;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
// What work does becomes durable and visible together, or not at all.
|
|
17
|
+
// BEGIN and COMMIT are batches of their own, with no parameters, as
|
|
18
|
+
// sp_executesql must end with the transaction count it began with.
|
|
19
|
+
export async function transaction(connection, work) {
|
|
20
|
+
await connection.query('BEGIN TRANSACTION');
|
|
21
|
+
try {
|
|
22
|
+
const result = await work();
|
|
23
|
+
await connection.query('COMMIT');
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
try {
|
|
28
|
+
await connection.query('IF @@TRANCOUNT > 0 ROLLBACK');
|
|
29
|
+
}
|
|
30
|
+
catch (rollback) {
|
|
31
|
+
throw new SuppressedError(rollback, error, 'The transaction failed, and its rollback failed too');
|
|
32
|
+
}
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// Holds the application lock elt:<name> until the transaction ends, so the
|
|
37
|
+
// transactions that take the lock of one name run one at a time (ADR 0009).
|
|
38
|
+
// The DDL that makes a schema and the objects its loads, views and
|
|
39
|
+
// checkpoints share takes the lock of the schema: SQL Server refuses a second
|
|
40
|
+
// session's CREATE SCHEMA or CREATE TABLE of the same name.
|
|
41
|
+
export async function takeLock(connection, name) {
|
|
42
|
+
await connection.query(`DECLARE @result int;
|
|
43
|
+
EXEC @result = sp_getapplock @Resource = @resource, @LockMode = N'Exclusive', @LockOwner = N'Transaction', @LockTimeout = -1;
|
|
44
|
+
IF @result < 0 THROW 50000, N'The lock was not granted', 1;`, { resource: `elt:${name}` });
|
|
45
|
+
}
|
|
46
|
+
// Makes the schema in a transaction of its own under its lock, when the
|
|
47
|
+
// database has none. connection is outside any transaction.
|
|
48
|
+
export async function makeSchema(connection, schema) {
|
|
49
|
+
const [made] = await connection.query('SELECT CASE WHEN SCHEMA_ID(@schema) IS NULL THEN 0 ELSE 1 END AS made', { schema });
|
|
50
|
+
if (made?.made === 1)
|
|
51
|
+
return;
|
|
52
|
+
await transaction(connection, async () => {
|
|
53
|
+
await takeLock(connection, schema);
|
|
54
|
+
await connection.query(`IF SCHEMA_ID(@schema) IS NULL EXEC (N'CREATE SCHEMA ${quote(schema).replaceAll("'", "''")}')`, { schema });
|
|
55
|
+
});
|
|
56
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { type CopyConfiguration, type Deduplication, type KeyValue } from '@eltjs/elt';
|
|
2
|
+
import type { SqlServerConnection, SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
|
|
3
|
+
import type { EncodedValue, SqlServerColumn } from './sql-server-column.ts';
|
|
4
|
+
import type { TargetHistory } from './sql-server-history.ts';
|
|
5
|
+
import type { SqlServerTable } from './sql-server-table.ts';
|
|
6
|
+
import { SqlServerWriter, type Written } from './sql-server-writer.ts';
|
|
7
|
+
export declare class SqlServerDeduplicatingWriter extends SqlServerWriter {
|
|
8
|
+
#private;
|
|
9
|
+
readonly deduplication: Deduplication;
|
|
10
|
+
readonly keys: readonly SqlServerColumn[];
|
|
11
|
+
readonly cursor?: SqlServerColumn;
|
|
12
|
+
constructor(configuration: CopyConfiguration, database: SqlServerDatabase, table: SqlServerTable, retention: Temporal.Duration);
|
|
13
|
+
protected inspect(connection: SqlServerConnection): Promise<void>;
|
|
14
|
+
protected kept(stage: string): string;
|
|
15
|
+
protected merge(connection: SqlServerConnection, stage: string, written: Written, into: string, history: TargetHistory | null): Promise<boolean>;
|
|
16
|
+
protected encode(record: unknown): EncodedValue[];
|
|
17
|
+
protected deletionRow(key: Readonly<Record<string, KeyValue>>): EncodedValue[];
|
|
18
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { deduplicationColumns, } from '@eltjs/elt';
|
|
2
|
+
import { SqlServerColumns } from './sql-server-columns.js';
|
|
3
|
+
import { SqlServerWriter, keyHash, op, seq, version, writtenParameters, } from './sql-server-writer.js';
|
|
4
|
+
// Keeps one row for each key of a stream: a record replaces the row of its
|
|
5
|
+
// key, and a deletion removes it, as if the operations of a commit applied one
|
|
6
|
+
// at a time. Rows match by the key hash, so two keys that differ only in case
|
|
7
|
+
// stay two rows (ADR 0004).
|
|
8
|
+
export class SqlServerDeduplicatingWriter extends SqlServerWriter {
|
|
9
|
+
deduplication;
|
|
10
|
+
keys;
|
|
11
|
+
cursor;
|
|
12
|
+
#guard;
|
|
13
|
+
constructor(configuration, database, table, retention) {
|
|
14
|
+
super(configuration, database, table, retention);
|
|
15
|
+
({
|
|
16
|
+
deduplication: this.deduplication,
|
|
17
|
+
keys: this.keys,
|
|
18
|
+
cursor: this.cursor,
|
|
19
|
+
guard: this.#guard,
|
|
20
|
+
} = deduplicationColumns(configuration, table.columns, SqlServerColumns.fromSchema(configuration.stream.jsonSchema), (column) => column.storageType));
|
|
21
|
+
}
|
|
22
|
+
async inspect(connection) {
|
|
23
|
+
const tracked = this.cursor === undefined ? this.keys : [...this.keys, this.cursor];
|
|
24
|
+
const nulls = await connection.query(`SELECT TOP (1) 1 AS [null] FROM ${this.qualifiedName} WHERE ${tracked.map((column) => `${column.quotedName} IS NULL`).join(' OR ')}`);
|
|
25
|
+
if (nulls.length > 0)
|
|
26
|
+
throw new TypeError('Existing deduplication keys and cursors must be non-null');
|
|
27
|
+
}
|
|
28
|
+
// The guard's value of the row alias, in the order the guard compares: a
|
|
29
|
+
// text cursor by its UTF-8 bytes, as COLLATE "C" in Postgres (ADR 0005).
|
|
30
|
+
#ordered(guard, alias) {
|
|
31
|
+
const value = `${alias}.${guard.quotedName}`;
|
|
32
|
+
return guard.storageType === 'nvarchar(max)'
|
|
33
|
+
? `CONVERT(varchar(max), ${value} COLLATE Latin1_General_100_BIN2_UTF8)`
|
|
34
|
+
: value;
|
|
35
|
+
}
|
|
36
|
+
// The stage as one operation for each key: the operations after the key's
|
|
37
|
+
// last deletion, or that deletion when no record follows it. replace keeps
|
|
38
|
+
// the newest extraction, so a restated fact overwrites the loaded one;
|
|
39
|
+
// cursor_newer keeps the greatest cursor, the first on ties. _elt_last is
|
|
40
|
+
// the place of the deletion that a kept record follows.
|
|
41
|
+
#reduced(stage) {
|
|
42
|
+
const guard = this.#guard;
|
|
43
|
+
const order = guard === undefined
|
|
44
|
+
? `[_elt_staged].${seq} DESC`
|
|
45
|
+
: `${this.#ordered(guard, '[_elt_staged]')} DESC, [_elt_staged].${seq}`;
|
|
46
|
+
return `SELECT [_elt_ranked].* FROM (
|
|
47
|
+
SELECT [_elt_staged].*, [_elt_deleted].[_elt_last], ROW_NUMBER() OVER (PARTITION BY [_elt_staged].${keyHash} ORDER BY IIF([_elt_staged].${op} = 'R', 0, 1), ${order}) AS [_elt_rank]
|
|
48
|
+
FROM ${stage} AS [_elt_staged]
|
|
49
|
+
LEFT JOIN (SELECT ${keyHash}, MAX(${seq}) AS [_elt_last] FROM ${stage} WHERE ${op} = 'D' GROUP BY ${keyHash}) AS [_elt_deleted] ON [_elt_deleted].${keyHash} = [_elt_staged].${keyHash}
|
|
50
|
+
WHERE [_elt_deleted].[_elt_last] IS NULL OR [_elt_staged].${seq} >= [_elt_deleted].[_elt_last]
|
|
51
|
+
) AS [_elt_ranked] WHERE [_elt_ranked].[_elt_rank] = 1`;
|
|
52
|
+
}
|
|
53
|
+
// The records that the reduced stage keeps, when a commit replaces the
|
|
54
|
+
// target.
|
|
55
|
+
kept(stage) {
|
|
56
|
+
return `${seq} IN (SELECT [_elt_reduced].${seq} FROM (${this.#reduced(stage)}) AS [_elt_reduced] WHERE [_elt_reduced].${op} = 'R')`;
|
|
57
|
+
}
|
|
58
|
+
// One MERGE on the key hash, whose output the history keeps (ADR 0005): a
|
|
59
|
+
// deletion removes its row; a record replaces the row when a deletion came
|
|
60
|
+
// before it, or when its values differ and the guard, if any, lets it in,
|
|
61
|
+
// which rejects out-of-order replay; and a record of a new key enters. A
|
|
62
|
+
// record with the stored row's values leaves the row, its commit and its
|
|
63
|
+
// loaded_at as they are. A hidden target has no history: no reader saw its
|
|
64
|
+
// rows.
|
|
65
|
+
async merge(connection, stage, written, into, history) {
|
|
66
|
+
const guard = this.#guard;
|
|
67
|
+
const guarded = guard === undefined
|
|
68
|
+
? ''
|
|
69
|
+
: ` AND ${this.#ordered(guard, '[_elt_new]')} > ${this.#ordered(guard, '[_elt_target]')}`;
|
|
70
|
+
const staged = this.staged('[_elt_new]');
|
|
71
|
+
const parameters = writtenParameters(written);
|
|
72
|
+
const merge = `MERGE ${into} AS [_elt_target]
|
|
73
|
+
USING (${this.#reduced(stage)}) AS [_elt_new] ON [_elt_target].${keyHash} = [_elt_new].${keyHash}
|
|
74
|
+
WHEN MATCHED AND [_elt_new].${op} = 'D' THEN DELETE
|
|
75
|
+
WHEN MATCHED AND ([_elt_new].[_elt_last] IS NOT NULL OR ([_elt_new].${version} IS DISTINCT FROM [_elt_target].${version}${guarded})) THEN
|
|
76
|
+
UPDATE SET ${this.written.flatMap((column, index) => (column === keyHash ? [] : [`${column} = ${staged[index]}`])).join(', ')}
|
|
77
|
+
WHEN NOT MATCHED BY TARGET AND [_elt_new].${op} = 'R' THEN
|
|
78
|
+
INSERT (${this.written.join(', ')}) VALUES (${staged.join(', ')})`;
|
|
79
|
+
if (history === null) {
|
|
80
|
+
const [merged] = await connection.query(`${merge};
|
|
81
|
+
SELECT @@ROWCOUNT AS [rows];`, parameters);
|
|
82
|
+
return merged?.rows !== 0;
|
|
83
|
+
}
|
|
84
|
+
const kept = await history.keep(connection, written.committedIn, into, `${merge}
|
|
85
|
+
OUTPUT deleted.*`, parameters);
|
|
86
|
+
if (kept > 0)
|
|
87
|
+
return true;
|
|
88
|
+
// A commit that kept no row changed one only if it inserted a key.
|
|
89
|
+
const [inserted] = await connection.query(`SELECT IIF(EXISTS (SELECT 1 FROM ${stage} AS [_elt_staged] JOIN ${into} AS [_elt_target] ON [_elt_target].${keyHash} = [_elt_staged].${keyHash} WHERE [_elt_target].[committed_in] = CAST(@committed AS bigint)), 1, 0) AS [inserted]`, parameters);
|
|
90
|
+
return inserted?.inserted === 1;
|
|
91
|
+
}
|
|
92
|
+
encode(record) {
|
|
93
|
+
this.deduplication.key(record);
|
|
94
|
+
if (this.cursor !== undefined)
|
|
95
|
+
this.deduplication.cursor(record);
|
|
96
|
+
return super.encode(record);
|
|
97
|
+
}
|
|
98
|
+
deletionRow(key) {
|
|
99
|
+
this.deduplication.key(key);
|
|
100
|
+
return this.table.columns.map((column) => this.keys.includes(column) ? column.encode(key) : null);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
|
|
2
|
+
export declare const numberOf: (error: unknown) => unknown;
|
|
3
|
+
export declare function refreshDependents(connection: SqlServerConnection, { roots, views, changed }: Dependents, change: () => Promise<void>): Promise<void>;
|
|
4
|
+
type Dependents = {
|
|
5
|
+
readonly roots: readonly string[];
|
|
6
|
+
readonly views: readonly string[];
|
|
7
|
+
readonly changed: string;
|
|
8
|
+
};
|
|
9
|
+
export {};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// The errors of a change that a view WITH SCHEMABINDING stops: a rename of a
|
|
2
|
+
// table it reads (15336), a drop of one (3729), and a change or drop of a
|
|
3
|
+
// column it reads (4922) (ADR 0006).
|
|
4
|
+
const boundNumbers = new Set([15336, 3729, 4922]);
|
|
5
|
+
// The error of sp_refreshview for a view that reads a column that the change
|
|
6
|
+
// dropped.
|
|
7
|
+
const invalidColumn = 207;
|
|
8
|
+
// The number of a SQL Server error, which tedious gives each one.
|
|
9
|
+
export const numberOf = (error) => Error.isError(error) && 'number' in error ? error.number : undefined;
|
|
10
|
+
// Runs change, which replaces or changes the roots, and then refreshes each
|
|
11
|
+
// view that reads one of them, directly or through other views, the
|
|
12
|
+
// shallowest first, so a view over * shows the new columns (ADR 0006).
|
|
13
|
+
// sql_expression_dependencies keeps a dependency by name, so it finds the
|
|
14
|
+
// views of a table that took the name of another. A view that the change no
|
|
15
|
+
// longer fits, or one WITH SCHEMABINDING that stops it, fails the change with
|
|
16
|
+
// an error that names the view; the caller's transaction then keeps the
|
|
17
|
+
// tables as they were. Each root names a relation of the database; views
|
|
18
|
+
// skips the ones that change makes again itself.
|
|
19
|
+
export async function refreshDependents(connection, { roots, views, changed }, change) {
|
|
20
|
+
try {
|
|
21
|
+
await change();
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
if (!boundNumbers.has(Number(numberOf(error))))
|
|
25
|
+
throw error;
|
|
26
|
+
const bound = await connection.query(`SELECT DISTINCT QUOTENAME(OBJECT_SCHEMA_NAME(d.[referencing_id])) + N'.' + QUOTENAME(OBJECT_NAME(d.[referencing_id])) AS [view]
|
|
27
|
+
FROM sys.sql_expression_dependencies AS d
|
|
28
|
+
WHERE d.[is_schema_bound_reference] = 1 AND d.[referenced_id] IN (SELECT OBJECT_ID([value]) FROM OPENJSON(@roots))
|
|
29
|
+
ORDER BY [view]`, { roots: JSON.stringify(roots) });
|
|
30
|
+
throw new TypeError(`${bound.map(({ view }) => String(view)).join(', ')} WITH SCHEMABINDING stops the change of ${changed}; drop the view or make it again without SCHEMABINDING: ${Error.isError(error) ? error.message : String(error)}`, { cause: error });
|
|
31
|
+
}
|
|
32
|
+
const readers = await connection.query(`WITH [readers] AS (
|
|
33
|
+
SELECT d.[referencing_id] AS [id], 1 AS [depth] FROM sys.sql_expression_dependencies AS d
|
|
34
|
+
WHERE d.[referenced_id] IN (SELECT OBJECT_ID([value]) FROM OPENJSON(@roots))
|
|
35
|
+
UNION ALL
|
|
36
|
+
SELECT d.[referencing_id], r.[depth] + 1 FROM [readers] AS r
|
|
37
|
+
JOIN sys.sql_expression_dependencies AS d ON d.[referenced_id] = r.[id]
|
|
38
|
+
)
|
|
39
|
+
SELECT QUOTENAME(OBJECT_SCHEMA_NAME([id])) + N'.' + QUOTENAME(OBJECT_NAME([id])) AS [view]
|
|
40
|
+
FROM [readers]
|
|
41
|
+
WHERE OBJECTPROPERTY([id], 'IsView') = 1 AND [id] NOT IN (SELECT OBJECT_ID([value]) FROM OPENJSON(@views))
|
|
42
|
+
GROUP BY [id] ORDER BY MAX([depth]), [view]`, { roots: JSON.stringify(roots), views: JSON.stringify(views) });
|
|
43
|
+
for (const { view } of readers)
|
|
44
|
+
try {
|
|
45
|
+
await connection.query('EXEC sp_refreshview @view', {
|
|
46
|
+
view: String(view),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
if (numberOf(error) !== invalidColumn)
|
|
51
|
+
throw error;
|
|
52
|
+
throw new TypeError(`${String(view)} reads ${changed} and does not fit its change: ${Error.isError(error) ? error.message : String(error)}`, { cause: error });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { DerivedStatus, type PipelineClock } from '@eltjs/elt';
|
|
2
|
+
import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
|
|
3
|
+
export declare const derivedStatusTable: string;
|
|
4
|
+
export declare class SqlServerDerivedStatus extends DerivedStatus {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(connection: SqlServerConnection, dataset: string, kind: 'view' | 'computed', clock: PipelineClock);
|
|
7
|
+
settled(count: number): Promise<void>;
|
|
8
|
+
protected write(values: Readonly<Record<string, string | number | null>>): Promise<void>;
|
|
9
|
+
}
|