@eltjs/elt-sqlserver 0.0.0-stage → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +487 -2
- package/dist/computed-dataset.d.ts +31 -0
- package/dist/computed-dataset.js +377 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +10 -0
- package/dist/sql-server-append-writer.d.ts +8 -0
- package/dist/sql-server-append-writer.js +13 -0
- package/dist/sql-server-as-of.d.ts +14 -0
- package/dist/sql-server-as-of.js +49 -0
- package/dist/sql-server-checkpoint-store.d.ts +10 -0
- package/dist/sql-server-checkpoint-store.js +101 -0
- package/dist/sql-server-column.d.ts +53 -0
- package/dist/sql-server-column.js +295 -0
- package/dist/sql-server-columns.d.ts +15 -0
- package/dist/sql-server-columns.js +49 -0
- package/dist/sql-server-connection.d.ts +6 -0
- package/dist/sql-server-connection.js +56 -0
- package/dist/sql-server-deduplicating-writer.d.ts +18 -0
- package/dist/sql-server-deduplicating-writer.js +102 -0
- package/dist/sql-server-dependents.d.ts +9 -0
- package/dist/sql-server-dependents.js +54 -0
- package/dist/sql-server-derived-status.d.ts +9 -0
- package/dist/sql-server-derived-status.js +60 -0
- package/dist/sql-server-descriptions.d.ts +6 -0
- package/dist/sql-server-descriptions.js +53 -0
- package/dist/sql-server-destination.d.ts +32 -0
- package/dist/sql-server-destination.js +95 -0
- package/dist/sql-server-failure-type.d.ts +2 -0
- package/dist/sql-server-failure-type.js +25 -0
- package/dist/sql-server-file-store.d.ts +21 -0
- package/dist/sql-server-file-store.js +91 -0
- package/dist/sql-server-fit.d.ts +20 -0
- package/dist/sql-server-fit.js +52 -0
- package/dist/sql-server-history.d.ts +15 -0
- package/dist/sql-server-history.js +148 -0
- package/dist/sql-server-identifiers.d.ts +10 -0
- package/dist/sql-server-identifiers.js +27 -0
- package/dist/sql-server-indexes.d.ts +20 -0
- package/dist/sql-server-indexes.js +68 -0
- package/dist/sql-server-load.d.ts +19 -0
- package/dist/sql-server-load.js +58 -0
- package/dist/sql-server-numbering.d.ts +32 -0
- package/dist/sql-server-numbering.js +255 -0
- package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
- package/dist/sql-server-overwrite-writer.d.ts +4 -0
- package/dist/sql-server-overwrite-writer.js +9 -0
- package/dist/sql-server-parts-table.d.ts +24 -0
- package/dist/sql-server-parts-table.js +165 -0
- package/dist/sql-server-passes.d.ts +6 -0
- package/dist/sql-server-passes.js +71 -0
- package/dist/sql-server-reader-view.d.ts +12 -0
- package/dist/sql-server-reader-view.js +71 -0
- package/dist/sql-server-rule-columns.d.ts +12 -0
- package/dist/sql-server-rule-columns.js +69 -0
- package/dist/sql-server-schemas.d.ts +4 -0
- package/dist/sql-server-schemas.js +8 -0
- package/dist/sql-server-setup-error.d.ts +3 -0
- package/dist/sql-server-setup-error.js +6 -0
- package/dist/sql-server-sync-history-schema.d.ts +7 -0
- package/dist/sql-server-sync-history-schema.js +120 -0
- package/dist/sql-server-sync-history.d.ts +14 -0
- package/dist/sql-server-sync-history.js +165 -0
- package/dist/sql-server-table.d.ts +10 -0
- package/dist/sql-server-table.js +40 -0
- package/dist/sql-server-views.d.ts +7 -0
- package/dist/sql-server-views.js +70 -0
- package/dist/sql-server-writer.d.ts +40 -0
- package/dist/sql-server-writer.js +914 -0
- package/dist/view.d.ts +23 -0
- package/dist/view.js +82 -0
- package/package.json +48 -4
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import { hash } from 'node:crypto';
|
|
2
|
+
import { Lock } from '@zukhruf/async';
|
|
3
|
+
import { DerivedDataset, TableRule, computedColumns, refreshComputed, refuseTakenColumns, reservedColumns, } from '@eltjs/elt';
|
|
4
|
+
import { makeSchema } from './sql-server-connection.js';
|
|
5
|
+
import { refreshDependents } from './sql-server-dependents.js';
|
|
6
|
+
import { SqlServerDerivedStatus } from './sql-server-derived-status.js';
|
|
7
|
+
import { TargetDescriptions } from './sql-server-descriptions.js';
|
|
8
|
+
import { SqlServerDestination, database } from './sql-server-destination.js';
|
|
9
|
+
import { storedColumns } from './sql-server-fit.js';
|
|
10
|
+
import { TargetHistory, rowHash } from './sql-server-history.js';
|
|
11
|
+
import { identifiers, quote, rawTable } from './sql-server-identifiers.js';
|
|
12
|
+
import { TableIndexes } from './sql-server-indexes.js';
|
|
13
|
+
import { SqlServerLoad } from './sql-server-load.js';
|
|
14
|
+
import { expireBefore } from './sql-server-numbering.js';
|
|
15
|
+
import { SqlServerPartsTable } from './sql-server-parts-table.js';
|
|
16
|
+
import { TargetReaderView } from './sql-server-reader-view.js';
|
|
17
|
+
import { batchValue, keyHash, ruleBatchType, ruleColumnType, ruleStored, storedValue, } from './sql-server-rule-columns.js';
|
|
18
|
+
import { inMarts, inWarehouse, marts, warehouse, } from './sql-server-schemas.js';
|
|
19
|
+
import { View, made } from './view.js';
|
|
20
|
+
// The columns of each table of the dataset that the destination writes for
|
|
21
|
+
// itself beside the rule's: the row number, its clustered key (ADR 0003),
|
|
22
|
+
// and the key hash, which is unique (ADR 0004).
|
|
23
|
+
const ownColumns = ['_elt_row', '_elt_key_hash'];
|
|
24
|
+
// Data computed on write from what the pipeline's copies load into one SQL
|
|
25
|
+
// Server database, as in elt-postgresql: a rule makes one output row for each
|
|
26
|
+
// input, such as the vector of each passage. The pipeline refreshes it as its
|
|
27
|
+
// copies commit, through refreshComputed of elt. Readers query the view
|
|
28
|
+
// marts.<name>, over the table raw_<name> in _warehouse, which a refresh
|
|
29
|
+
// makes again when its declaration changes, since it is rebuilt from the
|
|
30
|
+
// inputs. Each transaction of a refresh that changes a row is a numbered
|
|
31
|
+
// commit, each row names the commit that last wrote it, and the history
|
|
32
|
+
// keeps each row that a commit replaced or removed, so the view reads as of a
|
|
33
|
+
// position as the copies' reader views do.
|
|
34
|
+
// A table rule gives many parts for one input: the dataset keeps one output
|
|
35
|
+
// row for each key in raw_<name>, with no rule column, and its parts in
|
|
36
|
+
// raw_<name>_parts, which readers query through marts.<name>_parts. The
|
|
37
|
+
// commit that writes an output row writes its parts, and the two tables are
|
|
38
|
+
// one declaration, so a change to either makes both again.
|
|
39
|
+
// Each table has the indexes that its declaration names: indexes for the raw
|
|
40
|
+
// table, and partIndexes for the parts table of a table rule. Each refresh
|
|
41
|
+
// makes or drops the difference in place, before it writes a row (ADR 0013).
|
|
42
|
+
export class ComputedDataset extends DerivedDataset {
|
|
43
|
+
name;
|
|
44
|
+
// The database the dataset lives in.
|
|
45
|
+
into;
|
|
46
|
+
#from;
|
|
47
|
+
#compute;
|
|
48
|
+
// The rule columns of an output row: none for a table rule, whose columns
|
|
49
|
+
// are the parts'.
|
|
50
|
+
#ruleColumns;
|
|
51
|
+
#output;
|
|
52
|
+
constructor({ into, name, description, from, compute, indexes = [], partIndexes = [], }) {
|
|
53
|
+
super();
|
|
54
|
+
this.name = name;
|
|
55
|
+
this.into = into;
|
|
56
|
+
this.#from = Object.freeze([...from]);
|
|
57
|
+
refuseTakenColumns({
|
|
58
|
+
dataset: name,
|
|
59
|
+
rule: compute,
|
|
60
|
+
reserved: [...reservedColumns(compute), ...ownColumns],
|
|
61
|
+
key: (column) => identifiers.key(column),
|
|
62
|
+
});
|
|
63
|
+
this.#compute =
|
|
64
|
+
compute instanceof TableRule
|
|
65
|
+
? {
|
|
66
|
+
rule: compute,
|
|
67
|
+
parts: new SqlServerPartsTable({
|
|
68
|
+
dataset: name,
|
|
69
|
+
description,
|
|
70
|
+
rule: compute,
|
|
71
|
+
indexes: partIndexes,
|
|
72
|
+
}),
|
|
73
|
+
}
|
|
74
|
+
: { rule: compute };
|
|
75
|
+
this.#ruleColumns = compute instanceof TableRule ? {} : compute.output;
|
|
76
|
+
const table = inWarehouse(rawTable(name));
|
|
77
|
+
const declared = Object.freeze([
|
|
78
|
+
{ name: '_elt_row', type: 'bigint', required: true },
|
|
79
|
+
{ name: compute.key.name, type: 'nvarchar(max)', required: true },
|
|
80
|
+
...Object.keys(computedColumns).map((column) => ({
|
|
81
|
+
name: column,
|
|
82
|
+
type: 'nvarchar(max)',
|
|
83
|
+
required: column !== 'error',
|
|
84
|
+
})),
|
|
85
|
+
...Object.entries(this.#ruleColumns).map(([column, rule]) => ({
|
|
86
|
+
name: column,
|
|
87
|
+
type: ruleColumnType(column, rule),
|
|
88
|
+
required: false,
|
|
89
|
+
})),
|
|
90
|
+
{ name: 'committed_in', type: 'bigint', required: true },
|
|
91
|
+
{ name: '_elt_row_hash', type: 'binary(32)', required: true },
|
|
92
|
+
{ name: '_elt_key_hash', type: 'binary(32)', required: true },
|
|
93
|
+
]);
|
|
94
|
+
const columns = declared
|
|
95
|
+
.slice(1)
|
|
96
|
+
.map(({ name, type, required }) => `${quote(name)} ${type} ${required ? 'NOT NULL' : 'NULL'}`);
|
|
97
|
+
const history = new TargetHistory(hash('sha256', table).slice(0, 40));
|
|
98
|
+
this.#output = Object.freeze({
|
|
99
|
+
table,
|
|
100
|
+
declared,
|
|
101
|
+
statement: `CREATE TABLE ${table} ([_elt_row] bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY CLUSTERED, ${columns.join(', ')}, CHECK ([status] IN (N'computed', N'empty', N'failed')));
|
|
102
|
+
CREATE UNIQUE INDEX [key_hash] ON ${table} ([_elt_key_hash]);`,
|
|
103
|
+
history,
|
|
104
|
+
indexes: new TableIndexes({
|
|
105
|
+
dataset: name,
|
|
106
|
+
table,
|
|
107
|
+
columns: declared,
|
|
108
|
+
indexes,
|
|
109
|
+
}),
|
|
110
|
+
view: new TargetReaderView(name, table, history.qualifiedName, this.#columns, new TargetDescriptions(description, {
|
|
111
|
+
[compute.key.name]: compute.key.description,
|
|
112
|
+
...computedColumns,
|
|
113
|
+
...Object.fromEntries(Object.entries(this.#ruleColumns).map(([column, { description }]) => [column, description])),
|
|
114
|
+
})),
|
|
115
|
+
});
|
|
116
|
+
Object.freeze(this);
|
|
117
|
+
}
|
|
118
|
+
// The view readers query, which a View after the dataset waits for.
|
|
119
|
+
get qualifiedName() {
|
|
120
|
+
return `${marts}.${this.name}`;
|
|
121
|
+
}
|
|
122
|
+
// The key, the engine columns and the rule's, which readers see and the
|
|
123
|
+
// row hash counts.
|
|
124
|
+
get #columns() {
|
|
125
|
+
return [
|
|
126
|
+
this.#compute.rule.key.name,
|
|
127
|
+
...Object.keys(computedColumns),
|
|
128
|
+
...Object.keys(this.#ruleColumns),
|
|
129
|
+
];
|
|
130
|
+
}
|
|
131
|
+
// The tables of the dataset, which one declaration makes.
|
|
132
|
+
get #tables() {
|
|
133
|
+
return 'parts' in this.#compute
|
|
134
|
+
? [this.#output, this.#compute.parts]
|
|
135
|
+
: [this.#output];
|
|
136
|
+
}
|
|
137
|
+
validate(connections) {
|
|
138
|
+
for (const target of this.#from)
|
|
139
|
+
if (!(target instanceof View) && !this.#loads(connections, target))
|
|
140
|
+
throw new TypeError(`Computed dataset ${this.name} reads ${target.name}, which no copy of the pipeline loads into its database`);
|
|
141
|
+
const relations = made(connections, this.into);
|
|
142
|
+
for (const relation of [
|
|
143
|
+
this.qualifiedName,
|
|
144
|
+
`${warehouse}.${rawTable(this.name)}`,
|
|
145
|
+
...('parts' in this.#compute ? this.#compute.parts.names : []),
|
|
146
|
+
])
|
|
147
|
+
if (relations.has(identifiers.key(relation)))
|
|
148
|
+
throw new TypeError(`Computed dataset ${this.name} needs ${relation}, which a copy of the pipeline makes`);
|
|
149
|
+
}
|
|
150
|
+
async refresh({ connections, clock, signal, trigger, }) {
|
|
151
|
+
// Open from the start, so it closes a refresh that a process ended; with
|
|
152
|
+
// no commit it is no span, and leaves the positions as they are.
|
|
153
|
+
await using load = await SqlServerLoad.open(this.into[database], { dataset: this.name }, { clock, retention: this.into.retention });
|
|
154
|
+
const status = new SqlServerDerivedStatus(load.connection, this.qualifiedName, 'computed', clock);
|
|
155
|
+
const refresh = {
|
|
156
|
+
inputs: await this.#inputs(load.connection),
|
|
157
|
+
prepare: (keys) => this.#prepare(load, status, keys, signal),
|
|
158
|
+
status,
|
|
159
|
+
trigger,
|
|
160
|
+
signal,
|
|
161
|
+
};
|
|
162
|
+
const { rule } = this.#compute;
|
|
163
|
+
await (rule instanceof TableRule
|
|
164
|
+
? refreshComputed({ ...refresh, rule })
|
|
165
|
+
: refreshComputed({ ...refresh, rule }));
|
|
166
|
+
}
|
|
167
|
+
// Makes marts, brings the tables to their declaration, removes the rows
|
|
168
|
+
// whose key has no input, with their parts, and gives the output table to
|
|
169
|
+
// write. Each transaction takes the number of its commit at its first
|
|
170
|
+
// change, so one that changes no row takes none. The load has one session,
|
|
171
|
+
// so the transactions of the calls take turns.
|
|
172
|
+
async #prepare(load, status, keys, signal) {
|
|
173
|
+
const { connection } = load;
|
|
174
|
+
const { table, history } = this.#output;
|
|
175
|
+
const parts = 'parts' in this.#compute ? this.#compute.parts : undefined;
|
|
176
|
+
const commits = () => {
|
|
177
|
+
let number;
|
|
178
|
+
return () => (number ??= load.number());
|
|
179
|
+
};
|
|
180
|
+
await makeSchema(connection, marts);
|
|
181
|
+
const stored = await load.transaction(async () => {
|
|
182
|
+
const commit = commits();
|
|
183
|
+
await this.#declare(connection, commit);
|
|
184
|
+
const rows = new Map();
|
|
185
|
+
const gone = [];
|
|
186
|
+
for (const row of await connection.query(`SELECT ${quote(this.#compute.rule.key.name)} AS [key], [input_hash], [rule], [status] FROM ${table}`)) {
|
|
187
|
+
const key = String(row.key);
|
|
188
|
+
if (keys.has(key))
|
|
189
|
+
rows.set(key, {
|
|
190
|
+
inputHash: String(row.input_hash),
|
|
191
|
+
rule: String(row.rule),
|
|
192
|
+
status: String(row.status),
|
|
193
|
+
});
|
|
194
|
+
else
|
|
195
|
+
gone.push(keyHash(key));
|
|
196
|
+
}
|
|
197
|
+
if (gone.length > 0) {
|
|
198
|
+
await history.remove(connection, await commit(), table, '[_elt_key_hash] IN (SELECT CONVERT(binary(32), [value], 2) FROM OPENJSON(@gone))', { gone: JSON.stringify(gone) });
|
|
199
|
+
await parts?.remove(connection, await commit(), gone);
|
|
200
|
+
}
|
|
201
|
+
for (const declared of this.#tables)
|
|
202
|
+
await declared.history.expire(connection);
|
|
203
|
+
return rows;
|
|
204
|
+
});
|
|
205
|
+
for (const { indexes } of this.#tables)
|
|
206
|
+
await indexes.reconcile(connection);
|
|
207
|
+
const session = new Lock();
|
|
208
|
+
const settle = (work) => session.run(() => load.transaction(() => work(commits())), { signal });
|
|
209
|
+
return {
|
|
210
|
+
stored,
|
|
211
|
+
store: (output) => settle((commit) => this.#write(connection, commit, [output])),
|
|
212
|
+
settle: (outputs, settled) => settle(async (commit) => {
|
|
213
|
+
await this.#write(connection, commit, outputs);
|
|
214
|
+
await status.settled(settled);
|
|
215
|
+
}),
|
|
216
|
+
counts: async () => new Map((await connection.query(`SELECT [status], COUNT(*) AS [n] FROM ${table} GROUP BY [status]`)).map(({ status, n }) => [String(status), Number(n)])),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
// Writes each output row whose values differ from its stored row in the
|
|
220
|
+
// commit that commit numbers, in one MERGE whose output the history keeps
|
|
221
|
+
// (ADR 0005), and the parts of each row in the same commit. A row with the
|
|
222
|
+
// values of its stored row keeps its commit.
|
|
223
|
+
async #write(connection, commit, outputs) {
|
|
224
|
+
await this.#merge(connection, commit, outputs);
|
|
225
|
+
if ('parts' in this.#compute)
|
|
226
|
+
await this.#compute.parts.write(connection, commit, outputs);
|
|
227
|
+
}
|
|
228
|
+
// The MERGE of the output rows whose row hash changed.
|
|
229
|
+
async #merge(connection, commit, outputs) {
|
|
230
|
+
const { table, history } = this.#output;
|
|
231
|
+
const ruleColumns = Object.entries(this.#ruleColumns);
|
|
232
|
+
const rows = outputs.map((output) => {
|
|
233
|
+
const values = [
|
|
234
|
+
output.key,
|
|
235
|
+
output.inputHash,
|
|
236
|
+
output.rule,
|
|
237
|
+
output.status,
|
|
238
|
+
output.error,
|
|
239
|
+
...ruleColumns.map(([column]) => storedValue('values' in output ? output.values[column] : undefined)),
|
|
240
|
+
];
|
|
241
|
+
return {
|
|
242
|
+
keyHash: keyHash(output.key),
|
|
243
|
+
version: rowHash(this.#columns.map((name) => ({ name })), values),
|
|
244
|
+
values,
|
|
245
|
+
};
|
|
246
|
+
});
|
|
247
|
+
const before = new Map((await connection.query(`SELECT LOWER(CONVERT(char(64), [_elt_key_hash], 2)) AS [key], LOWER(CONVERT(char(64), [_elt_row_hash], 2)) AS [version]
|
|
248
|
+
FROM ${table} WHERE [_elt_key_hash] IN (SELECT CONVERT(binary(32), [value], 2) FROM OPENJSON(@keys))`, { keys: JSON.stringify(rows.map((row) => row.keyHash)) })).map(({ key, version }) => [String(key), String(version)]));
|
|
249
|
+
const changed = rows.filter((row) => before.get(row.keyHash) !== row.version);
|
|
250
|
+
if (changed.length === 0)
|
|
251
|
+
return;
|
|
252
|
+
// Each column as OPENJSON reads it and as the table stores it.
|
|
253
|
+
const fields = [
|
|
254
|
+
...[this.#compute.rule.key.name, ...Object.keys(computedColumns)].map((name) => ({
|
|
255
|
+
name,
|
|
256
|
+
type: 'nvarchar(max)',
|
|
257
|
+
stored: (at) => at,
|
|
258
|
+
})),
|
|
259
|
+
...ruleColumns.map(([name, column]) => ({
|
|
260
|
+
name,
|
|
261
|
+
type: ruleBatchType(column),
|
|
262
|
+
stored: (at) => ruleStored(name, column, at),
|
|
263
|
+
})),
|
|
264
|
+
];
|
|
265
|
+
const written = [
|
|
266
|
+
...fields.map(({ name }) => name),
|
|
267
|
+
'committed_in',
|
|
268
|
+
'_elt_row_hash',
|
|
269
|
+
'_elt_key_hash',
|
|
270
|
+
].map(quote);
|
|
271
|
+
const source = `SELECT ${[
|
|
272
|
+
...fields.map(({ name, stored }, index) => `${stored(`j.[v${index}]`)} AS ${quote(name)}`),
|
|
273
|
+
'CAST(@committed AS bigint) AS [committed_in]',
|
|
274
|
+
'CONVERT(binary(32), j.[h], 2) AS [_elt_row_hash]',
|
|
275
|
+
'CONVERT(binary(32), j.[k], 2) AS [_elt_key_hash]',
|
|
276
|
+
].join(', ')} FROM OPENJSON(@batch) WITH (${[
|
|
277
|
+
`[k] char(64) '$[0]'`,
|
|
278
|
+
`[h] char(64) '$[1]'`,
|
|
279
|
+
...fields.map(({ type }, index) => `[v${index}] ${type} '$[${index + 2}]'`),
|
|
280
|
+
].join(', ')}) AS j`;
|
|
281
|
+
const committed = await commit();
|
|
282
|
+
await history.keep(connection, committed, table, `MERGE ${table} AS t USING (${source}) AS s ON t.[_elt_key_hash] = s.[_elt_key_hash]
|
|
283
|
+
WHEN MATCHED THEN UPDATE SET ${written.map((column) => `t.${column} = s.${column}`).join(', ')}
|
|
284
|
+
WHEN NOT MATCHED THEN INSERT (${written.join(', ')}) VALUES (${written.map((column) => `s.${column}`).join(', ')})
|
|
285
|
+
OUTPUT deleted.*`, {
|
|
286
|
+
committed: String(committed),
|
|
287
|
+
batch: JSON.stringify(changed.map((row) => [
|
|
288
|
+
row.keyHash,
|
|
289
|
+
row.version,
|
|
290
|
+
...row.values.map(batchValue),
|
|
291
|
+
])),
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
// Makes the tables, their histories and their views, or makes every table
|
|
295
|
+
// again when the stored columns of one differ from its declaration, in a
|
|
296
|
+
// commit: the history of each keeps each row of its old table and takes the
|
|
297
|
+
// new shape, and the views over the old tables read the new ones (ADR
|
|
298
|
+
// 0006). A column whose type changes in a way that a history cannot follow,
|
|
299
|
+
// such as a vector of other dimensions, expires each position before the
|
|
300
|
+
// commit, which only the rows of the old history read.
|
|
301
|
+
async #declare(connection, commit) {
|
|
302
|
+
const tables = this.#tables;
|
|
303
|
+
const stored = [];
|
|
304
|
+
for (const { table } of tables)
|
|
305
|
+
stored.push(await storedColumns(connection, { table }));
|
|
306
|
+
const missing = stored.every((columns) => columns.length === 0);
|
|
307
|
+
const fits = tables.every(({ declared }, index) => stored[index]?.map(shape).join('\0') === declared.map(shape).join('\0'));
|
|
308
|
+
if (missing || fits) {
|
|
309
|
+
for (const { table, statement, history, view } of tables) {
|
|
310
|
+
if (missing)
|
|
311
|
+
await connection.query(statement);
|
|
312
|
+
await history.create(connection, table);
|
|
313
|
+
await view.describe(connection);
|
|
314
|
+
}
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
const committedIn = await commit();
|
|
318
|
+
await refreshDependents(connection, {
|
|
319
|
+
roots: tables.flatMap(({ table, history, view }) => [
|
|
320
|
+
table,
|
|
321
|
+
history.qualifiedName,
|
|
322
|
+
view.qualifiedName,
|
|
323
|
+
]),
|
|
324
|
+
views: tables.map(({ view }) => view.qualifiedName),
|
|
325
|
+
changed: tables.map(({ table }) => table).join(' and '),
|
|
326
|
+
}, async () => {
|
|
327
|
+
for (const [index, { table, history }] of tables.entries()) {
|
|
328
|
+
if (stored[index]?.length === 0)
|
|
329
|
+
continue;
|
|
330
|
+
await history.create(connection, table);
|
|
331
|
+
await history.keep(connection, committedIn, table, `SELECT * FROM ${table}`, {});
|
|
332
|
+
await connection.query(`DROP TABLE ${table}`);
|
|
333
|
+
}
|
|
334
|
+
for (const { table, statement, history, view } of tables) {
|
|
335
|
+
await connection.query(statement);
|
|
336
|
+
if ((await history.follow(connection, table)) === 'retyped')
|
|
337
|
+
await expireBefore(connection, committedIn, 'type');
|
|
338
|
+
await history.create(connection, table);
|
|
339
|
+
await view.replace(connection);
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
// Each input by key, from the rows of each relation of from that exists
|
|
344
|
+
// yet: what a copy into the database loaded, which validate checked,
|
|
345
|
+
// through its reader view, or what a view before this dataset gives. A key read twice keeps its first input.
|
|
346
|
+
// null while the reader view of a copy is gone, such as one a person
|
|
347
|
+
// dropped by hand: its copy makes it again, and the dataset waits with
|
|
348
|
+
// what it made, as a view waits. Read as no rows, it would delete every
|
|
349
|
+
// output, and the next refresh would pay the rule for each again.
|
|
350
|
+
async #inputs(connection) {
|
|
351
|
+
const inputs = new Map();
|
|
352
|
+
for (const target of this.#from) {
|
|
353
|
+
const relation = inMarts(target.name);
|
|
354
|
+
const [exists] = await connection.query('SELECT IIF(OBJECT_ID(@relation) IS NULL, 0, 1) AS [exists]', { relation });
|
|
355
|
+
if (exists?.exists !== 1) {
|
|
356
|
+
if (target instanceof View)
|
|
357
|
+
continue;
|
|
358
|
+
return null;
|
|
359
|
+
}
|
|
360
|
+
for (const row of await connection.query(`SELECT * FROM ${relation}`)) {
|
|
361
|
+
const read = this.#compute.rule.read(row);
|
|
362
|
+
if (read !== null && !inputs.has(read.key))
|
|
363
|
+
inputs.set(read.key, read.input);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
return inputs;
|
|
367
|
+
}
|
|
368
|
+
// Whether a copy of the pipeline loads target into the database of the
|
|
369
|
+
// dataset.
|
|
370
|
+
#loads(connections, target) {
|
|
371
|
+
const { location } = this.into[database];
|
|
372
|
+
return connections.some(({ destination, steps }) => destination instanceof SqlServerDestination &&
|
|
373
|
+
destination[database].location === location &&
|
|
374
|
+
steps.some(({ to }) => to === target));
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
const shape = ({ name, type, required }) => `${name} ${type} ${required}`;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export { ComputedDataset } from './computed-dataset.ts';
|
|
2
|
+
export type { SqlServerReading } from './sql-server-as-of.ts';
|
|
3
|
+
export { SqlServerCheckpointStore } from './sql-server-checkpoint-store.ts';
|
|
4
|
+
export { SqlServerColumn } from './sql-server-column.ts';
|
|
5
|
+
export { SqlServerColumns } from './sql-server-columns.ts';
|
|
6
|
+
export type { SqlServerLogin } from './sql-server-connection.ts';
|
|
7
|
+
export { SqlServerDestination } from './sql-server-destination.ts';
|
|
8
|
+
export type { SqlServerIndex } from './sql-server-indexes.ts';
|
|
9
|
+
export { readPassStatus } from './sql-server-passes.ts';
|
|
10
|
+
export { SqlServerSetupError } from './sql-server-setup-error.ts';
|
|
11
|
+
export { SqlServerSyncHistory } from './sql-server-sync-history.ts';
|
|
12
|
+
export { SqlServerTable } from './sql-server-table.ts';
|
|
13
|
+
export { View } from './view.ts';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { ComputedDataset } from './computed-dataset.js';
|
|
2
|
+
export { SqlServerCheckpointStore } from './sql-server-checkpoint-store.js';
|
|
3
|
+
export { SqlServerColumn } from './sql-server-column.js';
|
|
4
|
+
export { SqlServerColumns } from './sql-server-columns.js';
|
|
5
|
+
export { SqlServerDestination } from './sql-server-destination.js';
|
|
6
|
+
export { readPassStatus } from './sql-server-passes.js';
|
|
7
|
+
export { SqlServerSetupError } from './sql-server-setup-error.js';
|
|
8
|
+
export { SqlServerSyncHistory } from './sql-server-sync-history.js';
|
|
9
|
+
export { SqlServerTable } from './sql-server-table.js';
|
|
10
|
+
export { View } from './view.js';
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { SqlServerConnection } from '@eltjs/sdk-microsoft-sql-server';
|
|
2
|
+
import type { SqlServerColumn } from './sql-server-column.ts';
|
|
3
|
+
import type { TargetHistory } from './sql-server-history.ts';
|
|
4
|
+
import { SqlServerWriter, type Written } from './sql-server-writer.ts';
|
|
5
|
+
export declare class SqlServerAppendWriter extends SqlServerWriter {
|
|
6
|
+
readonly keys: readonly SqlServerColumn[];
|
|
7
|
+
protected merge(connection: SqlServerConnection, stage: string, written: Written, into: string, _history: TargetHistory | null): Promise<boolean>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { SqlServerWriter, seq, writtenParameters, } from './sql-server-writer.js';
|
|
2
|
+
// Appends each record of a stream to its raw table, in the order the source
|
|
3
|
+
// gave them. A key that the columns declare is unique, so a commit that
|
|
4
|
+
// repeats one fails.
|
|
5
|
+
export class SqlServerAppendWriter extends SqlServerWriter {
|
|
6
|
+
keys = this.table.columns.filter(({ isPrimaryKey }) => isPrimaryKey);
|
|
7
|
+
async merge(connection, stage, written, into, _history) {
|
|
8
|
+
const [appended] = await connection.query(`INSERT INTO ${into} (${this.written.join(', ')})
|
|
9
|
+
SELECT ${this.staged('[_elt_new]').join(', ')} FROM ${stage} AS [_elt_new] ORDER BY [_elt_new].${seq};
|
|
10
|
+
SELECT @@ROWCOUNT AS [rows];`, writtenParameters(written));
|
|
11
|
+
return appended?.rows !== 0;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SqlServerConnection, SqlServerDatabase } from '@eltjs/sdk-microsoft-sql-server';
|
|
2
|
+
/**
|
|
3
|
+
* The database as of one position: `sql` is a session of its own in one
|
|
4
|
+
* SNAPSHOT transaction, and each reader view, and each view over one, shows
|
|
5
|
+
* its target as it was at that position. Dispose it to end the session.
|
|
6
|
+
*/
|
|
7
|
+
export type SqlServerReading = {
|
|
8
|
+
readonly sql: SqlServerConnection;
|
|
9
|
+
readonly position: string;
|
|
10
|
+
readonly at: Temporal.Instant;
|
|
11
|
+
} & AsyncDisposable;
|
|
12
|
+
export declare function readAsOf(database: SqlServerDatabase, position: string | undefined): Promise<SqlServerReading | {
|
|
13
|
+
readonly expired: string;
|
|
14
|
+
}>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { parsePosition, readingExpiry } from '@eltjs/elt';
|
|
2
|
+
import { connect } from './sql-server-connection.js';
|
|
3
|
+
import { committedAt, numberingIn, requireRowVersions, settledMark, sqlServerDatabase, } from './sql-server-numbering.js';
|
|
4
|
+
import { now } from './sql-server-reader-view.js';
|
|
5
|
+
// Reads the database as of the position, or as of its newest position. The
|
|
6
|
+
// high-water mark is read before the snapshot, so the snapshot sees each
|
|
7
|
+
// commit at or below it; one SNAPSHOT transaction then holds the snapshot
|
|
8
|
+
// that the numbering and every later read see, so a commit after the
|
|
9
|
+
// reading begins shows in none of them. The session context keeps the
|
|
10
|
+
// position as a read-only key, which each reader view reads and the reader
|
|
11
|
+
// cannot change: below the newest commit that the snapshot sees, the number
|
|
12
|
+
// of the position, and else now, as a session that sets none reads
|
|
13
|
+
// (ADR 0010). A session context value is a sql_variant, which holds no
|
|
14
|
+
// nvarchar(max), so the number goes as a bigint.
|
|
15
|
+
export async function readAsOf(database, position) {
|
|
16
|
+
const asked = position === undefined
|
|
17
|
+
? undefined
|
|
18
|
+
: parsePosition(position, sqlServerDatabase);
|
|
19
|
+
await using stack = new AsyncDisposableStack();
|
|
20
|
+
const connection = stack.use(await connect(database));
|
|
21
|
+
await requireRowVersions(connection);
|
|
22
|
+
const settled = await settledMark(connection);
|
|
23
|
+
if (settled === null)
|
|
24
|
+
return { expired: 'No load has committed yet' };
|
|
25
|
+
await connection.query('SET TRANSACTION ISOLATION LEVEL SNAPSHOT; BEGIN TRANSACTION');
|
|
26
|
+
const numbered = await numberingIn(connection, settled);
|
|
27
|
+
if (asked !== undefined) {
|
|
28
|
+
const expired = readingExpiry(numbered, asked, sqlServerDatabase, 'the database');
|
|
29
|
+
if (expired !== undefined)
|
|
30
|
+
return { expired };
|
|
31
|
+
}
|
|
32
|
+
const number = asked?.number ?? numbered?.ends.at(-1);
|
|
33
|
+
const commit = number === undefined ? undefined : await committedAt(connection, number);
|
|
34
|
+
if (numbered === null || number === undefined || commit === undefined)
|
|
35
|
+
return {
|
|
36
|
+
expired: position === undefined
|
|
37
|
+
? 'The database has no position yet'
|
|
38
|
+
: `Position ${position} names no commit of the database`,
|
|
39
|
+
};
|
|
40
|
+
await connection.query(`DECLARE @position bigint = CAST(@number AS bigint);
|
|
41
|
+
EXEC sp_set_session_context @key = N'elt.position', @value = @position, @read_only = 1;`, { number: number < commit.newest ? String(number) : now });
|
|
42
|
+
const owned = stack.move();
|
|
43
|
+
return {
|
|
44
|
+
sql: connection,
|
|
45
|
+
position: `${numbered.epoch}:${number}`,
|
|
46
|
+
at: Temporal.Instant.fromEpochMilliseconds(commit.at.getTime()),
|
|
47
|
+
[Symbol.asyncDispose]: () => owned.disposeAsync(),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type CheckpointSession, CheckpointStore, type FailureType } from '@eltjs/elt';
|
|
2
|
+
import { type SqlServerLogin } from './sql-server-connection.ts';
|
|
3
|
+
export declare class SqlServerCheckpointStore extends CheckpointStore {
|
|
4
|
+
#private;
|
|
5
|
+
constructor({ login }: {
|
|
6
|
+
login: SqlServerLogin;
|
|
7
|
+
});
|
|
8
|
+
failureType(error: unknown): FailureType;
|
|
9
|
+
protected session<T>(ids: readonly string[], work: (session: CheckpointSession) => Promise<T>): Promise<T>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { Modes, Mutex } from '@zukhruf/mutex';
|
|
2
|
+
import { SqlServerStore } from '@zukhruf/mutex/sqlserver';
|
|
3
|
+
import { CheckpointStore, } from '@eltjs/elt';
|
|
4
|
+
import { SqlServerDatabase, } from '@eltjs/sdk-microsoft-sql-server';
|
|
5
|
+
import { connect, takeLock, transaction, } from './sql-server-connection.js';
|
|
6
|
+
import { failureType } from './sql-server-failure-type.js';
|
|
7
|
+
import { quote } from './sql-server-identifiers.js';
|
|
8
|
+
import { inWarehouse, warehouse } from './sql-server-schemas.js';
|
|
9
|
+
// The checkpoints of every copy into the database, beside the data they
|
|
10
|
+
// describe, by the id of the copy. Each acknowledged checkpoint is saved as
|
|
11
|
+
// soon as the load commits it; a crash in between replays that unit. The id
|
|
12
|
+
// is unique by its SHA-256, as a key of any length is (ADR 0004), and keeps
|
|
13
|
+
// each UTF-16 code unit, as binding and state do: the JSON type would refuse
|
|
14
|
+
// \u0000 and lone surrogates.
|
|
15
|
+
const table = inWarehouse('_elt_checkpoints');
|
|
16
|
+
// The fencing tokens of the locks of the copies.
|
|
17
|
+
const tokens = inWarehouse('_elt_checkpoint_tokens');
|
|
18
|
+
export class SqlServerCheckpointStore extends CheckpointStore {
|
|
19
|
+
#database;
|
|
20
|
+
// One key for each copy id, on a session of its own (ADR 0009). The server
|
|
21
|
+
// releases a key when the session that holds it ends. The heartbeat runs
|
|
22
|
+
// on real time, as no caller has another correct value.
|
|
23
|
+
#locks;
|
|
24
|
+
constructor({ login }) {
|
|
25
|
+
super();
|
|
26
|
+
const database = new SqlServerDatabase(login);
|
|
27
|
+
this.#database = database;
|
|
28
|
+
this.#locks = new Mutex(new SqlServerStore({
|
|
29
|
+
connect: () => database.connect(),
|
|
30
|
+
sequence: tokens,
|
|
31
|
+
heartbeatInterval: 10_000,
|
|
32
|
+
clock: globalThis,
|
|
33
|
+
}));
|
|
34
|
+
Object.freeze(this);
|
|
35
|
+
}
|
|
36
|
+
failureType(error) {
|
|
37
|
+
return failureType(error);
|
|
38
|
+
}
|
|
39
|
+
async session(ids, work) {
|
|
40
|
+
await using connection = await connect(this.#database);
|
|
41
|
+
await create(connection);
|
|
42
|
+
return await this.#held(ids, () => withTable(connection, work));
|
|
43
|
+
}
|
|
44
|
+
// Holds each copy's key in turn, then works. A copy already running fails
|
|
45
|
+
// the run fast; others run in parallel.
|
|
46
|
+
async #held(ids, work) {
|
|
47
|
+
const [id, ...rest] = ids;
|
|
48
|
+
if (id === undefined)
|
|
49
|
+
return await work();
|
|
50
|
+
const held = await this.#locks.acquire(id, () => this.#held(rest, work), {
|
|
51
|
+
mode: Modes.skipIfBusy(),
|
|
52
|
+
});
|
|
53
|
+
if (!held.acquired)
|
|
54
|
+
throw new TypeError(`Checkpoint ${id} is in use by another run`);
|
|
55
|
+
return held.value;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// Each save commits on its own, so it is durable when it resolves.
|
|
59
|
+
function withTable(connection, work) {
|
|
60
|
+
return work({
|
|
61
|
+
read: async (id) => {
|
|
62
|
+
const [saved] = await connection.query(`SELECT [binding], [reloading] FROM ${table} WHERE [key] = HASHBYTES('SHA2_256', @id)`, { id });
|
|
63
|
+
return saved === undefined
|
|
64
|
+
? undefined
|
|
65
|
+
: {
|
|
66
|
+
binding: String(saved.binding),
|
|
67
|
+
reloading: saved.reloading === true,
|
|
68
|
+
};
|
|
69
|
+
},
|
|
70
|
+
state: async (id) => {
|
|
71
|
+
const [saved] = await connection.query(`SELECT [state] FROM ${table} WHERE [key] = HASHBYTES('SHA2_256', @id)`, { id });
|
|
72
|
+
return saved === undefined ? undefined : String(saved.state);
|
|
73
|
+
},
|
|
74
|
+
save: async (id, { binding, reloading, state }) => {
|
|
75
|
+
await connection.query(`UPDATE ${table} SET [binding] = @binding, [reloading] = CAST(@reloading AS bit), [state] = @state WHERE [key] = HASHBYTES('SHA2_256', @id);
|
|
76
|
+
IF @@ROWCOUNT = 0
|
|
77
|
+
INSERT INTO ${table} ([id], [binding], [reloading], [state]) VALUES (@id, @binding, CAST(@reloading AS bit), @state);`, { id, binding, reloading: reloading ? '1' : '0', state });
|
|
78
|
+
},
|
|
79
|
+
reloaded: async (id) => {
|
|
80
|
+
await connection.query(`UPDATE ${table} SET [reloading] = 0 WHERE [key] = HASHBYTES('SHA2_256', @id)`, { id });
|
|
81
|
+
},
|
|
82
|
+
remove: async (id) => {
|
|
83
|
+
await connection.query(`DELETE FROM ${table} WHERE [key] = HASHBYTES('SHA2_256', @id)`, { id });
|
|
84
|
+
},
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
// Two runs make the table at once, and SQL Server refuses the second CREATE;
|
|
88
|
+
// the lock of _warehouse makes them take turns.
|
|
89
|
+
async function create(connection) {
|
|
90
|
+
const [found] = await connection.query(`SELECT IIF(OBJECT_ID(N'${table}', N'U') IS NULL OR OBJECT_ID(N'${tokens}', N'SO') IS NULL, 0, 1) AS [made]`);
|
|
91
|
+
if (found?.made === 1)
|
|
92
|
+
return;
|
|
93
|
+
await transaction(connection, async () => {
|
|
94
|
+
await takeLock(connection, warehouse);
|
|
95
|
+
await connection.query(`IF SCHEMA_ID(N'${warehouse}') IS NULL EXEC (N'CREATE SCHEMA ${quote(warehouse)}');
|
|
96
|
+
IF OBJECT_ID(N'${table}', N'U') IS NULL
|
|
97
|
+
CREATE TABLE ${table} ([id] nvarchar(max) NOT NULL, [key] AS CAST(HASHBYTES('SHA2_256', [id]) AS binary(32)) PERSISTED NOT NULL PRIMARY KEY, [binding] nvarchar(max) NOT NULL, [reloading] bit NOT NULL, [state] nvarchar(max) NOT NULL);
|
|
98
|
+
IF OBJECT_ID(N'${tokens}', N'SO') IS NULL
|
|
99
|
+
EXEC (N'CREATE SEQUENCE ${tokens} AS bigint START WITH 1');`);
|
|
100
|
+
});
|
|
101
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { type DeclaredFormat, type DocumentParser, FileRead, type FileReference } from '@eltjs/elt';
|
|
2
|
+
declare const storageTypes: {
|
|
3
|
+
readonly text: 'nvarchar(max)';
|
|
4
|
+
readonly integer: 'bigint';
|
|
5
|
+
readonly int64: 'bigint';
|
|
6
|
+
readonly real: 'float';
|
|
7
|
+
readonly decimal: 'nvarchar(max)';
|
|
8
|
+
readonly boolean: 'bit';
|
|
9
|
+
readonly date: 'date';
|
|
10
|
+
readonly timestamp: 'datetimeoffset(7)';
|
|
11
|
+
readonly local_timestamp: 'datetime2(7)';
|
|
12
|
+
readonly local_time: 'time(7)';
|
|
13
|
+
readonly blob: 'varbinary(max)';
|
|
14
|
+
readonly object: 'json';
|
|
15
|
+
};
|
|
16
|
+
type Kind = keyof typeof storageTypes;
|
|
17
|
+
type PlainKind = 'text' | 'integer' | 'real' | 'blob' | 'boolean' | 'date' | 'timestamp' | 'object';
|
|
18
|
+
export type SqlServerColumnType = PlainKind | DeclaredFormat;
|
|
19
|
+
type EncodedObject = Readonly<Record<string, string | number | boolean | null>>;
|
|
20
|
+
export type EncodedValue = string | number | boolean | null | readonly (string | number | boolean | EncodedObject)[];
|
|
21
|
+
export declare class SqlServerColumn {
|
|
22
|
+
#private;
|
|
23
|
+
readonly field: string;
|
|
24
|
+
readonly name: string;
|
|
25
|
+
readonly type: SqlServerColumnType;
|
|
26
|
+
readonly kind: Kind;
|
|
27
|
+
readonly isPrimaryKey: boolean;
|
|
28
|
+
readonly nullable: boolean;
|
|
29
|
+
readonly optional: boolean;
|
|
30
|
+
readonly array: boolean;
|
|
31
|
+
readonly fileRead?: FileRead;
|
|
32
|
+
constructor(field: string, type: SqlServerColumnType, options: {
|
|
33
|
+
nullable: boolean;
|
|
34
|
+
optional: boolean;
|
|
35
|
+
primaryKey: boolean;
|
|
36
|
+
array?: boolean;
|
|
37
|
+
fileRead?: FileRead;
|
|
38
|
+
name?: string;
|
|
39
|
+
});
|
|
40
|
+
primaryKey(): SqlServerColumn;
|
|
41
|
+
notNull(): SqlServerColumn;
|
|
42
|
+
from(file: FileReference): SqlServerColumn;
|
|
43
|
+
parse(parser: DocumentParser): SqlServerColumn;
|
|
44
|
+
named(name: string): SqlServerColumn;
|
|
45
|
+
get storesFile(): boolean;
|
|
46
|
+
get quotedName(): string;
|
|
47
|
+
get storageType(): string;
|
|
48
|
+
get batchType(): string;
|
|
49
|
+
stored(alias: string): string;
|
|
50
|
+
get definition(): string;
|
|
51
|
+
encode(record: unknown): EncodedValue;
|
|
52
|
+
}
|
|
53
|
+
export {};
|