@eltjs/elt-sqlserver 0.0.0-stage → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +485 -2
  3. package/dist/computed-dataset.d.ts +31 -0
  4. package/dist/computed-dataset.js +380 -0
  5. package/dist/index.d.ts +13 -0
  6. package/dist/index.js +10 -0
  7. package/dist/sql-server-append-writer.d.ts +8 -0
  8. package/dist/sql-server-append-writer.js +13 -0
  9. package/dist/sql-server-as-of.d.ts +14 -0
  10. package/dist/sql-server-as-of.js +44 -0
  11. package/dist/sql-server-checkpoint-store.d.ts +10 -0
  12. package/dist/sql-server-checkpoint-store.js +101 -0
  13. package/dist/sql-server-column.d.ts +53 -0
  14. package/dist/sql-server-column.js +295 -0
  15. package/dist/sql-server-columns.d.ts +15 -0
  16. package/dist/sql-server-columns.js +49 -0
  17. package/dist/sql-server-connection.d.ts +6 -0
  18. package/dist/sql-server-connection.js +56 -0
  19. package/dist/sql-server-deduplicating-writer.d.ts +18 -0
  20. package/dist/sql-server-deduplicating-writer.js +102 -0
  21. package/dist/sql-server-dependents.d.ts +9 -0
  22. package/dist/sql-server-dependents.js +54 -0
  23. package/dist/sql-server-derived-status.d.ts +9 -0
  24. package/dist/sql-server-derived-status.js +60 -0
  25. package/dist/sql-server-descriptions.d.ts +6 -0
  26. package/dist/sql-server-descriptions.js +53 -0
  27. package/dist/sql-server-destination.d.ts +32 -0
  28. package/dist/sql-server-destination.js +95 -0
  29. package/dist/sql-server-failure-type.d.ts +2 -0
  30. package/dist/sql-server-failure-type.js +25 -0
  31. package/dist/sql-server-file-store.d.ts +21 -0
  32. package/dist/sql-server-file-store.js +91 -0
  33. package/dist/sql-server-fit.d.ts +20 -0
  34. package/dist/sql-server-fit.js +52 -0
  35. package/dist/sql-server-history.d.ts +15 -0
  36. package/dist/sql-server-history.js +148 -0
  37. package/dist/sql-server-identifiers.d.ts +10 -0
  38. package/dist/sql-server-identifiers.js +27 -0
  39. package/dist/sql-server-indexes.d.ts +23 -0
  40. package/dist/sql-server-indexes.js +121 -0
  41. package/dist/sql-server-load.d.ts +19 -0
  42. package/dist/sql-server-load.js +58 -0
  43. package/dist/sql-server-numbering.d.ts +32 -0
  44. package/dist/sql-server-numbering.js +255 -0
  45. package/dist/sql-server-overwrite-deduplicating-writer.d.ts +4 -0
  46. package/dist/sql-server-overwrite-deduplicating-writer.js +8 -0
  47. package/dist/sql-server-overwrite-writer.d.ts +4 -0
  48. package/dist/sql-server-overwrite-writer.js +9 -0
  49. package/dist/sql-server-parts-table.d.ts +24 -0
  50. package/dist/sql-server-parts-table.js +167 -0
  51. package/dist/sql-server-passes.d.ts +6 -0
  52. package/dist/sql-server-passes.js +71 -0
  53. package/dist/sql-server-reader-view.d.ts +12 -0
  54. package/dist/sql-server-reader-view.js +71 -0
  55. package/dist/sql-server-rule-columns.d.ts +12 -0
  56. package/dist/sql-server-rule-columns.js +69 -0
  57. package/dist/sql-server-schemas.d.ts +4 -0
  58. package/dist/sql-server-schemas.js +8 -0
  59. package/dist/sql-server-setup-error.d.ts +3 -0
  60. package/dist/sql-server-setup-error.js +6 -0
  61. package/dist/sql-server-sync-history-schema.d.ts +7 -0
  62. package/dist/sql-server-sync-history-schema.js +120 -0
  63. package/dist/sql-server-sync-history.d.ts +14 -0
  64. package/dist/sql-server-sync-history.js +165 -0
  65. package/dist/sql-server-table.d.ts +10 -0
  66. package/dist/sql-server-table.js +40 -0
  67. package/dist/sql-server-views.d.ts +7 -0
  68. package/dist/sql-server-views.js +70 -0
  69. package/dist/sql-server-writer.d.ts +40 -0
  70. package/dist/sql-server-writer.js +914 -0
  71. package/dist/view.d.ts +23 -0
  72. package/dist/view.js +82 -0
  73. package/package.json +48 -4
@@ -0,0 +1,914 @@
1
+ import { hash } from 'node:crypto';
2
+ import { FileContent, SomePartitions, TargetOwnedError, Writer, describeTarget, readPartitions, reloadMode, wholeStream, } from '@eltjs/elt';
3
+ import { connect, makeSchema, takeLock, transaction, } from './sql-server-connection.js';
4
+ import { numberOf, refreshDependents } from './sql-server-dependents.js';
5
+ import { TargetDescriptions } from './sql-server-descriptions.js';
6
+ import { SqlServerFileStore } from './sql-server-file-store.js';
7
+ import { storedColumns, storedFit } from './sql-server-fit.js';
8
+ import { TargetHistory, rowHash } from './sql-server-history.js';
9
+ import { quote } from './sql-server-identifiers.js';
10
+ import { commitOnce, databaseCopy, expireBefore, forgetCommit, installNumbering, } from './sql-server-numbering.js';
11
+ import { TargetReaderView } from './sql-server-reader-view.js';
12
+ import { inWarehouse, marts, warehouse } from './sql-server-schemas.js';
13
+ // A batch goes to the stage at 1,000 rows or 16 MiB of JSON, whichever comes
14
+ // first, as one parameter that OPENJSON reads (ADR 0011).
15
+ const batchSize = 1000;
16
+ const batchBytes = 16 * 2 ** 20;
17
+ export const seq = '[_elt_seq]';
18
+ // R for a record, D for a deletion of its key.
19
+ export const op = '[_elt_op]';
20
+ export const version = '[_elt_row_hash]';
21
+ export const keyHash = '[_elt_key_hash]';
22
+ const rowNumber = '[_elt_row]';
23
+ // The tables in _warehouse where each writer keeps the owner of its target
24
+ // and the partitions of its open reload.
25
+ const writersTable = inWarehouse('_elt_writers');
26
+ const reloadsTable = inWarehouse('_elt_reloads');
27
+ // How long the swap of a reload waits for a reader of the target before the
28
+ // copy fails until its next run.
29
+ const swapTimeout = 3000;
30
+ // The error of a statement that waited swapTimeout for a lock.
31
+ const lockTimeout = 1222;
32
+ // The columns of a raw table that the destination writes for itself, which no
33
+ // stream declares.
34
+ const internalColumns = new Set([
35
+ '_elt_row',
36
+ 'loaded_at',
37
+ 'committed_in',
38
+ '_elt_row_hash',
39
+ '_elt_key_hash',
40
+ ]);
41
+ // Loads a stream into its raw table: what each mode shares, the claim of the
42
+ // target, the stage of each commit, and the clear. Each mode merges the stage
43
+ // into the table in its own way. Each row is told apart by _elt_row, its
44
+ // identity number, the clustered key of the table (ADR 0003).
45
+ export class SqlServerWriter extends Writer {
46
+ configuration;
47
+ table;
48
+ history;
49
+ #database;
50
+ #retention;
51
+ #descriptions;
52
+ #view;
53
+ constructor(configuration, database, table, retention) {
54
+ super(configuration.stream);
55
+ this.configuration = configuration;
56
+ this.table = table;
57
+ this.#database = database;
58
+ this.#retention = retention;
59
+ this.history = new TargetHistory(this.#hash);
60
+ const description = describeTarget(configuration, table.columns, (column) => `UNIQUEIDENTIFIER reference to the source file's original bytes. Join ${new SqlServerFileStore(table, column).view} on file = this value and concatenate bytes in order of n. NULL when the source file is unavailable.`);
61
+ const { missing } = description;
62
+ if (missing.length > 0)
63
+ throw new TypeError(`Reader view ${marts}.${table.name} needs JSON Schema descriptions for ${missing.join(', ')} of stream ${this.stream.name}`);
64
+ this.#descriptions = new TargetDescriptions(description.table, {
65
+ ...description.columns,
66
+ committed_in: `Number of the commit that last wrote this row: the row of ${warehouse}._elt_commits with this number tells when, and in which load.`,
67
+ });
68
+ this.#view = new TargetReaderView(table.name, this.qualifiedName, this.history.qualifiedName, [...table.columns.map(({ name }) => name), 'loaded_at'], new TargetDescriptions(description.table, description.columns));
69
+ }
70
+ get qualifiedName() {
71
+ return inWarehouse(this.table.raw);
72
+ }
73
+ get #hash() {
74
+ return hash('sha256', this.qualifiedName).slice(0, 40);
75
+ }
76
+ // A reload's hidden target, beside the target and invisible to readers.
77
+ get #hidden() {
78
+ return inWarehouse(`_elt_next_${this.#hash}`);
79
+ }
80
+ // The name that the target takes in the swap of a reload, before it goes.
81
+ get #replacedName() {
82
+ return `_elt_replaced_${this.#hash}`;
83
+ }
84
+ // The stores of the original bytes of the target's file columns.
85
+ get #stores() {
86
+ return this.table.columns
87
+ .filter((column) => column.storesFile)
88
+ .map((column) => new SqlServerFileStore(this.table, column));
89
+ }
90
+ // The key hash of a table with a key, as its only element: a key column of
91
+ // nvarchar(max) cannot be an index key, and its collation would take two
92
+ // keys that differ in case for one (ADR 0004).
93
+ get #keyHashes() {
94
+ return this.keys.length > 0 ? [keyHash] : [];
95
+ }
96
+ // The unique index on the key hash, named after the table, its key and the
97
+ // types of the key, so a table whose stored hashes are of another key, or
98
+ // of key values of another type, no longer fits, and no evolve takes it.
99
+ get #keyIndex() {
100
+ return this.keys.length > 0
101
+ ? `_elt_key_${hash('sha256', JSON.stringify([this.table.raw, this.keys.map(({ name, storageType }) => [name, storageType])])).slice(0, 40)}`
102
+ : null;
103
+ }
104
+ // Refuses, before anything is read, stored rows the load cannot keep.
105
+ async inspect(_connection) { }
106
+ // An overwrite replaces the rows of the target at the first commit of its
107
+ // run.
108
+ get replaces() {
109
+ return false;
110
+ }
111
+ // Which rows of the stage a commit that replaces the target keeps, as a
112
+ // condition on them.
113
+ kept(_stage) {
114
+ return `${op} = 'R'`;
115
+ }
116
+ // The ids of the rows of left that no row of right matches, as multisets:
117
+ // the k-th copy of some values on one side matches the k-th copy of the
118
+ // same values on the other, so no key is needed.
119
+ #unmatched(left, right) {
120
+ const numbered = ({ table, id, rows }) => `SELECT ${id} AS [_elt_id], ${version} AS [_elt_hash], ROW_NUMBER() OVER (PARTITION BY ${version} ORDER BY ${id}) AS [_elt_copy] FROM ${table}${rows === undefined ? '' : ` WHERE ${rows}`}`;
121
+ return `SELECT [_elt_mine].[_elt_id] FROM (${numbered(left)}) AS [_elt_mine] LEFT JOIN (${numbered(right)}) AS [_elt_theirs] ON [_elt_theirs].[_elt_hash] = [_elt_mine].[_elt_hash] AND [_elt_theirs].[_elt_copy] = [_elt_mine].[_elt_copy] WHERE [_elt_theirs].[_elt_id] IS NULL`;
122
+ }
123
+ // Replaces the rows of the target with the records that the stage keeps,
124
+ // by difference: the history keeps each row of the target that no record
125
+ // matches, and the row goes; a row with a match stays as it is, with its
126
+ // commit and its loaded_at; and each record that no row matches enters
127
+ // with this commit. What stays matches records, so it fits the index of
128
+ // the key hash, which the records that enter then fit too. Gives whether it
129
+ // changed a row.
130
+ async #replace(connection, stage, written) {
131
+ const target = { table: this.qualifiedName, id: rowNumber };
132
+ const staged = { table: stage, id: seq, rows: this.kept(stage) };
133
+ const removed = await this.history.remove(connection, written.committedIn, this.qualifiedName, `${rowNumber} IN (${this.#unmatched(target, staged)})`, {});
134
+ const [entered] = await connection.query(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')})
135
+ SELECT ${this.staged('[_elt_new]').join(', ')} FROM ${stage} AS [_elt_new] WHERE [_elt_new].${seq} IN (${this.#unmatched(staged, target)}) ORDER BY [_elt_new].${seq};
136
+ SELECT @@ROWCOUNT AS [rows];`, writtenParameters(written));
137
+ return removed > 0 || entered?.rows !== 0;
138
+ }
139
+ encode(record) {
140
+ return this.table.columns.map((column) => column.encode(record));
141
+ }
142
+ // Only a load that identifies rows by key can remove one: the key's values,
143
+ // at their columns, and null elsewhere.
144
+ deletionRow(_key) {
145
+ throw new TypeError('Only deduplicating loads can apply deletions');
146
+ }
147
+ // What a commit writes into each row: its values, when its load began, its
148
+ // number, and its hashes.
149
+ get written() {
150
+ return [
151
+ ...this.table.columns.map((column) => column.quotedName),
152
+ '[loaded_at]',
153
+ '[committed_in]',
154
+ version,
155
+ ...this.#keyHashes,
156
+ ];
157
+ }
158
+ // The value of each column of written, from the row of the stage that alias
159
+ // names.
160
+ staged(alias) {
161
+ return [
162
+ ...this.table.columns.map((column) => `${alias}.${column.quotedName}`),
163
+ 'CAST(@loaded AS datetimeoffset(3))',
164
+ 'CAST(@committed AS bigint)',
165
+ `${alias}.${version}`,
166
+ ...this.#keyHashes.map((column) => `${alias}.${column}`),
167
+ ];
168
+ }
169
+ // A table named name with the target's columns, clustered on its row
170
+ // number.
171
+ #tableSQL(name) {
172
+ if (this.table.columns.length === 0)
173
+ throw new TypeError('Resolve inferred columns before creating a table');
174
+ return `${name} (${rowNumber} bigint IDENTITY(1, 1) NOT NULL PRIMARY KEY CLUSTERED, ${[
175
+ ...this.table.columns.map((column) => column.definition),
176
+ '[loaded_at] datetimeoffset(3) NOT NULL',
177
+ '[committed_in] bigint NOT NULL',
178
+ `${version} binary(32) NULL`,
179
+ ...this.#keyHashes.map((column) => `${column} binary(32) NOT NULL`),
180
+ ].join(', ')})`;
181
+ }
182
+ // The owner lives beside the table it guards. A dropped table releases it,
183
+ // since nothing it held remains.
184
+ async #refuse(connection, writer) {
185
+ const [row] = await connection.query(`SELECT [writer] FROM ${writersTable} WHERE [target] = @target AND OBJECT_ID(QUOTENAME(@schema) + N'.' + QUOTENAME([target]), N'U') IS NOT NULL`, { target: this.table.raw, schema: warehouse });
186
+ if (row !== undefined && row.writer !== writer)
187
+ throw new TargetOwnedError(this.table.name, String(row.writer), writer);
188
+ }
189
+ // Records the owner with the target's first commit; prepare refused any
190
+ // other.
191
+ async #own(connection, writer) {
192
+ await connection.query(`DELETE FROM ${writersTable} WHERE OBJECT_ID(QUOTENAME(@schema) + N'.' + QUOTENAME([target]), N'U') IS NULL;
193
+ IF NOT EXISTS (SELECT 1 FROM ${writersTable} WHERE [target] = @target)
194
+ INSERT INTO ${writersTable} ([target], [writer]) VALUES (@target, @writer);`, { target: this.table.raw, writer, schema: warehouse });
195
+ }
196
+ // The target as its first commit makes it: its owner, the table, its
197
+ // history and its descriptions, and the reader view.
198
+ async #create(connection, writer) {
199
+ await this.#own(connection, writer);
200
+ await connection.query(`IF OBJECT_ID(@table, N'U') IS NULL
201
+ BEGIN
202
+ CREATE TABLE ${this.#tableSQL(this.qualifiedName)};
203
+ ${this.#keyIndex === null ? '' : `CREATE UNIQUE INDEX ${quote(this.#keyIndex)} ON ${this.qualifiedName} (${keyHash});`}
204
+ END`, { table: this.qualifiedName });
205
+ await this.history.create(connection, this.qualifiedName);
206
+ await this.#descriptions.write(connection, 'TABLE', warehouse, this.table.raw);
207
+ await this.#view.describe(connection);
208
+ }
209
+ // The rows of any of the partitions, as a condition over the columns of a
210
+ // table of the target's shape, with one JSON parameter for each partition.
211
+ // Text compares by its code points, as the source tells its partitions
212
+ // apart, not in the collation of the database, which takes two values that
213
+ // differ only in case for one. unnamed selects the rows that leave out a
214
+ // column of a partition, such as a staged deletion whose key does not name
215
+ // it.
216
+ #within(partitions) {
217
+ const parameters = {};
218
+ const named = new Set();
219
+ const each = partitions.map((partition, index) => {
220
+ const columns = Object.keys(partition).map((field) => {
221
+ const column = this.table.columns.find((column) => column.field === field);
222
+ if (column === undefined)
223
+ throw new TypeError(`Resetting a partition requires destination column ${field}`);
224
+ named.add(column);
225
+ return column;
226
+ });
227
+ const parameter = `partition${index}`;
228
+ // One row, as OPENJSON reads the rows of a batch.
229
+ parameters[parameter] = JSON.stringify([
230
+ columns.map((column) => column.encode(partition)),
231
+ ]);
232
+ return columns
233
+ .map((column, at) => {
234
+ const binary = column.storageType.startsWith('nvarchar')
235
+ ? ' COLLATE Latin1_General_100_BIN2'
236
+ : '';
237
+ return `${column.quotedName}${binary} = (SELECT ${column.stored('j.[v]')} FROM OPENJSON(@${parameter}) WITH ([v] ${column.batchType} '$[${at}]') AS j)${binary}`;
238
+ })
239
+ .join(' AND ');
240
+ });
241
+ return {
242
+ rows: each.length === 0
243
+ ? '1 = 0'
244
+ : each.map((condition) => `(${condition})`).join(' OR '),
245
+ unnamed: [...named]
246
+ .map((column) => `${column.quotedName} IS NULL`)
247
+ .join(' OR '),
248
+ parameters,
249
+ };
250
+ }
251
+ // The partitions that an open reload of the target reloads, in the
252
+ // writer's own table beside it, written in the commit that opens or
253
+ // continues the reload, so a run that resumes the reload continues it. A
254
+ // reload of the whole stream has no row.
255
+ async #reloading(connection) {
256
+ const [row] = await connection.query(`SELECT [partitions] FROM ${reloadsTable} WHERE [target] = @target`, { target: this.table.raw });
257
+ return row === undefined
258
+ ? wholeStream
259
+ : new SomePartitions(readPartitions(this.stream, JSON.parse(String(row.partitions))));
260
+ }
261
+ async #reload(connection, reloads) {
262
+ if (!(reloads instanceof SomePartitions)) {
263
+ await this.#reloaded(connection);
264
+ return;
265
+ }
266
+ await connection.query(`UPDATE ${reloadsTable} SET [partitions] = @partitions WHERE [target] = @target;
267
+ IF @@ROWCOUNT = 0 INSERT INTO ${reloadsTable} ([target], [partitions]) VALUES (@target, @partitions);`, {
268
+ target: this.table.raw,
269
+ partitions: JSON.stringify(reloads.partitions),
270
+ });
271
+ }
272
+ async #reloaded(connection) {
273
+ await connection.query(`DELETE FROM ${reloadsTable} WHERE [target] = @target`, { target: this.table.raw });
274
+ }
275
+ // The hidden target of a new reload, with the target's columns and the
276
+ // index of its key hash, which a deduplicating merge fills it by.
277
+ async #createHidden(connection) {
278
+ await connection.query(`DROP TABLE IF EXISTS ${this.#hidden};
279
+ CREATE TABLE ${this.#tableSQL(this.#hidden)};
280
+ ${this.#keyIndex === null ? '' : `CREATE UNIQUE INDEX ${quote(this.#keyIndex)} ON ${this.#hidden} (${keyHash});`}`);
281
+ }
282
+ // Gives a completed reload's rows to a target that still has the stream's
283
+ // shape, in place and by difference: the history keeps each row of the
284
+ // target, or of the rows of it that scope selects, that no row of the
285
+ // hidden target matches, and the row goes; a row with a match stays, with
286
+ // its commit and its loaded_at, and the target keeps its grants and the
287
+ // views over it. Each row of the hidden target that no row matches enters
288
+ // with this commit, and the loaded_at of the load that wrote it. Gives
289
+ // whether it changed a row.
290
+ async #reloadInPlace(connection, committedIn, scope) {
291
+ const parameters = scope?.parameters ?? {};
292
+ const target = {
293
+ table: this.qualifiedName,
294
+ id: rowNumber,
295
+ ...(scope === null ? {} : { rows: scope.rows }),
296
+ };
297
+ const reloaded = { table: this.#hidden, id: rowNumber };
298
+ const removed = await this.history.remove(connection, committedIn, this.qualifiedName, `${rowNumber} IN (${this.#unmatched(target, reloaded)})`, parameters);
299
+ const [entered] = await connection.query(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')})
300
+ SELECT ${[...this.table.columns.map((column) => column.quotedName), '[loaded_at]', 'CAST(@committed AS bigint)', version, ...this.#keyHashes].join(', ')}
301
+ FROM ${this.#hidden} WHERE ${rowNumber} IN (${this.#unmatched(reloaded, target)}) ORDER BY ${rowNumber};
302
+ SELECT @@ROWCOUNT AS [rows];`, { ...parameters, committed: String(committedIn) });
303
+ await connection.query(`DROP TABLE ${this.#hidden}`);
304
+ return removed > 0 || entered?.rows !== 0;
305
+ }
306
+ // The permissions of the target, and of each of its columns that the
307
+ // stream still has, as the statements that give them to the table that
308
+ // takes its place.
309
+ async #grants(connection) {
310
+ const names = new Set(this.table.columns.map(({ name }) => name));
311
+ const rows = await connection.query(`SELECT p.[state_desc], p.[permission_name], USER_NAME(p.[grantee_principal_id]) AS [grantee], COL_NAME(p.[major_id], p.[minor_id]) AS [column]
312
+ FROM sys.database_permissions AS p WHERE p.[class] = 1 AND p.[major_id] = OBJECT_ID(@table)`, { table: this.qualifiedName });
313
+ return rows.flatMap(({ state_desc, permission_name, grantee, column }) => {
314
+ if (column !== null && !names.has(String(column)))
315
+ return [];
316
+ const state = String(state_desc);
317
+ const on = `${this.qualifiedName}${column === null ? '' : ` (${quote(String(column))})`}`;
318
+ return [
319
+ state === 'GRANT_WITH_GRANT_OPTION'
320
+ ? `GRANT ${String(permission_name)} ON ${on} TO ${quote(String(grantee))} WITH GRANT OPTION`
321
+ : `${state} ${String(permission_name)} ON ${on} TO ${quote(String(grantee))}`,
322
+ ];
323
+ });
324
+ }
325
+ // Swaps a completed reload's hidden target in, when the target no longer
326
+ // has the stream's shape, so readers of the stored table wait only for the
327
+ // swap, not the reload. A reader that holds it past the timeout fails the
328
+ // copy until the next run. Its rows are new to readers at this commit, and
329
+ // the history keeps every row of the target it replaces, unless the change
330
+ // retypes a column that the history keeps: each position before it then
331
+ // expires. The table takes the target's permissions, the reader view is
332
+ // defined again over the new columns, and the
333
+ // views over the table, its history or its reader view refresh (ADR 0006).
334
+ // SET LOCK_TIMEOUT lasts to the end of a batch with parameters, so it is a
335
+ // batch with none.
336
+ async #swap(connection, committedIn) {
337
+ await connection.query(`SET LOCK_TIMEOUT ${swapTimeout}`);
338
+ try {
339
+ await connection.query(`UPDATE ${this.#hidden} SET [committed_in] = CAST(@committed AS bigint)`, { committed: String(committedIn) });
340
+ const relations = [
341
+ this.qualifiedName,
342
+ this.history.qualifiedName,
343
+ this.#view.qualifiedName,
344
+ ];
345
+ await refreshDependents(connection, {
346
+ roots: relations,
347
+ views: [this.#view.qualifiedName],
348
+ changed: this.qualifiedName,
349
+ }, async () => {
350
+ await this.#dropRetiredChunks(connection);
351
+ const [target] = await connection.query(`SELECT IIF(OBJECT_ID(@table, N'U') IS NULL, 0, 1) AS [made]`, { table: this.qualifiedName });
352
+ const replacing = target?.made === 1;
353
+ const grants = replacing ? await this.#grants(connection) : [];
354
+ if (replacing) {
355
+ if ((await this.history.follow(connection, this.#hidden)) ===
356
+ 'retyped')
357
+ await expireBefore(connection, committedIn, 'type');
358
+ else
359
+ await this.history.keep(connection, committedIn, this.qualifiedName, `SELECT * FROM ${this.qualifiedName}`, {});
360
+ await connection.query('EXEC sp_rename @table, @replaced', {
361
+ table: this.qualifiedName,
362
+ replaced: this.#replacedName,
363
+ });
364
+ }
365
+ await connection.query('EXEC sp_rename @hidden, @raw', {
366
+ hidden: this.#hidden,
367
+ raw: this.table.raw,
368
+ });
369
+ if (replacing)
370
+ await connection.query(`DROP TABLE ${inWarehouse(this.#replacedName)}`);
371
+ for (const grant of grants)
372
+ await connection.query(grant);
373
+ await this.history.create(connection, this.qualifiedName);
374
+ await this.#descriptions.write(connection, 'TABLE', warehouse, this.table.raw);
375
+ await this.#view.replace(connection);
376
+ });
377
+ }
378
+ finally {
379
+ await connection.query('SET LOCK_TIMEOUT -1');
380
+ }
381
+ }
382
+ // Refuses a conversion of a stored column from its type from to the type
383
+ // of column that would change a stored value, such as 1.5 in a float to
384
+ // the bigint 2, before the ALTER: each value that is not null must convert
385
+ // back to itself, as elt-postgresql checks. A conversion from text is the
386
+ // parse that the load applies to each value of the new type, so it is
387
+ // trusted, and a value that it cannot parse fails the ALTER. JSON has no
388
+ // comparison, so it compares as its text.
389
+ async #refuseInexactCast(connection, column, from) {
390
+ if (from === 'nvarchar(max)')
391
+ return;
392
+ const { quotedName: name, storageType: to } = column;
393
+ const back = `TRY_CAST(TRY_CAST(${name} AS ${to}) AS ${from})`;
394
+ const differs = from === 'json'
395
+ ? `CAST(${back} AS nvarchar(max)) COLLATE Latin1_General_100_BIN2 <> CAST(${name} AS nvarchar(max)) COLLATE Latin1_General_100_BIN2`
396
+ : `${back} <> ${name}`;
397
+ let changes;
398
+ try {
399
+ changes = await connection.query(`SELECT TOP (1) 1 AS [changed] FROM ${this.qualifiedName} WHERE ${name} IS NOT NULL AND (${back} IS NULL OR ${differs})`);
400
+ }
401
+ catch (cause) {
402
+ throw new TypeError(`column ${name} does not convert to ${to}: ${Error.isError(cause) ? cause.message : String(cause)}`, { cause });
403
+ }
404
+ if (changes.length > 0)
405
+ throw new TypeError(`column ${name} holds a value that a conversion to ${to} would change`);
406
+ }
407
+ // Drops the CHECK constraints on a column of the target, such as the one of
408
+ // an array column, which stop an ALTER or a DROP of the column.
409
+ async #dropChecks(connection, column) {
410
+ const checks = await connection.query(`SELECT k.[name] FROM sys.check_constraints AS k JOIN sys.columns AS c ON c.[object_id] = k.[parent_object_id] AND c.[column_id] = k.[parent_column_id]
411
+ WHERE k.[parent_object_id] = OBJECT_ID(@table) AND c.[name] = @column`, { table: this.qualifiedName, column });
412
+ for (const { name } of checks)
413
+ await connection.query(`ALTER TABLE ${this.qualifiedName} DROP CONSTRAINT ${quote(String(name))}`);
414
+ }
415
+ // Brings a stored table that the stream no longer fits to its shape in
416
+ // place, keeping every row, as elt-postgresql does: a column that the
417
+ // stream adds enters nullable, one that it drops goes, and a changed type
418
+ // converts with ALTER COLUMN, after the check that the conversion keeps
419
+ // each value. The change is one commit of its own, which waits for readers
420
+ // at most swapTimeout, as a swap does: a reader that holds the table fails
421
+ // the copy until the next run, and the transaction leaves the table as it
422
+ // was. A row with a value in a column that the change drops or converts
423
+ // loses or changes the value, so it is a new version: the history keeps
424
+ // the row as it was, and its row hash becomes NULL, which matches no
425
+ // record until a commit writes the row again. A column that the change
426
+ // adds is NULL in each row, which a row hash does not count. The history
427
+ // follows the new shape; a change of type that is not a widening expires
428
+ // each position before the commit. A commit that kept no row and expired
429
+ // no position takes back its commit row. The key hashes cannot be made
430
+ // again from the stored rows, so a table whose key index is of another key
431
+ // fails the copy.
432
+ async #evolve(load, stage) {
433
+ const { connection } = load;
434
+ const keyIndex = this.#keyIndex;
435
+ const table = this.qualifiedName;
436
+ const [hashes] = keyIndex === null
437
+ ? await connection.query(`SELECT IIF(COL_LENGTH(@table, N'_elt_key_hash') IS NULL, 1, 0) AS [fit]`, { table })
438
+ : await connection.query('SELECT IIF(EXISTS (SELECT 1 FROM sys.indexes WHERE [object_id] = OBJECT_ID(@table) AND [name] = @index), 1, 0) AS [fit]', { table, index: keyIndex });
439
+ if (hashes?.fit !== 1)
440
+ throw new TypeError(`${table} no longer has the shape of stream ${this.stream.name}, and its stored key hashes are of another key or of key values of another type; clear the copy to load it again`);
441
+ await connection.query(`SET LOCK_TIMEOUT ${swapTimeout}`);
442
+ try {
443
+ await load.commit(async (committedIn) => {
444
+ const stored = new Map((await storedColumns(connection, { table: this.qualifiedName })).map((column) => [column.name, column]));
445
+ const types = new Map((await storedColumns(connection, { stage })).map(({ name, type }) => [
446
+ name,
447
+ type,
448
+ ]));
449
+ const changed = [...stored.values()]
450
+ .filter(({ name, type }) => !internalColumns.has(name) && types.get(name) !== type)
451
+ .map(({ name }) => `${quote(name)} IS NOT NULL`)
452
+ .join(' OR ');
453
+ const kept = changed === ''
454
+ ? 0
455
+ : await this.history.keep(connection, committedIn, this.qualifiedName, `UPDATE ${this.qualifiedName} SET [committed_in] = CAST(@committed AS bigint), [loaded_at] = CAST(@loaded AS datetimeoffset(3)), ${version} = NULL OUTPUT deleted.* WHERE ${changed}`, writtenParameters({ loadedAt: load.loadedAt, committedIn }));
456
+ let retyped = false;
457
+ // refreshDependents names the schema-bound views that stop the
458
+ // change; any other error that is not the wait for a reader is a
459
+ // stored value that the new shape refuses.
460
+ try {
461
+ await refreshDependents(connection, {
462
+ roots: [
463
+ this.qualifiedName,
464
+ this.history.qualifiedName,
465
+ this.#view.qualifiedName,
466
+ ],
467
+ views: [this.#view.qualifiedName],
468
+ changed: this.qualifiedName,
469
+ }, async () => {
470
+ await this.#dropRetiredChunks(connection);
471
+ await this.#alter(connection, stored, types);
472
+ retyped =
473
+ (await this.history.follow(connection, this.qualifiedName)) ===
474
+ 'retyped';
475
+ if (retyped)
476
+ await expireBefore(connection, committedIn, 'type');
477
+ await this.history.create(connection, this.qualifiedName);
478
+ await this.#descriptions.write(connection, 'TABLE', warehouse, this.table.raw);
479
+ await this.#view.replace(connection);
480
+ });
481
+ }
482
+ catch (cause) {
483
+ if (cause instanceof TypeError || numberOf(cause) === lockTimeout)
484
+ throw cause;
485
+ throw new TypeError(`The rows stored in ${table} do not fit the new shape of stream ${this.stream.name}: ${Error.isError(cause) ? cause.message : String(cause)}. Clear the copy to load it again, or change the stream so they fit.`, { cause });
486
+ }
487
+ if (kept === 0 && !retyped)
488
+ await forgetCommit(connection, committedIn);
489
+ });
490
+ }
491
+ finally {
492
+ await connection.query('SET LOCK_TIMEOUT -1');
493
+ }
494
+ }
495
+ // The ALTER statements of #evolve, from the stored columns and the types
496
+ // that the stage spells for the stream's columns.
497
+ async #alter(connection, stored, types) {
498
+ const alter = (statement) => connection.query(`ALTER TABLE ${this.qualifiedName} ${statement}`);
499
+ for (const column of this.table.columns) {
500
+ const existing = stored.get(column.name);
501
+ if (existing === undefined) {
502
+ await alter(`ADD ${column.definition}`);
503
+ continue;
504
+ }
505
+ const retyped = existing.type !== types.get(column.name);
506
+ if (!retyped && existing.required === column.isPrimaryKey)
507
+ continue;
508
+ if (retyped)
509
+ await this.#refuseInexactCast(connection, column, existing.type);
510
+ await this.#dropChecks(connection, column.name);
511
+ await alter(`ALTER COLUMN ${column.quotedName} ${column.storageType} ${column.isPrimaryKey ? 'NOT NULL' : 'NULL'}`);
512
+ if (column.array)
513
+ await alter(`ADD CHECK (ISJSON(${column.quotedName}, ARRAY) = 1)`);
514
+ }
515
+ const declared = new Set(this.table.columns.map(({ name }) => name));
516
+ for (const name of stored.keys())
517
+ if (!declared.has(name) && !internalColumns.has(name)) {
518
+ await this.#dropChecks(connection, name);
519
+ await alter(`DROP COLUMN ${quote(name)}`);
520
+ }
521
+ }
522
+ // The field's values in each of the tables, a qualified name each: while
523
+ // a reload is open, the rows of the target and of its hidden target both
524
+ // still name files, and a row of the history keeps its file for a reading
525
+ // as of a position before the commit that replaced it.
526
+ #values(connection, tables) {
527
+ const { table } = this;
528
+ return async function* (field) {
529
+ const column = table.columns.find((column) => column.field === field);
530
+ if (column === undefined)
531
+ throw new TypeError(`Unknown target field: ${field}`);
532
+ for (const name of tables()) {
533
+ const stored = await storedColumns(connection, { table: name });
534
+ if (!stored.some((stored) => stored.name === column.name))
535
+ continue;
536
+ for (const row of await connection.query(`SELECT ${column.quotedName} AS [value] FROM ${name}`))
537
+ yield row.value;
538
+ }
539
+ };
540
+ }
541
+ // Drops the chunks of each stored column that is no longer one of the
542
+ // target's file columns, before the table changes: no row of the table
543
+ // that the change leaves names them.
544
+ async #dropRetiredChunks(connection) {
545
+ const files = new Set(this.table.columns
546
+ .filter(({ storesFile }) => storesFile)
547
+ .map(({ name }) => name));
548
+ for (const { name } of await storedColumns(connection, {
549
+ table: this.qualifiedName,
550
+ }))
551
+ if (!files.has(name) && !internalColumns.has(name))
552
+ await SqlServerFileStore.drop(connection, this.table, name);
553
+ }
554
+ // A clear of a stored table is a load of one commit of its connection. It
555
+ // empties the table, not drops it, so the views built on it keep working,
556
+ // and it forgets the rows it removes: each position before it expires.
557
+ async clear({ writer, clock, connection: connector }, committed) {
558
+ await using connection = await connect(this.#database);
559
+ await installNumbering(connection, await databaseCopy(connection));
560
+ await makeWriterTables(connection);
561
+ await transaction(connection, async () => {
562
+ const [row] = await connection.query(`SELECT [writer] FROM ${writersTable} WHERE [target] = @target`, { target: this.table.raw });
563
+ if (row !== undefined && row.writer !== writer)
564
+ throw new TargetOwnedError(this.table.name, String(row.writer), writer);
565
+ const [table] = await connection.query(`SELECT IIF(OBJECT_ID(@table, N'U') IS NULL, 0, 1) AS [made]`, { table: this.qualifiedName });
566
+ if (table?.made === 1) {
567
+ const committedIn = await commitOnce(connection, connector, new Date(clock.Date.now()), this.#retention);
568
+ await connection.query(`DELETE FROM ${this.qualifiedName}`);
569
+ await expireBefore(connection, committedIn, 'clear');
570
+ await this.history.expire(connection);
571
+ }
572
+ await connection.query(`DELETE FROM ${writersTable} WHERE [target] = @target`, { target: this.table.raw });
573
+ // Clearing a copy also abandons the reload it left open.
574
+ await connection.query(`DROP TABLE IF EXISTS ${this.#hidden}`);
575
+ await this.#reloaded(connection);
576
+ // No row is left in the table or its history, so no stored file stays.
577
+ for (const store of this.#stores)
578
+ await store.prune(connection, [this.history.qualifiedName]);
579
+ });
580
+ await committed?.(this.#values(connection, () => [
581
+ this.qualifiedName,
582
+ this.history.qualifiedName,
583
+ ]));
584
+ }
585
+ // One stream's load. Operations wait in a #temp table of the load's
586
+ // connection, so no other session sees them and a crash leaves nothing
587
+ // behind. prepare only refuses and stages, so a run that fails before a
588
+ // commit leaves no rows behind. Each commit is one transaction that merges
589
+ // the stage into the table, or into a reload's hidden target until
590
+ // complete() gives its rows to the table. The first commit also makes the
591
+ // table, or the hidden target of a new reload.
592
+ async prepare(load, { writer, reloading }) {
593
+ const { connection } = load;
594
+ const stage = `#elt_stage_${this.#hash}`;
595
+ // The rows of a commit that belong to the partitions a reload reloads.
596
+ const reloadStage = `#elt_reload_stage_${this.#hash}`;
597
+ const { columns } = this.table;
598
+ const staging = (table) => `DROP TABLE IF EXISTS ${table}; CREATE TABLE ${table} (${[
599
+ `${seq} bigint NOT NULL PRIMARY KEY`,
600
+ `${op} char(1) NOT NULL`,
601
+ ...columns.map((column) => `${column.quotedName} ${column.storageType} NULL`),
602
+ ...[version, ...this.#keyHashes].map((column) => `${column} binary(32) NULL`),
603
+ ].join(', ')});`;
604
+ await makeWriterTables(connection);
605
+ // A #temp table that a batch with parameters makes is dropped at the end
606
+ // of that batch, so the stages are made by a batch with none.
607
+ await connection.query(`${staging(stage)} ${staging(reloadStage)}`);
608
+ const stores = this.#stores;
609
+ for (const store of stores)
610
+ await store.stage(connection);
611
+ // What an open reload reloads; the whole stream until a RESET of a
612
+ // partition opens one.
613
+ let reloads = wholeStream;
614
+ const fit = (table) => storedFit(connection, this.table, {
615
+ table,
616
+ stage,
617
+ keyIndex: this.#keyIndex,
618
+ });
619
+ let mode = await load.transaction(async () => {
620
+ await this.#refuse(connection, writer);
621
+ const target = await fit(this.qualifiedName);
622
+ const decided = reloadMode({
623
+ reloading,
624
+ destinationSyncMode: this.configuration.destinationSyncMode,
625
+ target,
626
+ hidden: await fit(this.#hidden),
627
+ });
628
+ // A hidden target no reload continues is a leftover readers never saw.
629
+ if (decided === 'continue')
630
+ reloads = await this.#reloading(connection);
631
+ else {
632
+ await connection.query(`DROP TABLE IF EXISTS ${this.#hidden}`);
633
+ await this.#reloaded(connection);
634
+ }
635
+ await this.#view.refuse(connection, target === 'stale');
636
+ if (decided === 'load' && !this.replaces)
637
+ await this.inspect(connection);
638
+ return decided;
639
+ });
640
+ if (mode === 'evolve') {
641
+ await this.#evolve(load, stage);
642
+ await load.transaction(() => this.inspect(connection));
643
+ mode = 'load';
644
+ }
645
+ const fresh = mode === 'create' || mode === 'reload';
646
+ // The reload as the last commit left it, which a discard goes back to.
647
+ let committed = {
648
+ mode,
649
+ reloads,
650
+ };
651
+ const open = () => mode === 'reload' || mode === 'continue';
652
+ const into = () => (open() ? this.#hidden : this.qualifiedName);
653
+ // The tables whose rows name files: the target, its history, and the
654
+ // hidden target of an open reload.
655
+ const tables = () => [
656
+ ...(open() ? [this.qualifiedName, this.#hidden] : [this.qualifiedName]),
657
+ this.history.qualifiedName,
658
+ ];
659
+ // Each staged row as JSON: its place in the stream, its operation, its
660
+ // values, and the hashes, as hex, which the stage keeps as binary(32).
661
+ const hashes = [version, ...this.#keyHashes];
662
+ const insert = `INSERT INTO ${stage} (${[seq, op, ...columns.map((column) => column.quotedName), ...hashes].join(', ')})
663
+ SELECT ${['j.[s]', 'j.[o]', ...columns.map((column, index) => column.stored(`j.[v${index}]`)), ...hashes.map((_, index) => `CONVERT(binary(32), j.[h${index}], 2)`)].join(', ')}
664
+ FROM OPENJSON(@batch) WITH (${[
665
+ `[s] bigint '$[0]'`,
666
+ `[o] char(1) '$[1]'`,
667
+ ...columns.map((column, index) => `[v${index}] ${column.batchType} '$[${index + 2}]'${column.array ? ' AS JSON' : ''}`),
668
+ ...hashes.map((_, index) => `[h${index}] char(64) '$[${columns.length + 2 + index}]'`),
669
+ ].join(', ')}) AS j`;
670
+ const keyed = this.keys.map((key) => columns.indexOf(key));
671
+ let pending = [];
672
+ let pendingBytes = 0;
673
+ let next = 0;
674
+ const flush = async () => {
675
+ if (pending.length === 0)
676
+ return;
677
+ const rows = pending;
678
+ pending = [];
679
+ pendingBytes = 0;
680
+ await connection.query(insert, { batch: `[${rows.join(',')}]` });
681
+ };
682
+ // Partitions the stage dropped, which the next commit empties in the
683
+ // table it merges into.
684
+ let resets = [];
685
+ const drop = async () => {
686
+ pending = [];
687
+ pendingBytes = 0;
688
+ resets = [];
689
+ await connection.query(`TRUNCATE TABLE ${stage}`);
690
+ for (const store of stores)
691
+ await store.discard(connection);
692
+ };
693
+ // Moves the stage into the target, or the hidden target of an open
694
+ // reload, and gives whether a reader sees a row change. The first commit
695
+ // of an overwrite that loads into its target replaces the rows of the
696
+ // target. The rows of the partitions that a reload reloads go to the
697
+ // hidden target, the others to the target; a deletion whose key does not
698
+ // name its partition goes to both, so the reload cannot bring back a row
699
+ // that the source deleted.
700
+ const merge = async (written, replacing) => {
701
+ if (replacing)
702
+ return this.#replace(connection, stage, written);
703
+ if (!open())
704
+ return this.merge(connection, stage, written, this.qualifiedName, this.history);
705
+ if (!(reloads instanceof SomePartitions)) {
706
+ await this.merge(connection, stage, written, this.#hidden, null);
707
+ return false;
708
+ }
709
+ const { rows, unnamed, parameters } = this.#within(reloads.partitions);
710
+ await connection.query(`INSERT INTO ${reloadStage} SELECT * FROM ${stage} WHERE (${rows}) OR (${op} = 'D' AND (${unnamed}));
711
+ DELETE FROM ${stage} WHERE ${rows};`, parameters);
712
+ const changed = await this.merge(connection, stage, written, this.qualifiedName, this.history);
713
+ await this.merge(connection, reloadStage, written, this.#hidden, null);
714
+ await connection.query(`TRUNCATE TABLE ${reloadStage}`);
715
+ return changed;
716
+ };
717
+ // The first commit claims the target: it owns the hidden target of a
718
+ // reload, or makes the table, before its transaction making marts for
719
+ // the reader view, and the made table is a change even with no row; or it
720
+ // finds the table made. A commit of a made target that no reader sees
721
+ // change takes back its commit row.
722
+ const unclaimed = () => {
723
+ if (open())
724
+ return {
725
+ before: async () => { },
726
+ claim: () => this.#own(connection, writer),
727
+ changedNone: (committedIn) => forgetCommit(connection, committedIn),
728
+ replaces: false,
729
+ };
730
+ if (mode === 'create')
731
+ return {
732
+ before: () => makeSchema(connection, marts),
733
+ claim: () => this.#create(connection, writer),
734
+ changedNone: async () => { },
735
+ replaces: this.replaces,
736
+ };
737
+ return { ...claimed, replaces: this.replaces };
738
+ };
739
+ const claimed = {
740
+ before: async () => { },
741
+ claim: async () => { },
742
+ changedNone: (committedIn) => forgetCommit(connection, committedIn),
743
+ replaces: false,
744
+ };
745
+ let claiming = unclaimed;
746
+ return {
747
+ fresh,
748
+ get reloading() {
749
+ return open();
750
+ },
751
+ values: this.#values(connection, tables),
752
+ apply: async (operation) => {
753
+ if (operation.type === 'RESET') {
754
+ await flush();
755
+ const { partition } = operation;
756
+ if (partition === null) {
757
+ // The whole stream starts over in a new hidden target.
758
+ await drop();
759
+ mode = 'reload';
760
+ reloads = wholeStream;
761
+ return;
762
+ }
763
+ const { rows, parameters } = this.#within([partition]);
764
+ await connection.query(`DELETE FROM ${stage} WHERE ${rows}`, parameters);
765
+ resets.push(partition);
766
+ // The partition starts over in a hidden target, and readers see its
767
+ // rows as they were until the stream ends. An overwrite replaces
768
+ // the whole target anyway.
769
+ if (this.replaces)
770
+ return;
771
+ if (mode === 'load') {
772
+ mode = 'reload';
773
+ reloads = new SomePartitions([partition]);
774
+ }
775
+ else if (open() && reloads instanceof SomePartitions)
776
+ reloads = reloads.with(partition);
777
+ return;
778
+ }
779
+ // Each file is read here, before the source advances past it. The
780
+ // row keeps its id, and the row hash its content.
781
+ let data = operation.type === 'RECORD' ? operation.data : undefined;
782
+ const contents = new Map();
783
+ for (const store of stores) {
784
+ const content = Reflect.get(Object(data), store.column.field);
785
+ if (content instanceof FileContent) {
786
+ const { file, sha256 } = await store.save(connection, content);
787
+ data = { ...Object(data), [store.column.field]: file };
788
+ contents.set(store.column, sha256);
789
+ }
790
+ }
791
+ const row = operation.type === 'RECORD'
792
+ ? this.encode(data)
793
+ : this.deletionRow(operation.key);
794
+ const text = JSON.stringify([
795
+ next++,
796
+ operation.type === 'RECORD' ? 'R' : 'D',
797
+ ...row,
798
+ operation.type === 'RECORD'
799
+ ? rowHash(columns, columns.map((column, index) => contents.get(column) ?? row[index] ?? null))
800
+ : null,
801
+ ...this.#keyHashes.map(() => hash('sha256', JSON.stringify(keyed.map((index) => row[index])))),
802
+ ]);
803
+ const bytes = Buffer.byteLength(text);
804
+ if (pendingBytes + bytes > batchBytes)
805
+ await flush();
806
+ pending.push(text);
807
+ pendingBytes += bytes;
808
+ if (pending.length >= batchSize || pendingBytes >= batchBytes)
809
+ await flush();
810
+ },
811
+ commit: async () => {
812
+ await flush();
813
+ const claim = claiming();
814
+ await claim.before();
815
+ await load.commit(async (committedIn) => {
816
+ if (mode === 'reload')
817
+ await this.#createHidden(connection);
818
+ if (open())
819
+ await this.#reload(connection, reloads);
820
+ await claim.claim();
821
+ // The rows of a hidden target were never shown, so only the target
822
+ // keeps the rows a commit replaces.
823
+ let changed = false;
824
+ for (const partition of resets) {
825
+ const { rows, parameters } = this.#within([partition]);
826
+ if (open())
827
+ await connection.query(`DELETE FROM ${into()} WHERE ${rows}`, parameters);
828
+ else
829
+ changed =
830
+ (await this.history.remove(connection, committedIn, into(), rows, parameters)) > 0 || changed;
831
+ }
832
+ for (const store of stores)
833
+ await store.publish(connection);
834
+ changed =
835
+ (await merge({ loadedAt: load.loadedAt, committedIn }, claim.replaces)) || changed;
836
+ await connection.query(`TRUNCATE TABLE ${stage}`);
837
+ if (!changed)
838
+ await claim.changedNone(committedIn);
839
+ await this.history.expire(connection);
840
+ for (const store of stores)
841
+ await store.prune(connection, tables());
842
+ });
843
+ claiming = () => claimed;
844
+ // Its first commit made the hidden target, so the reload continues.
845
+ if (mode === 'reload')
846
+ mode = 'continue';
847
+ committed = { mode, reloads };
848
+ resets = [];
849
+ },
850
+ complete: async () => {
851
+ if (!open())
852
+ return;
853
+ await makeSchema(connection, marts);
854
+ await load.commit(async (committedIn) => {
855
+ const target = await fit(this.qualifiedName);
856
+ // A reload of partitions keeps the target's shape: a load whose
857
+ // stored target no longer fits refuses it in prepare.
858
+ let changed = true;
859
+ if (reloads instanceof SomePartitions && target === 'fits')
860
+ changed = await this.#reloadInPlace(connection, committedIn, this.#within(reloads.partitions));
861
+ else if (reloads instanceof SomePartitions)
862
+ throw new TypeError(`${this.qualifiedName} changed its shape while some of its partitions reloaded; clear the copy to load it again`);
863
+ else if (target === 'fits')
864
+ changed = await this.#reloadInPlace(connection, committedIn, null);
865
+ else
866
+ await this.#swap(connection, committedIn);
867
+ await this.#reloaded(connection);
868
+ if (!changed)
869
+ await forgetCommit(connection, committedIn);
870
+ await this.history.expire(connection);
871
+ for (const store of stores)
872
+ await store.prune(connection, [
873
+ this.qualifiedName,
874
+ this.history.qualifiedName,
875
+ ]);
876
+ });
877
+ mode = 'load';
878
+ reloads = wholeStream;
879
+ committed = { mode, reloads };
880
+ },
881
+ // A RESET that no commit took is undone with the operations after it:
882
+ // the source's checkpoint does not know it.
883
+ discard: async () => {
884
+ await drop();
885
+ ({ mode, reloads } = committed);
886
+ },
887
+ [Symbol.asyncDispose]: async () => {
888
+ await connection.query(`DROP TABLE IF EXISTS ${stage}; DROP TABLE IF EXISTS ${reloadStage}`);
889
+ for (const store of stores)
890
+ await store.unstage(connection);
891
+ },
892
+ };
893
+ }
894
+ }
895
+ // written as the parameters @loaded and @committed of staged.
896
+ export const writtenParameters = ({ loadedAt, committedIn }) => ({
897
+ loaded: loadedAt,
898
+ committed: String(committedIn),
899
+ });
900
+ // Makes the tables of the writers in a transaction of their own, before the
901
+ // commits of a load: SQL Server refuses a second session's CREATE TABLE of
902
+ // one name, so the lock of _warehouse makes two sessions take turns, and no
903
+ // commit holds that lock while it waits for its own target. connection is
904
+ // outside any transaction.
905
+ async function makeWriterTables(connection) {
906
+ const [made] = await connection.query(`SELECT IIF(OBJECT_ID(N'${writersTable}', N'U') IS NULL OR OBJECT_ID(N'${reloadsTable}', N'U') IS NULL, 0, 1) AS [made]`);
907
+ if (made?.made === 1)
908
+ return;
909
+ await transaction(connection, async () => {
910
+ await takeLock(connection, warehouse);
911
+ await connection.query(`IF OBJECT_ID(N'${writersTable}', N'U') IS NULL CREATE TABLE ${writersTable} ([target] nvarchar(128) NOT NULL PRIMARY KEY, [writer] nvarchar(max) NOT NULL);
912
+ IF OBJECT_ID(N'${reloadsTable}', N'U') IS NULL CREATE TABLE ${reloadsTable} ([target] nvarchar(128) NOT NULL PRIMARY KEY, [partitions] nvarchar(max) NOT NULL);`);
913
+ });
914
+ }