@eltjs/elt-postgresql 0.0.0-stage → 0.3.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +359 -2
  3. package/dist/computed-dataset.d.ts +24 -0
  4. package/dist/computed-dataset.js +346 -0
  5. package/dist/identifier.d.ts +2 -0
  6. package/dist/identifier.js +14 -0
  7. package/dist/index.d.ts +9 -0
  8. package/dist/index.js +8 -0
  9. package/dist/postgres-append-writer.d.ts +5 -0
  10. package/dist/postgres-append-writer.js +11 -0
  11. package/dist/postgres-as-of.d.ts +14 -0
  12. package/dist/postgres-as-of.js +51 -0
  13. package/dist/postgres-checkpoint-store.d.ts +10 -0
  14. package/dist/postgres-checkpoint-store.js +83 -0
  15. package/dist/postgres-column.d.ts +53 -0
  16. package/dist/postgres-column.js +302 -0
  17. package/dist/postgres-columns.d.ts +13 -0
  18. package/dist/postgres-columns.js +80 -0
  19. package/dist/postgres-comments.d.ts +6 -0
  20. package/dist/postgres-comments.js +20 -0
  21. package/dist/postgres-deduplicating-writer.d.ts +25 -0
  22. package/dist/postgres-deduplicating-writer.js +135 -0
  23. package/dist/postgres-dependents.d.ts +2 -0
  24. package/dist/postgres-dependents.js +105 -0
  25. package/dist/postgres-destination.d.ts +31 -0
  26. package/dist/postgres-destination.js +87 -0
  27. package/dist/postgres-file-store.d.ts +23 -0
  28. package/dist/postgres-file-store.js +80 -0
  29. package/dist/postgres-fit.d.ts +9 -0
  30. package/dist/postgres-fit.js +25 -0
  31. package/dist/postgres-history.d.ts +17 -0
  32. package/dist/postgres-history.js +141 -0
  33. package/dist/postgres-identifiers.d.ts +8 -0
  34. package/dist/postgres-identifiers.js +26 -0
  35. package/dist/postgres-load.d.ts +20 -0
  36. package/dist/postgres-load.js +73 -0
  37. package/dist/postgres-numbering.d.ts +42 -0
  38. package/dist/postgres-numbering.js +311 -0
  39. package/dist/postgres-overwrite-writer.d.ts +4 -0
  40. package/dist/postgres-overwrite-writer.js +8 -0
  41. package/dist/postgres-privileges.d.ts +7 -0
  42. package/dist/postgres-privileges.js +50 -0
  43. package/dist/postgres-reader-view.d.ts +11 -0
  44. package/dist/postgres-reader-view.js +64 -0
  45. package/dist/postgres-session.d.ts +11 -0
  46. package/dist/postgres-session.js +61 -0
  47. package/dist/postgres-sync-history-schema.d.ts +31 -0
  48. package/dist/postgres-sync-history-schema.js +100 -0
  49. package/dist/postgres-sync-history.d.ts +13 -0
  50. package/dist/postgres-sync-history.js +152 -0
  51. package/dist/postgres-table.d.ts +15 -0
  52. package/dist/postgres-table.js +73 -0
  53. package/dist/postgres-views.d.ts +10 -0
  54. package/dist/postgres-views.js +119 -0
  55. package/dist/postgres-writer.d.ts +39 -0
  56. package/dist/postgres-writer.js +813 -0
  57. package/dist/view.d.ts +22 -0
  58. package/dist/view.js +81 -0
  59. package/package.json +47 -4
@@ -0,0 +1,813 @@
1
+ import { hash } from 'node:crypto';
2
+ import { isDeepStrictEqual } from 'node:util';
3
+ import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, readPartitions, reloadMode, } from '@eltjs/elt';
4
+ import { quote } from './identifier.js';
5
+ import { TargetComments } from './postgres-comments.js';
6
+ import { rebindDependents } from './postgres-dependents.js';
7
+ import { PostgresFileStore } from './postgres-file-store.js';
8
+ import { storedColumns, storedFit } from './postgres-fit.js';
9
+ import { TargetHistory, rowHash, version } from './postgres-history.js';
10
+ import { commitOnce, expireBefore, installNumbering, } from './postgres-numbering.js';
11
+ import { TargetReaderView } from './postgres-reader-view.js';
12
+ import { PostgresSession, lockSchema, makeSchema, } from './postgres-session.js';
13
+ const batchSize = 1000;
14
+ // A batch also flushes once its JSON reaches this size, so rows of large text
15
+ // or binary values are sent a few at a time rather than a thousand at once.
16
+ const batchBytes = 16 * 2 ** 20;
17
+ // Postgres takes at most 1 GB in one message (PQ_LARGE_MESSAGE_LIMIT), less
18
+ // a margin for the batch's brackets and the message framing.
19
+ const rowBytes = 2 ** 30 - 2 ** 20;
20
+ export const seq = '"_elt_seq"';
21
+ export const op = '"_elt_op"';
22
+ // How long swapping in a rebuilt target waits for its readers before the
23
+ // copy gives up until the next run.
24
+ const swapTimeout = '3s';
25
+ export class PostgresWriter extends Writer {
26
+ configuration;
27
+ url;
28
+ schema;
29
+ table;
30
+ // Where the target keeps the rows that its commits replace or remove, and
31
+ // how long it keeps a position readable.
32
+ #history;
33
+ #retention;
34
+ #comments;
35
+ #view;
36
+ constructor(configuration, url, schema, table, retention) {
37
+ super(configuration.stream);
38
+ this.configuration = configuration;
39
+ this.url = url;
40
+ this.schema = schema;
41
+ this.table = table;
42
+ this.#retention = retention;
43
+ this.#history = new TargetHistory(schema, this.#hash);
44
+ const description = describeTarget(configuration, table.columns, (column) => `UUID reference to the source file's original bytes. Join ${new PostgresFileStore(schema, table, column).qualifiedName} on file = this value and concatenate bytes in order of n. NULL when the source file is unavailable.`);
45
+ this.#comments = new TargetComments(description.table, {
46
+ ...description.columns,
47
+ committed_in: 'Number of the commit that last wrote this row: the row of _warehouse._elt_commits with this number tells when, and in which load.',
48
+ });
49
+ this.#view =
50
+ table.readerView &&
51
+ new TargetReaderView(table.readerView, this.qualifiedName, this.#history.qualifiedName, [...table.columns.map(({ name }) => name), 'loaded_at'], new TargetComments(description.table, description.columns));
52
+ if (table.readerView === undefined)
53
+ return;
54
+ const { missing } = description;
55
+ if (missing.length > 0)
56
+ throw new TypeError(`Reader view ${table.readerView.schema}.${table.readerView.name} needs JSON Schema descriptions for ${missing.join(', ')} of stream ${this.stream.name}`);
57
+ }
58
+ // Refuses, before anything is read, stored rows the load cannot keep.
59
+ async inspect(_sql) { }
60
+ get qualifiedName() {
61
+ return `${quote(this.schema)}.${this.table.quotedName}`;
62
+ }
63
+ get createTableSQL() {
64
+ return `CREATE TABLE IF NOT EXISTS ${this.#tableSQL(this.qualifiedName)}`;
65
+ }
66
+ // A table named name with the target's columns.
67
+ #tableSQL(name) {
68
+ if (this.table.columns.length === 0)
69
+ throw new TypeError('Resolve inferred columns before creating a table');
70
+ return `${name} (${this.table.columns.map((column) => column.definition).join(', ')}, "loaded_at" TIMESTAMPTZ NOT NULL, "committed_in" INTEGER NOT NULL, ${version} TEXT)`;
71
+ }
72
+ // What a reload's hidden target needs once it is created, before the first
73
+ // merge fills it, and once it takes the target's name.
74
+ async build(_sql, _into) { }
75
+ async adopt(_sql) { }
76
+ // What a commit writes into each row: its values, when its load began,
77
+ // its number, and the hash of its values.
78
+ get written() {
79
+ return [
80
+ ...this.table.columns.map((column) => column.quotedName),
81
+ '"loaded_at"',
82
+ '"committed_in"',
83
+ version,
84
+ ];
85
+ }
86
+ // The staged values of each row, and what the commit writes beside them.
87
+ get staged() {
88
+ return `${this.table.columns.map((column) => column.quotedName).join(', ')}, $1::text::timestamptz, $2::integer, ${version}`;
89
+ }
90
+ encode(record) {
91
+ return this.table.columns.map((column) => column.encode(record));
92
+ }
93
+ // Only a load that identifies rows by key can remove one: the key's values,
94
+ // at their columns, and null elsewhere.
95
+ deletionRow(_key) {
96
+ throw new TypeError('Only deduplicating loads can apply deletions');
97
+ }
98
+ // An overwrite replaces the target at its first commit.
99
+ get replaces() {
100
+ return false;
101
+ }
102
+ // Which rows of the stage a commit that replaces the target keeps, as a
103
+ // condition on them.
104
+ kept(_stage) {
105
+ return `${op} = 'R'`;
106
+ }
107
+ // Moves the staged operations into the target, or the hidden target of a
108
+ // reload; every row of a run shares its loaded_at, and each row names the
109
+ // commit that wrote it. history keeps the rows of the target that the
110
+ // merge replaces or removes; it is null for a hidden target, whose rows no
111
+ // reader saw.
112
+ async merge(sql, stage, written, into, _history) {
113
+ await sql.unsafe(`INSERT INTO ${into} (${this.written.join(', ')}) SELECT ${this.staged} FROM ${stage} WHERE ${op} = 'R' ORDER BY ${seq}`, [written.loadedAt, written.committedIn]);
114
+ }
115
+ // The ids of the rows of left that no row of right matches, as multisets:
116
+ // the k-th copy of some values on one side matches the k-th copy of the
117
+ // same values on the other, so no key is needed.
118
+ #unmatched(left, right) {
119
+ const numbered = ({ table, id, rows }) => `SELECT ${id} AS "row", ${version} AS "hash", row_number() OVER (PARTITION BY ${version} ORDER BY ${id}) AS "copy" FROM ${table}${rows === undefined ? '' : ` WHERE ${rows}`}`;
120
+ return `SELECT "mine"."row" FROM (${numbered(left)}) AS "mine" LEFT JOIN (${numbered(right)}) AS "theirs" ON "theirs"."hash" = "mine"."hash" AND "theirs"."copy" = "mine"."copy" WHERE "theirs"."row" IS NULL`;
121
+ }
122
+ // Replaces the rows of the target with the records that the stage keeps,
123
+ // by difference: the history keeps each row of the target that no record
124
+ // matches, and the row goes; a row with a match stays as it is, with its
125
+ // commit and its loaded_at; and each record that no row matches enters
126
+ // with this commit. What stays matches records, so it fits the index of
127
+ // the mode, which the records that enter then fit too.
128
+ async #replace(sql, stage, written) {
129
+ const target = { table: this.qualifiedName, id: 'ctid' };
130
+ const staged = { table: stage, id: seq, rows: this.kept(stage) };
131
+ await this.#history.remove(sql, written.committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, staged)})`);
132
+ await this.initialize(sql, false);
133
+ await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.staged} FROM ${stage} WHERE ${seq} IN (${this.#unmatched(staged, target)}) ORDER BY ${seq}`, [written.loadedAt, written.committedIn]);
134
+ }
135
+ // The existing library-owned dedup indexes on this table.
136
+ async dedupIndexes(transaction) {
137
+ const rows = await transaction.unsafe(String.raw `SELECT indexname FROM pg_indexes WHERE schemaname = $1 AND tablename = $2 AND indexname LIKE '\_elt\_dedup\_%'`, [this.schema, this.table.name]);
138
+ return rows.map((row) => String(row.indexname));
139
+ }
140
+ get #writers() {
141
+ return `${quote(this.schema)}."_elt_writers"`;
142
+ }
143
+ // The owner lives beside the table it guards. A dropped table releases it,
144
+ // since nothing it held remains.
145
+ async #refuse(sql, writer) {
146
+ const [writers] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#writers]);
147
+ if (writers?.exists !== true)
148
+ return;
149
+ const [row] = await sql.unsafe(`SELECT "writer" FROM ${this.#writers} WHERE "target" = $1 AND to_regclass(format('%I.%I', $2::text, "target")) IS NOT NULL`, [this.table.name, this.schema]);
150
+ if (row !== undefined && row.writer !== writer)
151
+ throw new TargetOwnedError(this.table.name, String(row.writer), writer);
152
+ }
153
+ // Records the owner with the target's first commit; prepare refused any
154
+ // other.
155
+ async #own(sql, writer) {
156
+ await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#writers} ("target" TEXT PRIMARY KEY, "writer" TEXT NOT NULL)`);
157
+ await sql.unsafe(`DELETE FROM ${this.#writers} WHERE to_regclass(format('%I.%I', $1::text, "target")) IS NULL`, [this.schema]);
158
+ await sql.unsafe(`INSERT INTO ${this.#writers} ("target", "writer") VALUES ($1, $2) ON CONFLICT ("target") DO NOTHING`, [this.table.name, writer]);
159
+ }
160
+ // The target as its first commit, numbered committedIn, makes it: its
161
+ // owner, the table, its history and its comments, and the reader view. The
162
+ // first commits of other copies into the schema make its writers table
163
+ // too, so they take turns.
164
+ async #create(sql, writer, replacing, committedIn) {
165
+ await lockSchema(sql, this.schema);
166
+ await this.#own(sql, writer);
167
+ await this.initialize(sql, replacing);
168
+ await this.#describe(sql, async () => committedIn);
169
+ }
170
+ // The history in the target's shape, the comments of the table, and its
171
+ // reader view, which reads each column of the target from its history too.
172
+ // change gives the number of the commit that changes the target's shape.
173
+ async #describe(sql, change) {
174
+ await this.#history.create(sql, this.qualifiedName, change);
175
+ await this.#comments.write(sql, 'TABLE', this.schema, this.table.name);
176
+ if (this.#view === undefined)
177
+ return;
178
+ await this.#view.describe(sql);
179
+ }
180
+ // The field's values in each of the tables and in the history: while a
181
+ // reload is open, the rows of the target and of its hidden target both
182
+ // still refer to files, and a row of the history keeps its file for a
183
+ // reading as of a position before the commit that replaced it.
184
+ values(sql, tables) {
185
+ const { table, schema } = this;
186
+ return async function* (field) {
187
+ const column = table.columns.find((column) => column.field === field);
188
+ if (column === undefined)
189
+ throw new TypeError(`Unknown target field: ${field}`);
190
+ for (const name of tables()) {
191
+ const present = await sql.unsafe('SELECT 1 FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3', [schema, name, column.name]);
192
+ if (present.length === 0)
193
+ continue;
194
+ for await (const rows of sql
195
+ .unsafe(`SELECT ${column.quotedName} AS value FROM ${quote(schema)}.${quote(name)}`)
196
+ .cursor(batchSize))
197
+ for (const row of rows)
198
+ yield row.value;
199
+ }
200
+ };
201
+ }
202
+ // A clear of a stored table is a load of one commit of its connection.
203
+ async clear({ writer, clock, connection: connector }, committed) {
204
+ await using connection = new PostgresSession(this.url, 'elt');
205
+ await installNumbering(connection.sql);
206
+ await connection.sql.begin(async (transaction) => {
207
+ const [schema] = await transaction.unsafe('SELECT to_regnamespace($1) IS NOT NULL AS "exists"', [quote(this.schema)]);
208
+ if (schema?.exists !== true)
209
+ return;
210
+ const writers = this.#writers;
211
+ await transaction.unsafe(`CREATE TABLE IF NOT EXISTS ${writers} ("target" TEXT PRIMARY KEY, "writer" TEXT NOT NULL)`);
212
+ const [row] = await transaction.unsafe(`SELECT "writer" FROM ${writers} WHERE "target" = $1`, [this.table.name]);
213
+ if (row !== undefined && row.writer !== writer)
214
+ throw new TargetOwnedError(this.table.name, String(row.writer), writer);
215
+ // Emptied, not dropped: views built on the table, such as marts, keep
216
+ // working. The rows it removes are forgotten: each position before the
217
+ // clear expires, so the history keeps none of them, nor their files.
218
+ const [table] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
219
+ if (table?.exists === true) {
220
+ const committedIn = await commitOnce(transaction, connector, new Date(clock.Date.now()), this.#retention);
221
+ await transaction.unsafe(`DELETE FROM ${this.qualifiedName}`);
222
+ await expireBefore(transaction, committedIn, 'clear');
223
+ await this.#history.expire(transaction);
224
+ }
225
+ // Clearing a copy also abandons the reload it left open.
226
+ await transaction.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
227
+ await this.#reloaded(transaction);
228
+ for (const column of this.table.columns.filter((column) => column.storesFile)) {
229
+ const store = new PostgresFileStore(this.schema, this.table, column);
230
+ const [chunks] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [store.qualifiedName]);
231
+ // No row is left in the table or its history, so no stored file
232
+ // stays.
233
+ if (chunks?.exists === true)
234
+ await store.prune(transaction, [this.#history.qualifiedName]);
235
+ }
236
+ await transaction.unsafe(`DELETE FROM ${writers} WHERE "target" = $1`, [
237
+ this.table.name,
238
+ ]);
239
+ });
240
+ await committed?.(this.values(connection.sql, () => [this.table.name, this.#history.name]));
241
+ }
242
+ get #hash() {
243
+ return hash('sha256', this.qualifiedName).slice(0, 40);
244
+ }
245
+ // One staged row as JSON. A row too large to send even alone, past V8's
246
+ // longest string or Postgres's largest message, fails its stream, naming the
247
+ // column that holds the most and the record's key.
248
+ #serialize(row, record) {
249
+ let text;
250
+ try {
251
+ text = JSON.stringify(row);
252
+ }
253
+ catch (error) {
254
+ if (!(error instanceof RangeError))
255
+ throw error;
256
+ }
257
+ if (text !== undefined && Buffer.byteLength(text) <= rowBytes)
258
+ return text;
259
+ const { columns } = this.table;
260
+ const sizes = columns.map((column) => {
261
+ const value = Reflect.get(Object(record), column.field);
262
+ return typeof value === 'string' ? Buffer.byteLength(value) : 0;
263
+ });
264
+ const largestSize = Math.max(...sizes);
265
+ const largest = columns[sizes.indexOf(largestSize)];
266
+ const key = Object.fromEntries(this.stream.primaryKey.map((field) => [
267
+ field,
268
+ Reflect.get(Object(record), field),
269
+ ]));
270
+ throw new RangeError(`A record of ${this.qualifiedName} with key ${JSON.stringify(key)} is too large to stage: its largest value, in column ${largest?.name}, is ${largestSize} bytes, and Postgres takes at most 1 GB in one message`);
271
+ }
272
+ // A reload's hidden target, beside the target and invisible to readers.
273
+ get #hiddenName() {
274
+ return `_elt_next_${this.#hash}`;
275
+ }
276
+ get #hidden() {
277
+ return `${quote(this.schema)}.${quote(this.#hiddenName)}`;
278
+ }
279
+ // Drops the chunks of every stored column that is no longer one of the
280
+ // target's file columns, before the table is replaced: no row of the table
281
+ // that takes its place refers to them.
282
+ async #dropRetiredChunks(sql) {
283
+ const files = new Set(this.table.columns
284
+ .filter(({ storesFile }) => storesFile)
285
+ .map(({ name }) => name));
286
+ for (const { name } of await storedColumns(sql, this.qualifiedName))
287
+ if (!files.has(name))
288
+ await sql.unsafe(`DROP TABLE IF EXISTS ${PostgresFileStore.qualifiedName(this.schema, this.table, name)}`);
289
+ }
290
+ // Swaps a completed reload's hidden target in last, when the target no
291
+ // longer has the stream's shape, so readers of the stored table wait only
292
+ // for the swap, not the reload. A reader that holds it past the timeout
293
+ // fails the copy until the next run. The views that read the target, its
294
+ // history or its reader view read the swapped-in table, and it keeps the
295
+ // target's privileges.
296
+ // Its rows are new to readers at this commit, and the history keeps every
297
+ // row of the target it replaces.
298
+ async #swap(sql, committedIn) {
299
+ await sql.unsafe(`SET LOCAL lock_timeout = '${swapTimeout}'`);
300
+ await sql.unsafe(`UPDATE ${this.#hidden} SET "committed_in" = $1`, [
301
+ committedIn,
302
+ ]);
303
+ await rebindDependents(sql, this.#readable, async () => {
304
+ await this.#view?.drop(sql);
305
+ await this.#dropRetiredChunks(sql);
306
+ const [target] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
307
+ if (target?.exists === true)
308
+ await this.#history.keep(sql, committedIn, this.qualifiedName, `FROM ${this.qualifiedName} AS "_elt_old"`);
309
+ await sql.unsafe(`DROP TABLE IF EXISTS ${this.qualifiedName}`);
310
+ await sql.unsafe(`ALTER TABLE ${this.#hidden} RENAME TO ${this.table.quotedName}`);
311
+ await this.adopt(sql);
312
+ await this.#describe(sql, async () => committedIn);
313
+ });
314
+ }
315
+ // What readers and the views built on them read of the target.
316
+ get #readable() {
317
+ return [
318
+ this.qualifiedName,
319
+ this.#history.qualifiedName,
320
+ ...(this.#view === undefined ? [] : [this.#view.qualifiedName]),
321
+ ];
322
+ }
323
+ // Gives a completed reload's rows to a target that still has the stream's
324
+ // shape, in place and by difference: the history keeps each row of the
325
+ // target that no row of the hidden target matches, and the row goes; a row
326
+ // with a match stays, with its commit and its loaded_at, and the target
327
+ // keeps its indexes, its triggers and what depends on it. Each row of the
328
+ // hidden target that no row matches enters with this commit, and the
329
+ // loaded_at of the load that wrote it.
330
+ async #reloadInPlace(sql, committedIn, target = { table: this.qualifiedName, id: 'ctid' }) {
331
+ const reloaded = { table: this.#hidden, id: 'ctid' };
332
+ await this.#history.remove(sql, committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, reloaded)})`);
333
+ await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.table.columns.map((column) => column.quotedName).join(', ')}, "loaded_at", $1::integer, ${version} FROM ${this.#hidden} WHERE ctid IN (${this.#unmatched(reloaded, target)}) ORDER BY ctid`, [committedIn]);
334
+ await sql.unsafe(`DROP TABLE ${this.#hidden}`);
335
+ }
336
+ // Gives a completed reload of some partitions their rows of the hidden
337
+ // target, in place and by difference: the rows of the target in those
338
+ // partitions are replaced as #reloadInPlace replaces a whole target, and
339
+ // the rows of every other partition stay as they are.
340
+ async #reloadPartitionsInPlace(sql, partitions, committedIn) {
341
+ const scoped = quote(`_elt_reloaded_${this.#hash}`);
342
+ const [within, values] = this.#within(partitions);
343
+ await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${scoped}`);
344
+ await sql.unsafe(`CREATE TEMP TABLE ${scoped} AS SELECT ctid AS "row" FROM ${this.qualifiedName} WHERE ${within}`, values);
345
+ await this.#reloadInPlace(sql, committedIn, {
346
+ table: this.qualifiedName,
347
+ id: 'ctid',
348
+ rows: `ctid IN (SELECT "row" FROM pg_temp.${scoped})`,
349
+ });
350
+ await sql.unsafe(`DROP TABLE pg_temp.${scoped}`);
351
+ }
352
+ // Refuses a cast of column from its stored type from that would change a
353
+ // stored value, such as double precision 1.5 to bigint 2, before the ALTER,
354
+ // as a STRICT SQLite table refuses it: each value that is not null must
355
+ // cast back to itself. A cast from text is the parse the load applies to
356
+ // each value of the new type, so it is trusted, and a value it cannot parse
357
+ // fails the ALTER.
358
+ async #refuseInexactCast(sql, column, from) {
359
+ if (from === 'text' || from === 'text[]')
360
+ return;
361
+ const { quotedName: name, storageType: to } = column;
362
+ let changes;
363
+ try {
364
+ changes = await sql.unsafe(`SELECT 1 FROM ${this.qualifiedName} WHERE ${name} IS NOT NULL AND ${name}::${to}::${from} IS DISTINCT FROM ${name} LIMIT 1`);
365
+ }
366
+ catch (cause) {
367
+ throw new TypeError(`column ${name} does not cast to ${to}: ${errorMessage(cause)}`, { cause });
368
+ }
369
+ if (changes.length > 0)
370
+ throw new TypeError(`column ${name} holds a value that a cast to ${to} would change`);
371
+ }
372
+ // Brings a stored table the stream no longer fits to its shape in place,
373
+ // keeping every row, as Airbyte's Postgres destination does: a column the
374
+ // stream added is added nullable, one it dropped is dropped, a changed type
375
+ // is cast, and NOT NULL and the primary key follow the target's explicit
376
+ // key columns. A value that will not cast exactly, or rows that repeat a
377
+ // new key, fail the load, naming the column, and the transaction leaves the
378
+ // table as it was. Readers wait for the change as they wait for a swap, and
379
+ // the views that read the target, its history or its reader view read it
380
+ // again after the change.
381
+ // A row with a value in a column that the change drops or casts loses or
382
+ // changes the value, so it is a new version, in a commit of the load of its
383
+ // own: the history keeps the row as it was. Its row hash counted the old
384
+ // value, so it becomes NULL, which matches no record until a commit writes
385
+ // the row again. A column that the change adds is NULL in each row, which a
386
+ // row hash does not count, so it makes no new version.
387
+ // A change of type that the history cannot follow by a widening makes the
388
+ // change a commit too, which expires each position before it.
389
+ async #evolve(sql, stage, load) {
390
+ let changedIn;
391
+ const change = async () => (changedIn ??= await load.number(sql));
392
+ await sql.unsafe(`SET LOCAL lock_timeout = '${swapTimeout}'`);
393
+ // Postgres keeps the primary key as a constraint apart from NOT NULL, and
394
+ // refuses to drop NOT NULL from a column it covers.
395
+ const storedKey = await sql.unsafe("SELECT c.conname, a.attname FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY (c.conkey) WHERE c.conrelid = to_regclass($1) AND c.contype = 'p'", [this.qualifiedName]);
396
+ const key = this.table.columns.filter((column) => column.isPrimaryKey);
397
+ const rekeyed = !isDeepStrictEqual(storedKey.map(({ attname }) => attname).toSorted(), key.map(({ name }) => name).toSorted());
398
+ const stored = new Map((await storedColumns(sql, this.qualifiedName)).map((column) => [
399
+ column.name,
400
+ column,
401
+ ]));
402
+ const types = new Map((await storedColumns(sql, `pg_temp.${stage}`)).map(({ name, type }) => [
403
+ name,
404
+ type,
405
+ ]));
406
+ const kept = new Set(['loaded_at', 'committed_in', '_elt_row_hash']);
407
+ const declared = new Set(this.table.columns.map(({ name }) => name));
408
+ const changed = [...stored.values()]
409
+ .filter(({ name, type }) => !kept.has(name) && (!declared.has(name) || types.get(name) !== type))
410
+ .map(({ name }) => `${quote(name)} IS NOT NULL`)
411
+ .join(' OR ');
412
+ await rebindDependents(sql, this.#readable, async () => {
413
+ await this.#view?.drop(sql);
414
+ if (changed !== '') {
415
+ const committedIn = await change();
416
+ await this.#history.keep(sql, committedIn, this.qualifiedName, `FROM ${this.qualifiedName} AS "_elt_old" WHERE ${changed}`);
417
+ await sql.unsafe(`UPDATE ${this.qualifiedName} SET "committed_in" = $1, "loaded_at" = $2::text::timestamptz, ${version} = NULL WHERE ${changed}`, [committedIn, load.loadedAt]);
418
+ }
419
+ try {
420
+ await this.#dropRetiredChunks(sql);
421
+ const [constraint] = storedKey;
422
+ if (rekeyed && constraint !== undefined)
423
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} DROP CONSTRAINT ${quote(constraint.conname)}`);
424
+ for (const column of this.table.columns) {
425
+ kept.add(column.name);
426
+ const type = types.get(column.name);
427
+ const existing = stored.get(column.name);
428
+ if (existing === undefined) {
429
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD COLUMN ${column.quotedName} ${type}`);
430
+ continue;
431
+ }
432
+ if (existing.type !== type) {
433
+ await this.#refuseInexactCast(sql, column, existing.type);
434
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column.quotedName} TYPE ${type} USING ${column.quotedName}::${type}`);
435
+ }
436
+ if (existing.required !== column.isPrimaryKey)
437
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column.quotedName} ${column.isPrimaryKey ? 'SET' : 'DROP'} NOT NULL`);
438
+ }
439
+ for (const name of stored.keys())
440
+ if (!kept.has(name))
441
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} DROP COLUMN ${quote(name)}`);
442
+ if (rekeyed && key.length > 0)
443
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD PRIMARY KEY (${key.map(({ quotedName }) => quotedName).join(', ')})`);
444
+ }
445
+ catch (cause) {
446
+ throw new TypeError(`The rows stored in ${this.qualifiedName} do not fit the new shape of stream ${this.stream.name}: ${errorMessage(cause)}. Clear the copy to load it again, or change the stream so they fit.`, { cause });
447
+ }
448
+ await this.#describe(sql, change);
449
+ });
450
+ }
451
+ // The rows of one partition, or every row when the stream is not
452
+ // partitioned, as a WHERE clause over the target's or the stage's columns.
453
+ #scope(partition) {
454
+ if (partition === null)
455
+ return ['', []];
456
+ const [condition, values] = this.#within([partition]);
457
+ return [` WHERE ${condition}`, values];
458
+ }
459
+ // The rows of any of the partitions, as a condition over the target's or
460
+ // the stage's columns, with one JSON parameter for each partition: NULL for
461
+ // a row that does not name its partition, such as a staged DELETE whose
462
+ // key leaves the partition out.
463
+ #within(partitions) {
464
+ const each = partitions.map((partition, index) => this.#partition(partition, `$${index + 1}::text::json`));
465
+ return [
466
+ each.length === 0
467
+ ? 'FALSE'
468
+ : each.map(([condition]) => `(${condition})`).join(' OR '),
469
+ each.map(([, value]) => value),
470
+ ];
471
+ }
472
+ #partition(partition, parameter) {
473
+ const columns = Object.keys(partition).map((field) => {
474
+ const column = this.table.columns.find((column) => column.field === field);
475
+ if (column === undefined)
476
+ throw new TypeError(`Resetting a partition requires destination column ${field}`);
477
+ return column;
478
+ });
479
+ return [
480
+ columns
481
+ .map((column, index) => `${column.quotedName} = ${column.valueFrom(parameter, index)}`)
482
+ .join(' AND '),
483
+ JSON.stringify(columns.map((column) => column.encode(partition))),
484
+ ];
485
+ }
486
+ get #reloads() {
487
+ return `${quote(this.schema)}."_elt_reloads"`;
488
+ }
489
+ // The partitions that an open reload of the target reloads, in the
490
+ // writer's own table beside it, written in the commit that opens or
491
+ // continues the reload, so a run that resumes the reload continues it. A
492
+ // reload of the whole stream has no row.
493
+ async #reloading(sql) {
494
+ const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
495
+ if (made?.exists !== true)
496
+ return wholeStream;
497
+ const [row] = await sql.unsafe(`SELECT "partitions" FROM ${this.#reloads} WHERE "target" = $1`, [this.table.name]);
498
+ return row === undefined
499
+ ? wholeStream
500
+ : new SomePartitions(readPartitions(this.stream, JSON.parse(row.partitions)));
501
+ }
502
+ async #reload(sql, reloads) {
503
+ if (!(reloads instanceof SomePartitions)) {
504
+ await this.#reloaded(sql);
505
+ return;
506
+ }
507
+ // The first reloads of the schema's tables make the table together.
508
+ await lockSchema(sql, this.schema);
509
+ await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#reloads} ("target" TEXT PRIMARY KEY, "partitions" TEXT NOT NULL)`);
510
+ await sql.unsafe(`INSERT INTO ${this.#reloads} ("target", "partitions") VALUES ($1, $2) ON CONFLICT ("target") DO UPDATE SET "partitions" = excluded."partitions"`, [this.table.name, JSON.stringify(reloads.partitions)]);
511
+ }
512
+ async #reloaded(sql) {
513
+ const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
514
+ if (made?.exists === true)
515
+ await sql.unsafe(`DELETE FROM ${this.#reloads} WHERE "target" = $1`, [
516
+ this.table.name,
517
+ ]);
518
+ }
519
+ // One stream's load. Operations wait in a session-private TEMP stage, so no
520
+ // other session sees them and a crash leaves nothing behind. prepare only
521
+ // refuses, evolves a stored table the stream no longer fits unless the load
522
+ // overwrites it, and stages, so a run that fails before a commit leaves no
523
+ // rows behind.
524
+ // Each commit is one transaction that merges the stage, with the result of
525
+ // applying its operations one at a time, into the target, or into a
526
+ // reload's hidden target until complete() gives its rows to the target.
527
+ // The first commit also creates the target, or the hidden target of a new
528
+ // reload.
529
+ async prepare(load, { writer, reloading }) {
530
+ const { sql } = load;
531
+ const stage = quote(`_elt_stage_${this.#hash}`);
532
+ // The rows of a commit that belong to the partitions a reload reloads.
533
+ const reloadStage = quote(`_elt_reload_stage_${this.#hash}`);
534
+ const staging = (table) => `CREATE TEMP TABLE ${table} (${seq} BIGINT PRIMARY KEY, ${op} TEXT NOT NULL, ${this.table.columns.map((column) => `${column.quotedName} ${column.storageType}`).join(', ')}, ${version} TEXT)`;
535
+ // What an open reload reloads; the whole stream until a RESET of a
536
+ // partition opens one.
537
+ let reloads = wholeStream;
538
+ const stores = this.table.columns
539
+ .filter((column) => column.storesFile)
540
+ .map((column) => new PostgresFileStore(this.schema, this.table, column));
541
+ let mode = await load.transaction(async (sql) => {
542
+ await this.#refuse(sql, writer);
543
+ await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${stage}`);
544
+ await sql.unsafe(staging(stage));
545
+ await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${reloadStage}`);
546
+ await sql.unsafe(staging(reloadStage));
547
+ for (const store of stores)
548
+ await store.stage(sql);
549
+ const target = await storedFit(sql, this.table, this.qualifiedName, stage);
550
+ const hidden = await storedFit(sql, this.table, this.#hidden, stage);
551
+ let decided = reloadMode({
552
+ reloading,
553
+ destinationSyncMode: this.configuration.destinationSyncMode,
554
+ target,
555
+ hidden,
556
+ });
557
+ // A hidden target no reload continues is a leftover readers never saw.
558
+ if (decided === 'continue')
559
+ reloads = await this.#reloading(sql);
560
+ else {
561
+ await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
562
+ await this.#reloaded(sql);
563
+ }
564
+ await this.#view?.refuse(sql, target === 'stale');
565
+ if (decided === 'evolve') {
566
+ await this.#evolve(sql, stage, load);
567
+ decided = 'load';
568
+ }
569
+ if (decided === 'load' && !this.replaces)
570
+ await this.inspect(sql);
571
+ return decided;
572
+ });
573
+ const fresh = mode === 'create' || mode === 'reload';
574
+ // The reload as the last commit left it, which a discard goes back to.
575
+ let committed = {
576
+ mode,
577
+ reloads,
578
+ };
579
+ const open = () => mode === 'reload' || mode === 'continue';
580
+ const into = () => (open() ? this.#hidden : this.qualifiedName);
581
+ const tables = () => [
582
+ ...(open() ? [this.qualifiedName, this.#hidden] : [this.qualifiedName]),
583
+ this.#history.qualifiedName,
584
+ ];
585
+ const { columns } = this.table;
586
+ // One JSON parameter per batch, cast back per column: no bind-parameter
587
+ // limit. Declared text so the driver sends it as given.
588
+ const insert = `INSERT INTO ${stage} (${seq}, ${op}, ${columns.map((column) => column.quotedName).join(', ')}, ${version}) SELECT (row->>0)::bigint, row->>1, ${columns.map((column, index) => column.valueFrom('row', index + 2)).join(', ')}, row->>${columns.length + 2} FROM json_array_elements($1::text::json) AS row`;
589
+ // Each staged row as its JSON, and how many bytes they add up to.
590
+ let pending = [];
591
+ let pendingBytes = 0;
592
+ let next = 0;
593
+ const flush = async () => {
594
+ if (pending.length === 0)
595
+ return;
596
+ const rows = pending;
597
+ pending = [];
598
+ pendingBytes = 0;
599
+ await sql.unsafe(insert, [`[${rows.join(',')}]`]);
600
+ };
601
+ // Partitions the stage dropped, which the next commit empties in the
602
+ // table it merges into.
603
+ let resets = [];
604
+ const drop = async () => {
605
+ pending = [];
606
+ pendingBytes = 0;
607
+ resets = [];
608
+ await sql.unsafe(`TRUNCATE ${stage}`);
609
+ for (const store of stores)
610
+ await store.discard(sql);
611
+ };
612
+ // The first commit claims the target: it owns the hidden target of a
613
+ // reload, or creates or adopts the table, and before its transaction it
614
+ // makes the schema of a new target. Later commits find it claimed. claim
615
+ // gives whether the commit replaces the rows of the target with the
616
+ // stage's.
617
+ const unclaimed = {
618
+ before: async () => {
619
+ if (mode === 'create')
620
+ await makeSchema(load.sql, this.schema);
621
+ },
622
+ claim: async (sql, committedIn) => {
623
+ if (open()) {
624
+ await this.#own(sql, writer);
625
+ return false;
626
+ }
627
+ await this.#create(sql, writer, this.replaces, committedIn);
628
+ return this.replaces;
629
+ },
630
+ };
631
+ const claimed = {
632
+ before: async () => { },
633
+ claim: async () => false,
634
+ };
635
+ let claiming = unclaimed;
636
+ return {
637
+ fresh,
638
+ get reloading() {
639
+ return open();
640
+ },
641
+ values: this.values(sql, () => [
642
+ ...(open() ? [this.table.name, this.#hiddenName] : [this.table.name]),
643
+ this.#history.name,
644
+ ]),
645
+ apply: async (operation) => {
646
+ if (operation.type === 'RESET') {
647
+ await flush();
648
+ const { partition } = operation;
649
+ if (partition === null) {
650
+ // The whole stream starts over in a new hidden target.
651
+ await drop();
652
+ mode = 'reload';
653
+ reloads = wholeStream;
654
+ return;
655
+ }
656
+ const [rows, values] = this.#scope(partition);
657
+ await sql.unsafe(`DELETE FROM ${stage}${rows}`, values);
658
+ resets.push(partition);
659
+ // The partition starts over in a hidden target, and readers see its
660
+ // rows as they were until the stream ends. An overwrite replaces
661
+ // the whole target anyway.
662
+ if (this.replaces)
663
+ return;
664
+ if (mode === 'load') {
665
+ mode = 'reload';
666
+ reloads = new SomePartitions([partition]);
667
+ }
668
+ else if (open() && reloads instanceof SomePartitions)
669
+ reloads = reloads.with(partition);
670
+ return;
671
+ }
672
+ // Each file is read here, before the source advances past it. The
673
+ // row keeps its id, and the row hash its content: each load gives a
674
+ // file a new id, so a row whose file is the same matches.
675
+ let data = operation.type === 'RECORD' ? operation.data : undefined;
676
+ const contents = new Map();
677
+ for (const store of stores) {
678
+ const content = Reflect.get(Object(data), store.column.field);
679
+ if (content instanceof FileContent) {
680
+ const { file, sha256 } = await store.save(sql, content);
681
+ data = { ...Object(data), [store.column.field]: file };
682
+ contents.set(store.column, sha256);
683
+ }
684
+ }
685
+ const row = operation.type === 'RECORD'
686
+ ? this.encode(data)
687
+ : this.deletionRow(operation.key);
688
+ const hashed = operation.type === 'RECORD'
689
+ ? rowHash(columns, columns.map((column, index) => contents.get(column) ?? row[index] ?? null))
690
+ : null;
691
+ const text = this.#serialize([next++, operation.type === 'RECORD' ? 'R' : 'D', ...row, hashed], data);
692
+ const bytes = Buffer.byteLength(text);
693
+ if (pendingBytes + bytes > batchBytes)
694
+ await flush();
695
+ pending.push(text);
696
+ pendingBytes += bytes;
697
+ if (pending.length >= batchSize || pendingBytes >= batchBytes)
698
+ await flush();
699
+ },
700
+ commit: async () => {
701
+ await flush();
702
+ await claiming.before();
703
+ await load.commit(async (sql, committedIn) => {
704
+ if (mode === 'reload') {
705
+ await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
706
+ await sql.unsafe(`CREATE TABLE ${this.#tableSQL(this.#hidden)}`);
707
+ await this.build(sql, this.#hidden);
708
+ }
709
+ if (open())
710
+ await this.#reload(sql, reloads);
711
+ const replacing = await claiming.claim(sql, committedIn);
712
+ // The rows of a hidden target were never shown, so only the target
713
+ // keeps the rows a commit replaces.
714
+ const history = open() ? null : this.#history;
715
+ for (const partition of resets) {
716
+ const [rows, values] = this.#scope(partition);
717
+ if (history === null)
718
+ await sql.unsafe(`DELETE FROM ${into()}${rows}`, values);
719
+ else
720
+ await history.remove(sql, committedIn, into(), rows, values);
721
+ }
722
+ for (const store of stores)
723
+ await store.publish(sql);
724
+ const written = { loadedAt: load.loadedAt, committedIn };
725
+ if (replacing)
726
+ await this.#replace(sql, stage, written);
727
+ else if (open() && reloads instanceof SomePartitions) {
728
+ // The rows of the reloaded partitions go to the hidden target,
729
+ // the others to the target. A DELETE whose key does not name
730
+ // its partition goes to both, so the reload cannot bring back a
731
+ // row that the source deleted.
732
+ const [within, values] = this.#within(reloads.partitions);
733
+ await sql.unsafe(`INSERT INTO ${reloadStage} SELECT * FROM ${stage} WHERE (${within}) IS TRUE OR (${op} = 'D' AND (${within}) IS NULL)`, values);
734
+ await sql.unsafe(`DELETE FROM ${stage} WHERE (${within}) IS TRUE`, values);
735
+ await this.merge(sql, stage, written, this.qualifiedName, this.#history);
736
+ await this.merge(sql, reloadStage, written, this.#hidden, null);
737
+ await sql.unsafe(`TRUNCATE ${reloadStage}`);
738
+ }
739
+ else
740
+ await this.merge(sql, stage, written, into(), history);
741
+ await sql.unsafe(`TRUNCATE ${stage}`);
742
+ await this.#history.expire(sql);
743
+ for (const store of stores)
744
+ await store.prune(sql, tables());
745
+ });
746
+ claiming = claimed;
747
+ // Its first commit made the hidden target, so the reload continues.
748
+ if (mode === 'reload')
749
+ mode = 'continue';
750
+ committed = { mode, reloads };
751
+ resets = [];
752
+ },
753
+ complete: async () => {
754
+ if (!open())
755
+ return;
756
+ await load.commit(async (sql, committedIn) => {
757
+ const target = await storedFit(sql, this.table, this.qualifiedName, stage);
758
+ // A reload of partitions keeps the target's shape: prepare evolves
759
+ // a stored target before it loads.
760
+ if (reloads instanceof SomePartitions && target === 'fits')
761
+ await this.#reloadPartitionsInPlace(sql, reloads.partitions, committedIn);
762
+ else if (reloads instanceof SomePartitions)
763
+ throw new TypeError(`${this.qualifiedName} changed its shape while some of its partitions reloaded; clear the copy to load it again`);
764
+ else if (target === 'fits')
765
+ await this.#reloadInPlace(sql, committedIn);
766
+ else
767
+ await this.#swap(sql, committedIn);
768
+ await this.#reloaded(sql);
769
+ await this.#history.expire(sql);
770
+ for (const store of stores)
771
+ await store.prune(sql, [
772
+ this.qualifiedName,
773
+ this.#history.qualifiedName,
774
+ ]);
775
+ });
776
+ mode = 'load';
777
+ reloads = wholeStream;
778
+ committed = { mode, reloads };
779
+ },
780
+ // A RESET that no commit took is undone with the operations after it:
781
+ // the source's checkpoint does not know it.
782
+ discard: async () => {
783
+ await drop();
784
+ ({ mode, reloads } = committed);
785
+ },
786
+ [Symbol.asyncDispose]: async () => {
787
+ await drop();
788
+ await sql.unsafe(`DROP TABLE pg_temp.${stage}`);
789
+ await sql.unsafe(`DROP TABLE pg_temp.${reloadStage}`);
790
+ for (const store of stores)
791
+ await store.unstage(sql);
792
+ },
793
+ };
794
+ }
795
+ }
796
+ // What an open reload reloads: the whole stream, or some partitions of it,
797
+ // while the others load into the target as before, as Delta Lake's
798
+ // replaceWhere replaces only the rows that its condition selects.
799
+ class WholeStream {
800
+ }
801
+ class SomePartitions {
802
+ partitions;
803
+ constructor(partitions) {
804
+ this.partitions = Object.freeze([...partitions]);
805
+ }
806
+ // These partitions and partition, which a later RESET reloads too.
807
+ with(partition) {
808
+ return this.partitions.some((other) => isDeepStrictEqual(other, partition))
809
+ ? this
810
+ : new SomePartitions([...this.partitions, partition]);
811
+ }
812
+ }
813
+ const wholeStream = new WholeStream();