@eltjs/elt-postgresql 0.0.0-stage → 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +359 -2
  3. package/dist/computed-dataset.d.ts +24 -0
  4. package/dist/computed-dataset.js +346 -0
  5. package/dist/identifier.d.ts +2 -0
  6. package/dist/identifier.js +14 -0
  7. package/dist/index.d.ts +9 -0
  8. package/dist/index.js +8 -0
  9. package/dist/postgres-append-writer.d.ts +5 -0
  10. package/dist/postgres-append-writer.js +11 -0
  11. package/dist/postgres-as-of.d.ts +14 -0
  12. package/dist/postgres-as-of.js +51 -0
  13. package/dist/postgres-checkpoint-store.d.ts +10 -0
  14. package/dist/postgres-checkpoint-store.js +83 -0
  15. package/dist/postgres-column.d.ts +53 -0
  16. package/dist/postgres-column.js +302 -0
  17. package/dist/postgres-columns.d.ts +13 -0
  18. package/dist/postgres-columns.js +80 -0
  19. package/dist/postgres-comments.d.ts +6 -0
  20. package/dist/postgres-comments.js +20 -0
  21. package/dist/postgres-deduplicating-writer.d.ts +25 -0
  22. package/dist/postgres-deduplicating-writer.js +135 -0
  23. package/dist/postgres-dependents.d.ts +2 -0
  24. package/dist/postgres-dependents.js +105 -0
  25. package/dist/postgres-destination.d.ts +31 -0
  26. package/dist/postgres-destination.js +87 -0
  27. package/dist/postgres-file-store.d.ts +23 -0
  28. package/dist/postgres-file-store.js +80 -0
  29. package/dist/postgres-fit.d.ts +9 -0
  30. package/dist/postgres-fit.js +25 -0
  31. package/dist/postgres-history.d.ts +17 -0
  32. package/dist/postgres-history.js +141 -0
  33. package/dist/postgres-identifiers.d.ts +8 -0
  34. package/dist/postgres-identifiers.js +26 -0
  35. package/dist/postgres-load.d.ts +20 -0
  36. package/dist/postgres-load.js +73 -0
  37. package/dist/postgres-numbering.d.ts +42 -0
  38. package/dist/postgres-numbering.js +311 -0
  39. package/dist/postgres-overwrite-writer.d.ts +4 -0
  40. package/dist/postgres-overwrite-writer.js +8 -0
  41. package/dist/postgres-privileges.d.ts +7 -0
  42. package/dist/postgres-privileges.js +50 -0
  43. package/dist/postgres-reader-view.d.ts +11 -0
  44. package/dist/postgres-reader-view.js +64 -0
  45. package/dist/postgres-session.d.ts +11 -0
  46. package/dist/postgres-session.js +61 -0
  47. package/dist/postgres-sync-history-schema.d.ts +31 -0
  48. package/dist/postgres-sync-history-schema.js +100 -0
  49. package/dist/postgres-sync-history.d.ts +13 -0
  50. package/dist/postgres-sync-history.js +152 -0
  51. package/dist/postgres-table.d.ts +15 -0
  52. package/dist/postgres-table.js +73 -0
  53. package/dist/postgres-views.d.ts +10 -0
  54. package/dist/postgres-views.js +119 -0
  55. package/dist/postgres-writer.d.ts +39 -0
  56. package/dist/postgres-writer.js +668 -0
  57. package/dist/view.d.ts +22 -0
  58. package/dist/view.js +81 -0
  59. package/package.json +47 -4
@@ -0,0 +1,668 @@
1
+ import { hash } from 'node:crypto';
2
+ import { isDeepStrictEqual } from 'node:util';
3
+ import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, reloadMode, } from '@eltjs/elt';
4
+ import { quote } from './identifier.js';
5
+ import { TargetComments } from './postgres-comments.js';
6
+ import { rebindDependents } from './postgres-dependents.js';
7
+ import { PostgresFileStore } from './postgres-file-store.js';
8
+ import { storedColumns, storedFit } from './postgres-fit.js';
9
+ import { TargetHistory, rowHash, version } from './postgres-history.js';
10
+ import { commitOnce, expireBefore, installNumbering, } from './postgres-numbering.js';
11
+ import { TargetReaderView } from './postgres-reader-view.js';
12
+ import { PostgresSession, lockSchema, makeSchema } from './postgres-session.js';
13
+ const batchSize = 1000;
14
+ // A batch also flushes once its JSON reaches this size, so rows of large text
15
+ // or binary values are sent a few at a time rather than a thousand at once.
16
+ const batchBytes = 16 * 2 ** 20;
17
+ // Postgres takes at most 1 GB in one message (PQ_LARGE_MESSAGE_LIMIT), less
18
+ // a margin for the batch's brackets and the message framing.
19
+ const rowBytes = 2 ** 30 - 2 ** 20;
20
+ export const seq = '"_elt_seq"';
21
+ export const op = '"_elt_op"';
22
+ // How long swapping in a rebuilt target waits for its readers before the
23
+ // copy gives up until the next run.
24
+ const swapTimeout = '3s';
25
+ export class PostgresWriter extends Writer {
26
+ configuration;
27
+ url;
28
+ schema;
29
+ table;
30
+ // Where the target keeps the rows that its commits replace or remove, and
31
+ // how long it keeps a position readable.
32
+ #history;
33
+ #retention;
34
+ #comments;
35
+ #view;
36
+ constructor(configuration, url, schema, table, retention) {
37
+ super(configuration.stream);
38
+ this.configuration = configuration;
39
+ this.url = url;
40
+ this.schema = schema;
41
+ this.table = table;
42
+ this.#retention = retention;
43
+ this.#history = new TargetHistory(schema, this.#hash);
44
+ const description = describeTarget(configuration, table.columns, (column) => `UUID reference to the source file's original bytes. Join ${new PostgresFileStore(schema, table, column).qualifiedName} on file = this value and concatenate bytes in order of n. NULL when the source file is unavailable.`);
45
+ this.#comments = new TargetComments(description.table, {
46
+ ...description.columns,
47
+ committed_in: 'Number of the commit that last wrote this row: the row of _warehouse._elt_commits with this number tells when, and in which load.',
48
+ });
49
+ this.#view =
50
+ table.readerView &&
51
+ new TargetReaderView(table.readerView, this.qualifiedName, this.#history.qualifiedName, [...table.columns.map(({ name }) => name), 'loaded_at'], new TargetComments(description.table, description.columns));
52
+ if (table.readerView === undefined)
53
+ return;
54
+ const { missing } = description;
55
+ if (missing.length > 0)
56
+ throw new TypeError(`Reader view ${table.readerView.schema}.${table.readerView.name} needs JSON Schema descriptions for ${missing.join(', ')} of stream ${this.stream.name}`);
57
+ }
58
+ // Refuses, before anything is read, stored rows the load cannot keep.
59
+ async inspect(_sql) { }
60
+ get qualifiedName() {
61
+ return `${quote(this.schema)}.${this.table.quotedName}`;
62
+ }
63
+ get createTableSQL() {
64
+ return `CREATE TABLE IF NOT EXISTS ${this.#tableSQL(this.qualifiedName)}`;
65
+ }
66
+ // A table named name with the target's columns.
67
+ #tableSQL(name) {
68
+ if (this.table.columns.length === 0)
69
+ throw new TypeError('Resolve inferred columns before creating a table');
70
+ return `${name} (${this.table.columns.map((column) => column.definition).join(', ')}, "loaded_at" TIMESTAMPTZ NOT NULL, "committed_in" INTEGER NOT NULL, ${version} TEXT)`;
71
+ }
72
+ // What a reload's hidden target needs once it is created, before the first
73
+ // merge fills it, and once it takes the target's name.
74
+ async build(_sql, _into) { }
75
+ async adopt(_sql) { }
76
+ // What a commit writes into each row: its values, when its load began,
77
+ // its number, and the hash of its values.
78
+ get written() {
79
+ return [
80
+ ...this.table.columns.map((column) => column.quotedName),
81
+ '"loaded_at"',
82
+ '"committed_in"',
83
+ version,
84
+ ];
85
+ }
86
+ // The staged values of each row, and what the commit writes beside them.
87
+ get staged() {
88
+ return `${this.table.columns.map((column) => column.quotedName).join(', ')}, $1::text::timestamptz, $2::integer, ${version}`;
89
+ }
90
+ encode(record) {
91
+ return this.table.columns.map((column) => column.encode(record));
92
+ }
93
+ // Only a load that identifies rows by key can remove one: the key's values,
94
+ // at their columns, and null elsewhere.
95
+ deletionRow(_key) {
96
+ throw new TypeError('Only deduplicating loads can apply deletions');
97
+ }
98
+ // An overwrite replaces the target at its first commit.
99
+ get replaces() {
100
+ return false;
101
+ }
102
+ // Which rows of the stage a commit that replaces the target keeps, as a
103
+ // condition on them.
104
+ kept(_stage) {
105
+ return `${op} = 'R'`;
106
+ }
107
+ // Moves the staged operations into the target, or the hidden target of a
108
+ // reload; every row of a run shares its loaded_at, and each row names the
109
+ // commit that wrote it. history keeps the rows of the target that the
110
+ // merge replaces or removes; it is null for a hidden target, whose rows no
111
+ // reader saw.
112
+ async merge(sql, stage, written, into, _history) {
113
+ await sql.unsafe(`INSERT INTO ${into} (${this.written.join(', ')}) SELECT ${this.staged} FROM ${stage} WHERE ${op} = 'R' ORDER BY ${seq}`, [written.loadedAt, written.committedIn]);
114
+ }
115
+ // The ids of the rows of left that no row of right matches, as multisets:
116
+ // the k-th copy of some values on one side matches the k-th copy of the
117
+ // same values on the other, so no key is needed.
118
+ #unmatched(left, right) {
119
+ const numbered = ({ table, id, rows }) => `SELECT ${id} AS "row", ${version} AS "hash", row_number() OVER (PARTITION BY ${version} ORDER BY ${id}) AS "copy" FROM ${table}${rows === undefined ? '' : ` WHERE ${rows}`}`;
120
+ return `SELECT "mine"."row" FROM (${numbered(left)}) AS "mine" LEFT JOIN (${numbered(right)}) AS "theirs" ON "theirs"."hash" = "mine"."hash" AND "theirs"."copy" = "mine"."copy" WHERE "theirs"."row" IS NULL`;
121
+ }
122
+ // Replaces the rows of the target with the records that the stage keeps,
123
+ // by difference: the history keeps each row of the target that no record
124
+ // matches, and the row goes; a row with a match stays as it is, with its
125
+ // commit and its loaded_at; and each record that no row matches enters
126
+ // with this commit. What stays matches records, so it fits the index of
127
+ // the mode, which the records that enter then fit too.
128
+ async #replace(sql, stage, written) {
129
+ const target = { table: this.qualifiedName, id: 'ctid' };
130
+ const staged = { table: stage, id: seq, rows: this.kept(stage) };
131
+ await this.#history.remove(sql, written.committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, staged)})`);
132
+ await this.initialize(sql, false);
133
+ await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.staged} FROM ${stage} WHERE ${seq} IN (${this.#unmatched(staged, target)}) ORDER BY ${seq}`, [written.loadedAt, written.committedIn]);
134
+ }
135
+ // The existing library-owned dedup indexes on this table.
136
+ async dedupIndexes(transaction) {
137
+ const rows = await transaction.unsafe(String.raw `SELECT indexname FROM pg_indexes WHERE schemaname = $1 AND tablename = $2 AND indexname LIKE '\_elt\_dedup\_%'`, [this.schema, this.table.name]);
138
+ return rows.map((row) => String(row.indexname));
139
+ }
140
+ get #writers() {
141
+ return `${quote(this.schema)}."_elt_writers"`;
142
+ }
143
+ // The owner lives beside the table it guards. A dropped table releases it,
144
+ // since nothing it held remains.
145
+ async #refuse(sql, writer) {
146
+ const [writers] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#writers]);
147
+ if (writers?.exists !== true)
148
+ return;
149
+ const [row] = await sql.unsafe(`SELECT "writer" FROM ${this.#writers} WHERE "target" = $1 AND to_regclass(format('%I.%I', $2::text, "target")) IS NOT NULL`, [this.table.name, this.schema]);
150
+ if (row !== undefined && row.writer !== writer)
151
+ throw new TargetOwnedError(this.table.name, String(row.writer), writer);
152
+ }
153
+ // Records the owner with the target's first commit; prepare refused any
154
+ // other.
155
+ async #own(sql, writer) {
156
+ await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#writers} ("target" TEXT PRIMARY KEY, "writer" TEXT NOT NULL)`);
157
+ await sql.unsafe(`DELETE FROM ${this.#writers} WHERE to_regclass(format('%I.%I', $1::text, "target")) IS NULL`, [this.schema]);
158
+ await sql.unsafe(`INSERT INTO ${this.#writers} ("target", "writer") VALUES ($1, $2) ON CONFLICT ("target") DO NOTHING`, [this.table.name, writer]);
159
+ }
160
+ // The target as its first commit, numbered committedIn, makes it: its
161
+ // owner, the table, its history and its comments, and the reader view. The
162
+ // first commits of other copies into the schema make its writers table
163
+ // too, so they take turns.
164
+ async #create(sql, writer, replacing, committedIn) {
165
+ await lockSchema(sql, this.schema);
166
+ await this.#own(sql, writer);
167
+ await this.initialize(sql, replacing);
168
+ await this.#describe(sql, async () => committedIn);
169
+ }
170
+ // The history in the target's shape, the comments of the table, and its
171
+ // reader view, which reads each column of the target from its history too.
172
+ // change gives the number of the commit that changes the target's shape.
173
+ async #describe(sql, change) {
174
+ await this.#history.create(sql, this.qualifiedName, change);
175
+ await this.#comments.write(sql, 'TABLE', this.schema, this.table.name);
176
+ if (this.#view === undefined)
177
+ return;
178
+ await this.#view.describe(sql);
179
+ }
180
+ // The field's values in each of the tables and in the history: while a
181
+ // reload is open, the rows of the target and of its hidden target both
182
+ // still refer to files, and a row of the history keeps its file for a
183
+ // reading as of a position before the commit that replaced it.
184
+ values(sql, tables) {
185
+ const { table, schema } = this;
186
+ return async function* (field) {
187
+ const column = table.columns.find((column) => column.field === field);
188
+ if (column === undefined)
189
+ throw new TypeError(`Unknown target field: ${field}`);
190
+ for (const name of tables()) {
191
+ const present = await sql.unsafe('SELECT 1 FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3', [schema, name, column.name]);
192
+ if (present.length === 0)
193
+ continue;
194
+ for await (const rows of sql
195
+ .unsafe(`SELECT ${column.quotedName} AS value FROM ${quote(schema)}.${quote(name)}`)
196
+ .cursor(batchSize))
197
+ for (const row of rows)
198
+ yield row.value;
199
+ }
200
+ };
201
+ }
202
+ // A clear of a stored table is a load of one commit of its connection.
203
+ async clear({ writer, clock, connection: connector }, committed) {
204
+ await using connection = new PostgresSession(this.url, 'elt');
205
+ await installNumbering(connection.sql);
206
+ await connection.sql.begin(async (transaction) => {
207
+ const [schema] = await transaction.unsafe('SELECT to_regnamespace($1) IS NOT NULL AS "exists"', [quote(this.schema)]);
208
+ if (schema?.exists !== true)
209
+ return;
210
+ const writers = this.#writers;
211
+ await transaction.unsafe(`CREATE TABLE IF NOT EXISTS ${writers} ("target" TEXT PRIMARY KEY, "writer" TEXT NOT NULL)`);
212
+ const [row] = await transaction.unsafe(`SELECT "writer" FROM ${writers} WHERE "target" = $1`, [this.table.name]);
213
+ if (row !== undefined && row.writer !== writer)
214
+ throw new TargetOwnedError(this.table.name, String(row.writer), writer);
215
+ // Emptied, not dropped: views built on the table, such as marts, keep
216
+ // working. The rows it removes are forgotten: each position before the
217
+ // clear expires, so the history keeps none of them, nor their files.
218
+ const [table] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
219
+ if (table?.exists === true) {
220
+ const committedIn = await commitOnce(transaction, connector, new Date(clock.Date.now()), this.#retention);
221
+ await transaction.unsafe(`DELETE FROM ${this.qualifiedName}`);
222
+ await expireBefore(transaction, committedIn, 'clear');
223
+ await this.#history.expire(transaction);
224
+ }
225
+ // Clearing a copy also abandons the reload it left open.
226
+ await transaction.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
227
+ for (const column of this.table.columns.filter((column) => column.storesFile)) {
228
+ const store = new PostgresFileStore(this.schema, this.table, column);
229
+ const [chunks] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [store.qualifiedName]);
230
+ // No row is left in the table or its history, so no stored file
231
+ // stays.
232
+ if (chunks?.exists === true)
233
+ await store.prune(transaction, [this.#history.qualifiedName]);
234
+ }
235
+ await transaction.unsafe(`DELETE FROM ${writers} WHERE "target" = $1`, [
236
+ this.table.name,
237
+ ]);
238
+ });
239
+ await committed?.(this.values(connection.sql, () => [this.table.name, this.#history.name]));
240
+ }
241
+ get #hash() {
242
+ return hash('sha256', this.qualifiedName).slice(0, 40);
243
+ }
244
+ // One staged row as JSON. A row too large to send even alone, past V8's
245
+ // longest string or Postgres's largest message, fails its stream, naming the
246
+ // column that holds the most and the record's key.
247
+ #serialize(row, record) {
248
+ let text;
249
+ try {
250
+ text = JSON.stringify(row);
251
+ }
252
+ catch (error) {
253
+ if (!(error instanceof RangeError))
254
+ throw error;
255
+ }
256
+ if (text !== undefined && Buffer.byteLength(text) <= rowBytes)
257
+ return text;
258
+ const { columns } = this.table;
259
+ const sizes = columns.map((column) => {
260
+ const value = Reflect.get(Object(record), column.field);
261
+ return typeof value === 'string' ? Buffer.byteLength(value) : 0;
262
+ });
263
+ const largestSize = Math.max(...sizes);
264
+ const largest = columns[sizes.indexOf(largestSize)];
265
+ const key = Object.fromEntries(this.stream.primaryKey.map((field) => [
266
+ field,
267
+ Reflect.get(Object(record), field),
268
+ ]));
269
+ throw new RangeError(`A record of ${this.qualifiedName} with key ${JSON.stringify(key)} is too large to stage: its largest value, in column ${largest?.name}, is ${largestSize} bytes, and Postgres takes at most 1 GB in one message`);
270
+ }
271
+ // A reload's hidden target, beside the target and invisible to readers.
272
+ get #hiddenName() {
273
+ return `_elt_next_${this.#hash}`;
274
+ }
275
+ get #hidden() {
276
+ return `${quote(this.schema)}.${quote(this.#hiddenName)}`;
277
+ }
278
+ // Drops the chunks of every stored column that is no longer one of the
279
+ // target's file columns, before the table is replaced: no row of the table
280
+ // that takes its place refers to them.
281
+ async #dropRetiredChunks(sql) {
282
+ const files = new Set(this.table.columns
283
+ .filter(({ storesFile }) => storesFile)
284
+ .map(({ name }) => name));
285
+ for (const { name } of await storedColumns(sql, this.qualifiedName))
286
+ if (!files.has(name))
287
+ await sql.unsafe(`DROP TABLE IF EXISTS ${PostgresFileStore.qualifiedName(this.schema, this.table, name)}`);
288
+ }
289
+ // Swaps a completed reload's hidden target in last, when the target no
290
+ // longer has the stream's shape, so readers of the stored table wait only
291
+ // for the swap, not the reload. A reader that holds it past the timeout
292
+ // fails the copy until the next run. The views that read the target, its
293
+ // history or its reader view read the swapped-in table, and it keeps the
294
+ // target's privileges.
295
+ // Its rows are new to readers at this commit, and the history keeps every
296
+ // row of the target it replaces.
297
+ async #swap(sql, committedIn) {
298
+ await sql.unsafe(`SET LOCAL lock_timeout = '${swapTimeout}'`);
299
+ await sql.unsafe(`UPDATE ${this.#hidden} SET "committed_in" = $1`, [
300
+ committedIn,
301
+ ]);
302
+ await rebindDependents(sql, this.#readable, async () => {
303
+ await this.#view?.drop(sql);
304
+ await this.#dropRetiredChunks(sql);
305
+ const [target] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
306
+ if (target?.exists === true)
307
+ await this.#history.keep(sql, committedIn, this.qualifiedName, `FROM ${this.qualifiedName} AS "_elt_old"`);
308
+ await sql.unsafe(`DROP TABLE IF EXISTS ${this.qualifiedName}`);
309
+ await sql.unsafe(`ALTER TABLE ${this.#hidden} RENAME TO ${this.table.quotedName}`);
310
+ await this.adopt(sql);
311
+ await this.#describe(sql, async () => committedIn);
312
+ });
313
+ }
314
+ // What readers and the views built on them read of the target.
315
+ get #readable() {
316
+ return [
317
+ this.qualifiedName,
318
+ this.#history.qualifiedName,
319
+ ...(this.#view === undefined ? [] : [this.#view.qualifiedName]),
320
+ ];
321
+ }
322
+ // Gives a completed reload's rows to a target that still has the stream's
323
+ // shape, in place and by difference: the history keeps each row of the
324
+ // target that no row of the hidden target matches, and the row goes; a row
325
+ // with a match stays, with its commit and its loaded_at, and the target
326
+ // keeps its indexes, its triggers and what depends on it. Each row of the
327
+ // hidden target that no row matches enters with this commit, and the
328
+ // loaded_at of the load that wrote it.
329
+ async #reloadInPlace(sql, committedIn) {
330
+ const target = { table: this.qualifiedName, id: 'ctid' };
331
+ const reloaded = { table: this.#hidden, id: 'ctid' };
332
+ await this.#history.remove(sql, committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, reloaded)})`);
333
+ await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.table.columns.map((column) => column.quotedName).join(', ')}, "loaded_at", $1::integer, ${version} FROM ${this.#hidden} WHERE ctid IN (${this.#unmatched(reloaded, target)}) ORDER BY ctid`, [committedIn]);
334
+ await sql.unsafe(`DROP TABLE ${this.#hidden}`);
335
+ }
336
+ // Refuses a cast of column from its stored type from that would change a
337
+ // stored value, such as double precision 1.5 to bigint 2, before the ALTER,
338
+ // as a STRICT SQLite table refuses it: each value that is not null must
339
+ // cast back to itself. A cast from text is the parse the load applies to
340
+ // each value of the new type, so it is trusted, and a value it cannot parse
341
+ // fails the ALTER.
342
+ async #refuseInexactCast(sql, column, from) {
343
+ if (from === 'text' || from === 'text[]')
344
+ return;
345
+ const { quotedName: name, storageType: to } = column;
346
+ let changes;
347
+ try {
348
+ changes = await sql.unsafe(`SELECT 1 FROM ${this.qualifiedName} WHERE ${name} IS NOT NULL AND ${name}::${to}::${from} IS DISTINCT FROM ${name} LIMIT 1`);
349
+ }
350
+ catch (cause) {
351
+ throw new TypeError(`column ${name} does not cast to ${to}: ${errorMessage(cause)}`, { cause });
352
+ }
353
+ if (changes.length > 0)
354
+ throw new TypeError(`column ${name} holds a value that a cast to ${to} would change`);
355
+ }
356
+ // Brings a stored table the stream no longer fits to its shape in place,
357
+ // keeping every row, as Airbyte's Postgres destination does: a column the
358
+ // stream added is added nullable, one it dropped is dropped, a changed type
359
+ // is cast, and NOT NULL and the primary key follow the target's explicit
360
+ // key columns. A value that will not cast exactly, or rows that repeat a
361
+ // new key, fail the load, naming the column, and the transaction leaves the
362
+ // table as it was. Readers wait for the change as they wait for a swap, and
363
+ // the views that read the target, its history or its reader view read it
364
+ // again after the change.
365
+ // A row with a value in a column that the change drops or casts loses or
366
+ // changes the value, so it is a new version, in a commit of the load of its
367
+ // own: the history keeps the row as it was. Its row hash counted the old
368
+ // value, so it becomes NULL, which matches no record until a commit writes
369
+ // the row again. A column that the change adds is NULL in each row, which a
370
+ // row hash does not count, so it makes no new version.
371
+ // A change of type that the history cannot follow by a widening makes the
372
+ // change a commit too, which expires each position before it.
373
+ async #evolve(sql, stage, load) {
374
+ let changedIn;
375
+ const change = async () => (changedIn ??= await load.number(sql));
376
+ await sql.unsafe(`SET LOCAL lock_timeout = '${swapTimeout}'`);
377
+ // Postgres keeps the primary key as a constraint apart from NOT NULL, and
378
+ // refuses to drop NOT NULL from a column it covers.
379
+ const storedKey = await sql.unsafe("SELECT c.conname, a.attname FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY (c.conkey) WHERE c.conrelid = to_regclass($1) AND c.contype = 'p'", [this.qualifiedName]);
380
+ const key = this.table.columns.filter((column) => column.isPrimaryKey);
381
+ const rekeyed = !isDeepStrictEqual(storedKey.map(({ attname }) => attname).toSorted(), key.map(({ name }) => name).toSorted());
382
+ const stored = new Map((await storedColumns(sql, this.qualifiedName)).map((column) => [
383
+ column.name,
384
+ column,
385
+ ]));
386
+ const types = new Map((await storedColumns(sql, `pg_temp.${stage}`)).map(({ name, type }) => [
387
+ name,
388
+ type,
389
+ ]));
390
+ const kept = new Set(['loaded_at', 'committed_in', '_elt_row_hash']);
391
+ const declared = new Set(this.table.columns.map(({ name }) => name));
392
+ const changed = [...stored.values()]
393
+ .filter(({ name, type }) => !kept.has(name) && (!declared.has(name) || types.get(name) !== type))
394
+ .map(({ name }) => `${quote(name)} IS NOT NULL`)
395
+ .join(' OR ');
396
+ await rebindDependents(sql, this.#readable, async () => {
397
+ await this.#view?.drop(sql);
398
+ if (changed !== '') {
399
+ const committedIn = await change();
400
+ await this.#history.keep(sql, committedIn, this.qualifiedName, `FROM ${this.qualifiedName} AS "_elt_old" WHERE ${changed}`);
401
+ await sql.unsafe(`UPDATE ${this.qualifiedName} SET "committed_in" = $1, "loaded_at" = $2::text::timestamptz, ${version} = NULL WHERE ${changed}`, [committedIn, load.loadedAt]);
402
+ }
403
+ try {
404
+ await this.#dropRetiredChunks(sql);
405
+ const [constraint] = storedKey;
406
+ if (rekeyed && constraint !== undefined)
407
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} DROP CONSTRAINT ${quote(constraint.conname)}`);
408
+ for (const column of this.table.columns) {
409
+ kept.add(column.name);
410
+ const type = types.get(column.name);
411
+ const existing = stored.get(column.name);
412
+ if (existing === undefined) {
413
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD COLUMN ${column.quotedName} ${type}`);
414
+ continue;
415
+ }
416
+ if (existing.type !== type) {
417
+ await this.#refuseInexactCast(sql, column, existing.type);
418
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column.quotedName} TYPE ${type} USING ${column.quotedName}::${type}`);
419
+ }
420
+ if (existing.required !== column.isPrimaryKey)
421
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column.quotedName} ${column.isPrimaryKey ? 'SET' : 'DROP'} NOT NULL`);
422
+ }
423
+ for (const name of stored.keys())
424
+ if (!kept.has(name))
425
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} DROP COLUMN ${quote(name)}`);
426
+ if (rekeyed && key.length > 0)
427
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD PRIMARY KEY (${key.map(({ quotedName }) => quotedName).join(', ')})`);
428
+ }
429
+ catch (cause) {
430
+ throw new TypeError(`The rows stored in ${this.qualifiedName} do not fit the new shape of stream ${this.stream.name}: ${errorMessage(cause)}. Clear the copy to load it again, or change the stream so they fit.`, { cause });
431
+ }
432
+ await this.#describe(sql, change);
433
+ });
434
+ }
435
+ // The rows of one partition, or every row when the stream is not
436
+ // partitioned, as a WHERE clause over the target's or the stage's columns.
437
+ #scope(partition) {
438
+ if (partition === null)
439
+ return ['', []];
440
+ const columns = Object.keys(partition).map((field) => {
441
+ const column = this.table.columns.find((column) => column.field === field);
442
+ if (column === undefined)
443
+ throw new TypeError(`Resetting a partition requires destination column ${field}`);
444
+ return column;
445
+ });
446
+ return [
447
+ ` WHERE ${columns.map((column, index) => `${column.quotedName} = ${column.valueFrom('$1::text::json', index)}`).join(' AND ')}`,
448
+ [JSON.stringify(columns.map((column) => column.encode(partition)))],
449
+ ];
450
+ }
451
+ // One stream's load. Operations wait in a session-private TEMP stage, so no
452
+ // other session sees them and a crash leaves nothing behind. prepare only
453
+ // refuses, evolves a stored table the stream no longer fits unless the load
454
+ // overwrites it, and stages, so a run that fails before a commit leaves no
455
+ // rows behind.
456
+ // Each commit is one transaction that merges the stage, with the result of
457
+ // applying its operations one at a time, into the target, or into a
458
+ // reload's hidden target until complete() gives its rows to the target.
459
+ // The first commit also creates the target, or the hidden target of a new
460
+ // reload.
461
+ async prepare(load, { writer, reloading }) {
462
+ const { sql } = load;
463
+ const stage = quote(`_elt_stage_${this.#hash}`);
464
+ const stores = this.table.columns
465
+ .filter((column) => column.storesFile)
466
+ .map((column) => new PostgresFileStore(this.schema, this.table, column));
467
+ let mode = await load.transaction(async (sql) => {
468
+ await this.#refuse(sql, writer);
469
+ await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${stage}`);
470
+ await sql.unsafe(`CREATE TEMP TABLE ${stage} (${seq} BIGINT PRIMARY KEY, ${op} TEXT NOT NULL, ${this.table.columns.map((column) => `${column.quotedName} ${column.storageType}`).join(', ')}, ${version} TEXT)`);
471
+ for (const store of stores)
472
+ await store.stage(sql);
473
+ const target = await storedFit(sql, this.table, this.qualifiedName, stage);
474
+ const hidden = await storedFit(sql, this.table, this.#hidden, stage);
475
+ let decided = reloadMode({
476
+ reloading,
477
+ destinationSyncMode: this.configuration.destinationSyncMode,
478
+ target,
479
+ hidden,
480
+ });
481
+ // A hidden target no reload continues is a leftover readers never saw.
482
+ if (decided !== 'continue')
483
+ await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
484
+ await this.#view?.refuse(sql, target === 'stale');
485
+ if (decided === 'evolve') {
486
+ await this.#evolve(sql, stage, load);
487
+ decided = 'load';
488
+ }
489
+ if (decided === 'load' && !this.replaces)
490
+ await this.inspect(sql);
491
+ return decided;
492
+ });
493
+ const fresh = mode === 'create' || mode === 'reload';
494
+ const open = () => mode === 'reload' || mode === 'continue';
495
+ const into = () => (open() ? this.#hidden : this.qualifiedName);
496
+ const tables = () => [
497
+ ...(open() ? [this.qualifiedName, this.#hidden] : [this.qualifiedName]),
498
+ this.#history.qualifiedName,
499
+ ];
500
+ const { columns } = this.table;
501
+ // One JSON parameter per batch, cast back per column: no bind-parameter
502
+ // limit. Declared text so the driver sends it as given.
503
+ const insert = `INSERT INTO ${stage} (${seq}, ${op}, ${columns.map((column) => column.quotedName).join(', ')}, ${version}) SELECT (row->>0)::bigint, row->>1, ${columns.map((column, index) => column.valueFrom('row', index + 2)).join(', ')}, row->>${columns.length + 2} FROM json_array_elements($1::text::json) AS row`;
504
+ // Each staged row as its JSON, and how many bytes they add up to.
505
+ let pending = [];
506
+ let pendingBytes = 0;
507
+ let next = 0;
508
+ const flush = async () => {
509
+ if (pending.length === 0)
510
+ return;
511
+ const rows = pending;
512
+ pending = [];
513
+ pendingBytes = 0;
514
+ await sql.unsafe(insert, [`[${rows.join(',')}]`]);
515
+ };
516
+ // Partitions the stage dropped, which the next commit empties in the
517
+ // table it merges into.
518
+ let resets = [];
519
+ const drop = async () => {
520
+ pending = [];
521
+ pendingBytes = 0;
522
+ resets = [];
523
+ await sql.unsafe(`TRUNCATE ${stage}`);
524
+ for (const store of stores)
525
+ await store.discard(sql);
526
+ };
527
+ // The first commit claims the target: it owns the hidden target of a
528
+ // reload, or creates or adopts the table, and before its transaction it
529
+ // makes the schema of a new target. Later commits find it claimed. claim
530
+ // gives whether the commit replaces the rows of the target with the
531
+ // stage's.
532
+ const unclaimed = {
533
+ before: async () => {
534
+ if (mode === 'create')
535
+ await makeSchema(load.sql, this.schema);
536
+ },
537
+ claim: async (sql, committedIn) => {
538
+ if (open()) {
539
+ await this.#own(sql, writer);
540
+ return false;
541
+ }
542
+ await this.#create(sql, writer, this.replaces, committedIn);
543
+ return this.replaces;
544
+ },
545
+ };
546
+ const claimed = {
547
+ before: async () => { },
548
+ claim: async () => false,
549
+ };
550
+ let claiming = unclaimed;
551
+ return {
552
+ fresh,
553
+ get reloading() {
554
+ return open();
555
+ },
556
+ values: this.values(sql, () => [
557
+ ...(open() ? [this.table.name, this.#hiddenName] : [this.table.name]),
558
+ this.#history.name,
559
+ ]),
560
+ apply: async (operation) => {
561
+ if (operation.type === 'RESET') {
562
+ await flush();
563
+ const { partition } = operation;
564
+ if (partition === null) {
565
+ // The whole stream starts over in a new hidden target.
566
+ await drop();
567
+ mode = 'reload';
568
+ return;
569
+ }
570
+ const [rows, values] = this.#scope(partition);
571
+ await sql.unsafe(`DELETE FROM ${stage}${rows}`, values);
572
+ resets.push(partition);
573
+ return;
574
+ }
575
+ // Each file is read here, before the source advances past it. The
576
+ // row keeps its id, and the row hash its content: each load gives a
577
+ // file a new id, so a row whose file is the same matches.
578
+ let data = operation.type === 'RECORD' ? operation.data : undefined;
579
+ const contents = new Map();
580
+ for (const store of stores) {
581
+ const content = Reflect.get(Object(data), store.column.field);
582
+ if (content instanceof FileContent) {
583
+ const { file, sha256 } = await store.save(sql, content);
584
+ data = { ...Object(data), [store.column.field]: file };
585
+ contents.set(store.column, sha256);
586
+ }
587
+ }
588
+ const row = operation.type === 'RECORD'
589
+ ? this.encode(data)
590
+ : this.deletionRow(operation.key);
591
+ const hashed = operation.type === 'RECORD'
592
+ ? rowHash(columns, columns.map((column, index) => contents.get(column) ?? row[index] ?? null))
593
+ : null;
594
+ const text = this.#serialize([next++, operation.type === 'RECORD' ? 'R' : 'D', ...row, hashed], data);
595
+ const bytes = Buffer.byteLength(text);
596
+ if (pendingBytes + bytes > batchBytes)
597
+ await flush();
598
+ pending.push(text);
599
+ pendingBytes += bytes;
600
+ if (pending.length >= batchSize || pendingBytes >= batchBytes)
601
+ await flush();
602
+ },
603
+ commit: async () => {
604
+ await flush();
605
+ await claiming.before();
606
+ await load.commit(async (sql, committedIn) => {
607
+ if (mode === 'reload') {
608
+ await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
609
+ await sql.unsafe(`CREATE TABLE ${this.#tableSQL(this.#hidden)}`);
610
+ await this.build(sql, this.#hidden);
611
+ }
612
+ const replacing = await claiming.claim(sql, committedIn);
613
+ // The rows of a hidden target were never shown, so only the target
614
+ // keeps the rows a commit replaces.
615
+ const history = open() ? null : this.#history;
616
+ for (const partition of resets) {
617
+ const [rows, values] = this.#scope(partition);
618
+ if (history === null)
619
+ await sql.unsafe(`DELETE FROM ${into()}${rows}`, values);
620
+ else
621
+ await history.remove(sql, committedIn, into(), rows, values);
622
+ }
623
+ for (const store of stores)
624
+ await store.publish(sql);
625
+ const written = { loadedAt: load.loadedAt, committedIn };
626
+ if (replacing)
627
+ await this.#replace(sql, stage, written);
628
+ else
629
+ await this.merge(sql, stage, written, into(), history);
630
+ await sql.unsafe(`TRUNCATE ${stage}`);
631
+ await this.#history.expire(sql);
632
+ for (const store of stores)
633
+ await store.prune(sql, tables());
634
+ });
635
+ claiming = claimed;
636
+ // Its first commit made the hidden target, so the reload continues.
637
+ if (mode === 'reload')
638
+ mode = 'continue';
639
+ resets = [];
640
+ },
641
+ complete: async () => {
642
+ if (!open())
643
+ return;
644
+ await load.commit(async (sql, committedIn) => {
645
+ const target = await storedFit(sql, this.table, this.qualifiedName, stage);
646
+ if (target === 'fits')
647
+ await this.#reloadInPlace(sql, committedIn);
648
+ else
649
+ await this.#swap(sql, committedIn);
650
+ await this.#history.expire(sql);
651
+ for (const store of stores)
652
+ await store.prune(sql, [
653
+ this.qualifiedName,
654
+ this.#history.qualifiedName,
655
+ ]);
656
+ });
657
+ mode = 'load';
658
+ },
659
+ discard: drop,
660
+ [Symbol.asyncDispose]: async () => {
661
+ await drop();
662
+ await sql.unsafe(`DROP TABLE pg_temp.${stage}`);
663
+ for (const store of stores)
664
+ await store.unstage(sql);
665
+ },
666
+ };
667
+ }
668
+ }