@eltjs/elt-postgresql 0.0.0-stage → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +813 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
|
@@ -0,0 +1,813 @@
|
|
|
1
|
+
import { hash } from 'node:crypto';
|
|
2
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
3
|
+
import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, readPartitions, reloadMode, } from '@eltjs/elt';
|
|
4
|
+
import { quote } from './identifier.js';
|
|
5
|
+
import { TargetComments } from './postgres-comments.js';
|
|
6
|
+
import { rebindDependents } from './postgres-dependents.js';
|
|
7
|
+
import { PostgresFileStore } from './postgres-file-store.js';
|
|
8
|
+
import { storedColumns, storedFit } from './postgres-fit.js';
|
|
9
|
+
import { TargetHistory, rowHash, version } from './postgres-history.js';
|
|
10
|
+
import { commitOnce, expireBefore, installNumbering, } from './postgres-numbering.js';
|
|
11
|
+
import { TargetReaderView } from './postgres-reader-view.js';
|
|
12
|
+
import { PostgresSession, lockSchema, makeSchema, } from './postgres-session.js';
|
|
13
|
+
const batchSize = 1000;
|
|
14
|
+
// A batch also flushes once its JSON reaches this size, so rows of large text
|
|
15
|
+
// or binary values are sent a few at a time rather than a thousand at once.
|
|
16
|
+
const batchBytes = 16 * 2 ** 20;
|
|
17
|
+
// Postgres takes at most 1 GB in one message (PQ_LARGE_MESSAGE_LIMIT), less
|
|
18
|
+
// a margin for the batch's brackets and the message framing.
|
|
19
|
+
const rowBytes = 2 ** 30 - 2 ** 20;
|
|
20
|
+
export const seq = '"_elt_seq"';
|
|
21
|
+
export const op = '"_elt_op"';
|
|
22
|
+
// How long swapping in a rebuilt target waits for its readers before the
|
|
23
|
+
// copy gives up until the next run.
|
|
24
|
+
const swapTimeout = '3s';
|
|
25
|
+
export class PostgresWriter extends Writer {
|
|
26
|
+
configuration;
|
|
27
|
+
url;
|
|
28
|
+
schema;
|
|
29
|
+
table;
|
|
30
|
+
// Where the target keeps the rows that its commits replace or remove, and
|
|
31
|
+
// how long it keeps a position readable.
|
|
32
|
+
#history;
|
|
33
|
+
#retention;
|
|
34
|
+
#comments;
|
|
35
|
+
#view;
|
|
36
|
+
constructor(configuration, url, schema, table, retention) {
|
|
37
|
+
super(configuration.stream);
|
|
38
|
+
this.configuration = configuration;
|
|
39
|
+
this.url = url;
|
|
40
|
+
this.schema = schema;
|
|
41
|
+
this.table = table;
|
|
42
|
+
this.#retention = retention;
|
|
43
|
+
this.#history = new TargetHistory(schema, this.#hash);
|
|
44
|
+
const description = describeTarget(configuration, table.columns, (column) => `UUID reference to the source file's original bytes. Join ${new PostgresFileStore(schema, table, column).qualifiedName} on file = this value and concatenate bytes in order of n. NULL when the source file is unavailable.`);
|
|
45
|
+
this.#comments = new TargetComments(description.table, {
|
|
46
|
+
...description.columns,
|
|
47
|
+
committed_in: 'Number of the commit that last wrote this row: the row of _warehouse._elt_commits with this number tells when, and in which load.',
|
|
48
|
+
});
|
|
49
|
+
this.#view =
|
|
50
|
+
table.readerView &&
|
|
51
|
+
new TargetReaderView(table.readerView, this.qualifiedName, this.#history.qualifiedName, [...table.columns.map(({ name }) => name), 'loaded_at'], new TargetComments(description.table, description.columns));
|
|
52
|
+
if (table.readerView === undefined)
|
|
53
|
+
return;
|
|
54
|
+
const { missing } = description;
|
|
55
|
+
if (missing.length > 0)
|
|
56
|
+
throw new TypeError(`Reader view ${table.readerView.schema}.${table.readerView.name} needs JSON Schema descriptions for ${missing.join(', ')} of stream ${this.stream.name}`);
|
|
57
|
+
}
|
|
58
|
+
// Refuses, before anything is read, stored rows the load cannot keep.
|
|
59
|
+
async inspect(_sql) { }
|
|
60
|
+
get qualifiedName() {
|
|
61
|
+
return `${quote(this.schema)}.${this.table.quotedName}`;
|
|
62
|
+
}
|
|
63
|
+
get createTableSQL() {
|
|
64
|
+
return `CREATE TABLE IF NOT EXISTS ${this.#tableSQL(this.qualifiedName)}`;
|
|
65
|
+
}
|
|
66
|
+
// A table named name with the target's columns.
|
|
67
|
+
#tableSQL(name) {
|
|
68
|
+
if (this.table.columns.length === 0)
|
|
69
|
+
throw new TypeError('Resolve inferred columns before creating a table');
|
|
70
|
+
return `${name} (${this.table.columns.map((column) => column.definition).join(', ')}, "loaded_at" TIMESTAMPTZ NOT NULL, "committed_in" INTEGER NOT NULL, ${version} TEXT)`;
|
|
71
|
+
}
|
|
72
|
+
// What a reload's hidden target needs once it is created, before the first
|
|
73
|
+
// merge fills it, and once it takes the target's name.
|
|
74
|
+
async build(_sql, _into) { }
|
|
75
|
+
async adopt(_sql) { }
|
|
76
|
+
// What a commit writes into each row: its values, when its load began,
|
|
77
|
+
// its number, and the hash of its values.
|
|
78
|
+
get written() {
|
|
79
|
+
return [
|
|
80
|
+
...this.table.columns.map((column) => column.quotedName),
|
|
81
|
+
'"loaded_at"',
|
|
82
|
+
'"committed_in"',
|
|
83
|
+
version,
|
|
84
|
+
];
|
|
85
|
+
}
|
|
86
|
+
// The staged values of each row, and what the commit writes beside them.
|
|
87
|
+
get staged() {
|
|
88
|
+
return `${this.table.columns.map((column) => column.quotedName).join(', ')}, $1::text::timestamptz, $2::integer, ${version}`;
|
|
89
|
+
}
|
|
90
|
+
encode(record) {
|
|
91
|
+
return this.table.columns.map((column) => column.encode(record));
|
|
92
|
+
}
|
|
93
|
+
// Only a load that identifies rows by key can remove one: the key's values,
|
|
94
|
+
// at their columns, and null elsewhere.
|
|
95
|
+
deletionRow(_key) {
|
|
96
|
+
throw new TypeError('Only deduplicating loads can apply deletions');
|
|
97
|
+
}
|
|
98
|
+
// An overwrite replaces the target at its first commit.
|
|
99
|
+
get replaces() {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
// Which rows of the stage a commit that replaces the target keeps, as a
|
|
103
|
+
// condition on them.
|
|
104
|
+
kept(_stage) {
|
|
105
|
+
return `${op} = 'R'`;
|
|
106
|
+
}
|
|
107
|
+
// Moves the staged operations into the target, or the hidden target of a
|
|
108
|
+
// reload; every row of a run shares its loaded_at, and each row names the
|
|
109
|
+
// commit that wrote it. history keeps the rows of the target that the
|
|
110
|
+
// merge replaces or removes; it is null for a hidden target, whose rows no
|
|
111
|
+
// reader saw.
|
|
112
|
+
async merge(sql, stage, written, into, _history) {
|
|
113
|
+
await sql.unsafe(`INSERT INTO ${into} (${this.written.join(', ')}) SELECT ${this.staged} FROM ${stage} WHERE ${op} = 'R' ORDER BY ${seq}`, [written.loadedAt, written.committedIn]);
|
|
114
|
+
}
|
|
115
|
+
// The ids of the rows of left that no row of right matches, as multisets:
|
|
116
|
+
// the k-th copy of some values on one side matches the k-th copy of the
|
|
117
|
+
// same values on the other, so no key is needed.
|
|
118
|
+
#unmatched(left, right) {
|
|
119
|
+
const numbered = ({ table, id, rows }) => `SELECT ${id} AS "row", ${version} AS "hash", row_number() OVER (PARTITION BY ${version} ORDER BY ${id}) AS "copy" FROM ${table}${rows === undefined ? '' : ` WHERE ${rows}`}`;
|
|
120
|
+
return `SELECT "mine"."row" FROM (${numbered(left)}) AS "mine" LEFT JOIN (${numbered(right)}) AS "theirs" ON "theirs"."hash" = "mine"."hash" AND "theirs"."copy" = "mine"."copy" WHERE "theirs"."row" IS NULL`;
|
|
121
|
+
}
|
|
122
|
+
// Replaces the rows of the target with the records that the stage keeps,
|
|
123
|
+
// by difference: the history keeps each row of the target that no record
|
|
124
|
+
// matches, and the row goes; a row with a match stays as it is, with its
|
|
125
|
+
// commit and its loaded_at; and each record that no row matches enters
|
|
126
|
+
// with this commit. What stays matches records, so it fits the index of
|
|
127
|
+
// the mode, which the records that enter then fit too.
|
|
128
|
+
async #replace(sql, stage, written) {
|
|
129
|
+
const target = { table: this.qualifiedName, id: 'ctid' };
|
|
130
|
+
const staged = { table: stage, id: seq, rows: this.kept(stage) };
|
|
131
|
+
await this.#history.remove(sql, written.committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, staged)})`);
|
|
132
|
+
await this.initialize(sql, false);
|
|
133
|
+
await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.staged} FROM ${stage} WHERE ${seq} IN (${this.#unmatched(staged, target)}) ORDER BY ${seq}`, [written.loadedAt, written.committedIn]);
|
|
134
|
+
}
|
|
135
|
+
// The existing library-owned dedup indexes on this table.
|
|
136
|
+
async dedupIndexes(transaction) {
|
|
137
|
+
const rows = await transaction.unsafe(String.raw `SELECT indexname FROM pg_indexes WHERE schemaname = $1 AND tablename = $2 AND indexname LIKE '\_elt\_dedup\_%'`, [this.schema, this.table.name]);
|
|
138
|
+
return rows.map((row) => String(row.indexname));
|
|
139
|
+
}
|
|
140
|
+
get #writers() {
|
|
141
|
+
return `${quote(this.schema)}."_elt_writers"`;
|
|
142
|
+
}
|
|
143
|
+
// The owner lives beside the table it guards. A dropped table releases it,
|
|
144
|
+
// since nothing it held remains.
|
|
145
|
+
async #refuse(sql, writer) {
|
|
146
|
+
const [writers] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#writers]);
|
|
147
|
+
if (writers?.exists !== true)
|
|
148
|
+
return;
|
|
149
|
+
const [row] = await sql.unsafe(`SELECT "writer" FROM ${this.#writers} WHERE "target" = $1 AND to_regclass(format('%I.%I', $2::text, "target")) IS NOT NULL`, [this.table.name, this.schema]);
|
|
150
|
+
if (row !== undefined && row.writer !== writer)
|
|
151
|
+
throw new TargetOwnedError(this.table.name, String(row.writer), writer);
|
|
152
|
+
}
|
|
153
|
+
// Records the owner with the target's first commit; prepare refused any
|
|
154
|
+
// other.
|
|
155
|
+
async #own(sql, writer) {
|
|
156
|
+
await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#writers} ("target" TEXT PRIMARY KEY, "writer" TEXT NOT NULL)`);
|
|
157
|
+
await sql.unsafe(`DELETE FROM ${this.#writers} WHERE to_regclass(format('%I.%I', $1::text, "target")) IS NULL`, [this.schema]);
|
|
158
|
+
await sql.unsafe(`INSERT INTO ${this.#writers} ("target", "writer") VALUES ($1, $2) ON CONFLICT ("target") DO NOTHING`, [this.table.name, writer]);
|
|
159
|
+
}
|
|
160
|
+
// The target as its first commit, numbered committedIn, makes it: its
|
|
161
|
+
// owner, the table, its history and its comments, and the reader view. The
|
|
162
|
+
// first commits of other copies into the schema make its writers table
|
|
163
|
+
// too, so they take turns.
|
|
164
|
+
async #create(sql, writer, replacing, committedIn) {
|
|
165
|
+
await lockSchema(sql, this.schema);
|
|
166
|
+
await this.#own(sql, writer);
|
|
167
|
+
await this.initialize(sql, replacing);
|
|
168
|
+
await this.#describe(sql, async () => committedIn);
|
|
169
|
+
}
|
|
170
|
+
// The history in the target's shape, the comments of the table, and its
|
|
171
|
+
// reader view, which reads each column of the target from its history too.
|
|
172
|
+
// change gives the number of the commit that changes the target's shape.
|
|
173
|
+
async #describe(sql, change) {
|
|
174
|
+
await this.#history.create(sql, this.qualifiedName, change);
|
|
175
|
+
await this.#comments.write(sql, 'TABLE', this.schema, this.table.name);
|
|
176
|
+
if (this.#view === undefined)
|
|
177
|
+
return;
|
|
178
|
+
await this.#view.describe(sql);
|
|
179
|
+
}
|
|
180
|
+
// The field's values in each of the tables and in the history: while a
|
|
181
|
+
// reload is open, the rows of the target and of its hidden target both
|
|
182
|
+
// still refer to files, and a row of the history keeps its file for a
|
|
183
|
+
// reading as of a position before the commit that replaced it.
|
|
184
|
+
values(sql, tables) {
|
|
185
|
+
const { table, schema } = this;
|
|
186
|
+
return async function* (field) {
|
|
187
|
+
const column = table.columns.find((column) => column.field === field);
|
|
188
|
+
if (column === undefined)
|
|
189
|
+
throw new TypeError(`Unknown target field: ${field}`);
|
|
190
|
+
for (const name of tables()) {
|
|
191
|
+
const present = await sql.unsafe('SELECT 1 FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3', [schema, name, column.name]);
|
|
192
|
+
if (present.length === 0)
|
|
193
|
+
continue;
|
|
194
|
+
for await (const rows of sql
|
|
195
|
+
.unsafe(`SELECT ${column.quotedName} AS value FROM ${quote(schema)}.${quote(name)}`)
|
|
196
|
+
.cursor(batchSize))
|
|
197
|
+
for (const row of rows)
|
|
198
|
+
yield row.value;
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
// A clear of a stored table is a load of one commit of its connection.
|
|
203
|
+
async clear({ writer, clock, connection: connector }, committed) {
|
|
204
|
+
await using connection = new PostgresSession(this.url, 'elt');
|
|
205
|
+
await installNumbering(connection.sql);
|
|
206
|
+
await connection.sql.begin(async (transaction) => {
|
|
207
|
+
const [schema] = await transaction.unsafe('SELECT to_regnamespace($1) IS NOT NULL AS "exists"', [quote(this.schema)]);
|
|
208
|
+
if (schema?.exists !== true)
|
|
209
|
+
return;
|
|
210
|
+
const writers = this.#writers;
|
|
211
|
+
await transaction.unsafe(`CREATE TABLE IF NOT EXISTS ${writers} ("target" TEXT PRIMARY KEY, "writer" TEXT NOT NULL)`);
|
|
212
|
+
const [row] = await transaction.unsafe(`SELECT "writer" FROM ${writers} WHERE "target" = $1`, [this.table.name]);
|
|
213
|
+
if (row !== undefined && row.writer !== writer)
|
|
214
|
+
throw new TargetOwnedError(this.table.name, String(row.writer), writer);
|
|
215
|
+
// Emptied, not dropped: views built on the table, such as marts, keep
|
|
216
|
+
// working. The rows it removes are forgotten: each position before the
|
|
217
|
+
// clear expires, so the history keeps none of them, nor their files.
|
|
218
|
+
const [table] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
|
|
219
|
+
if (table?.exists === true) {
|
|
220
|
+
const committedIn = await commitOnce(transaction, connector, new Date(clock.Date.now()), this.#retention);
|
|
221
|
+
await transaction.unsafe(`DELETE FROM ${this.qualifiedName}`);
|
|
222
|
+
await expireBefore(transaction, committedIn, 'clear');
|
|
223
|
+
await this.#history.expire(transaction);
|
|
224
|
+
}
|
|
225
|
+
// Clearing a copy also abandons the reload it left open.
|
|
226
|
+
await transaction.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
|
|
227
|
+
await this.#reloaded(transaction);
|
|
228
|
+
for (const column of this.table.columns.filter((column) => column.storesFile)) {
|
|
229
|
+
const store = new PostgresFileStore(this.schema, this.table, column);
|
|
230
|
+
const [chunks] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [store.qualifiedName]);
|
|
231
|
+
// No row is left in the table or its history, so no stored file
|
|
232
|
+
// stays.
|
|
233
|
+
if (chunks?.exists === true)
|
|
234
|
+
await store.prune(transaction, [this.#history.qualifiedName]);
|
|
235
|
+
}
|
|
236
|
+
await transaction.unsafe(`DELETE FROM ${writers} WHERE "target" = $1`, [
|
|
237
|
+
this.table.name,
|
|
238
|
+
]);
|
|
239
|
+
});
|
|
240
|
+
await committed?.(this.values(connection.sql, () => [this.table.name, this.#history.name]));
|
|
241
|
+
}
|
|
242
|
+
get #hash() {
|
|
243
|
+
return hash('sha256', this.qualifiedName).slice(0, 40);
|
|
244
|
+
}
|
|
245
|
+
// One staged row as JSON. A row too large to send even alone, past V8's
|
|
246
|
+
// longest string or Postgres's largest message, fails its stream, naming the
|
|
247
|
+
// column that holds the most and the record's key.
|
|
248
|
+
#serialize(row, record) {
|
|
249
|
+
let text;
|
|
250
|
+
try {
|
|
251
|
+
text = JSON.stringify(row);
|
|
252
|
+
}
|
|
253
|
+
catch (error) {
|
|
254
|
+
if (!(error instanceof RangeError))
|
|
255
|
+
throw error;
|
|
256
|
+
}
|
|
257
|
+
if (text !== undefined && Buffer.byteLength(text) <= rowBytes)
|
|
258
|
+
return text;
|
|
259
|
+
const { columns } = this.table;
|
|
260
|
+
const sizes = columns.map((column) => {
|
|
261
|
+
const value = Reflect.get(Object(record), column.field);
|
|
262
|
+
return typeof value === 'string' ? Buffer.byteLength(value) : 0;
|
|
263
|
+
});
|
|
264
|
+
const largestSize = Math.max(...sizes);
|
|
265
|
+
const largest = columns[sizes.indexOf(largestSize)];
|
|
266
|
+
const key = Object.fromEntries(this.stream.primaryKey.map((field) => [
|
|
267
|
+
field,
|
|
268
|
+
Reflect.get(Object(record), field),
|
|
269
|
+
]));
|
|
270
|
+
throw new RangeError(`A record of ${this.qualifiedName} with key ${JSON.stringify(key)} is too large to stage: its largest value, in column ${largest?.name}, is ${largestSize} bytes, and Postgres takes at most 1 GB in one message`);
|
|
271
|
+
}
|
|
272
|
+
// A reload's hidden target, beside the target and invisible to readers.
|
|
273
|
+
get #hiddenName() {
|
|
274
|
+
return `_elt_next_${this.#hash}`;
|
|
275
|
+
}
|
|
276
|
+
get #hidden() {
|
|
277
|
+
return `${quote(this.schema)}.${quote(this.#hiddenName)}`;
|
|
278
|
+
}
|
|
279
|
+
// Drops the chunks of every stored column that is no longer one of the
|
|
280
|
+
// target's file columns, before the table is replaced: no row of the table
|
|
281
|
+
// that takes its place refers to them.
|
|
282
|
+
async #dropRetiredChunks(sql) {
|
|
283
|
+
const files = new Set(this.table.columns
|
|
284
|
+
.filter(({ storesFile }) => storesFile)
|
|
285
|
+
.map(({ name }) => name));
|
|
286
|
+
for (const { name } of await storedColumns(sql, this.qualifiedName))
|
|
287
|
+
if (!files.has(name))
|
|
288
|
+
await sql.unsafe(`DROP TABLE IF EXISTS ${PostgresFileStore.qualifiedName(this.schema, this.table, name)}`);
|
|
289
|
+
}
|
|
290
|
+
// Swaps a completed reload's hidden target in last, when the target no
|
|
291
|
+
// longer has the stream's shape, so readers of the stored table wait only
|
|
292
|
+
// for the swap, not the reload. A reader that holds it past the timeout
|
|
293
|
+
// fails the copy until the next run. The views that read the target, its
|
|
294
|
+
// history or its reader view read the swapped-in table, and it keeps the
|
|
295
|
+
// target's privileges.
|
|
296
|
+
// Its rows are new to readers at this commit, and the history keeps every
|
|
297
|
+
// row of the target it replaces.
|
|
298
|
+
async #swap(sql, committedIn) {
|
|
299
|
+
await sql.unsafe(`SET LOCAL lock_timeout = '${swapTimeout}'`);
|
|
300
|
+
await sql.unsafe(`UPDATE ${this.#hidden} SET "committed_in" = $1`, [
|
|
301
|
+
committedIn,
|
|
302
|
+
]);
|
|
303
|
+
await rebindDependents(sql, this.#readable, async () => {
|
|
304
|
+
await this.#view?.drop(sql);
|
|
305
|
+
await this.#dropRetiredChunks(sql);
|
|
306
|
+
const [target] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
|
|
307
|
+
if (target?.exists === true)
|
|
308
|
+
await this.#history.keep(sql, committedIn, this.qualifiedName, `FROM ${this.qualifiedName} AS "_elt_old"`);
|
|
309
|
+
await sql.unsafe(`DROP TABLE IF EXISTS ${this.qualifiedName}`);
|
|
310
|
+
await sql.unsafe(`ALTER TABLE ${this.#hidden} RENAME TO ${this.table.quotedName}`);
|
|
311
|
+
await this.adopt(sql);
|
|
312
|
+
await this.#describe(sql, async () => committedIn);
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
// What readers and the views built on them read of the target.
|
|
316
|
+
get #readable() {
|
|
317
|
+
return [
|
|
318
|
+
this.qualifiedName,
|
|
319
|
+
this.#history.qualifiedName,
|
|
320
|
+
...(this.#view === undefined ? [] : [this.#view.qualifiedName]),
|
|
321
|
+
];
|
|
322
|
+
}
|
|
323
|
+
// Gives a completed reload's rows to a target that still has the stream's
|
|
324
|
+
// shape, in place and by difference: the history keeps each row of the
|
|
325
|
+
// target that no row of the hidden target matches, and the row goes; a row
|
|
326
|
+
// with a match stays, with its commit and its loaded_at, and the target
|
|
327
|
+
// keeps its indexes, its triggers and what depends on it. Each row of the
|
|
328
|
+
// hidden target that no row matches enters with this commit, and the
|
|
329
|
+
// loaded_at of the load that wrote it.
|
|
330
|
+
async #reloadInPlace(sql, committedIn, target = { table: this.qualifiedName, id: 'ctid' }) {
|
|
331
|
+
const reloaded = { table: this.#hidden, id: 'ctid' };
|
|
332
|
+
await this.#history.remove(sql, committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, reloaded)})`);
|
|
333
|
+
await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.table.columns.map((column) => column.quotedName).join(', ')}, "loaded_at", $1::integer, ${version} FROM ${this.#hidden} WHERE ctid IN (${this.#unmatched(reloaded, target)}) ORDER BY ctid`, [committedIn]);
|
|
334
|
+
await sql.unsafe(`DROP TABLE ${this.#hidden}`);
|
|
335
|
+
}
|
|
336
|
+
// Gives a completed reload of some partitions their rows of the hidden
|
|
337
|
+
// target, in place and by difference: the rows of the target in those
|
|
338
|
+
// partitions are replaced as #reloadInPlace replaces a whole target, and
|
|
339
|
+
// the rows of every other partition stay as they are.
|
|
340
|
+
async #reloadPartitionsInPlace(sql, partitions, committedIn) {
|
|
341
|
+
const scoped = quote(`_elt_reloaded_${this.#hash}`);
|
|
342
|
+
const [within, values] = this.#within(partitions);
|
|
343
|
+
await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${scoped}`);
|
|
344
|
+
await sql.unsafe(`CREATE TEMP TABLE ${scoped} AS SELECT ctid AS "row" FROM ${this.qualifiedName} WHERE ${within}`, values);
|
|
345
|
+
await this.#reloadInPlace(sql, committedIn, {
|
|
346
|
+
table: this.qualifiedName,
|
|
347
|
+
id: 'ctid',
|
|
348
|
+
rows: `ctid IN (SELECT "row" FROM pg_temp.${scoped})`,
|
|
349
|
+
});
|
|
350
|
+
await sql.unsafe(`DROP TABLE pg_temp.${scoped}`);
|
|
351
|
+
}
|
|
352
|
+
// Refuses a cast of column from its stored type from that would change a
|
|
353
|
+
// stored value, such as double precision 1.5 to bigint 2, before the ALTER,
|
|
354
|
+
// as a STRICT SQLite table refuses it: each value that is not null must
|
|
355
|
+
// cast back to itself. A cast from text is the parse the load applies to
|
|
356
|
+
// each value of the new type, so it is trusted, and a value it cannot parse
|
|
357
|
+
// fails the ALTER.
|
|
358
|
+
async #refuseInexactCast(sql, column, from) {
|
|
359
|
+
if (from === 'text' || from === 'text[]')
|
|
360
|
+
return;
|
|
361
|
+
const { quotedName: name, storageType: to } = column;
|
|
362
|
+
let changes;
|
|
363
|
+
try {
|
|
364
|
+
changes = await sql.unsafe(`SELECT 1 FROM ${this.qualifiedName} WHERE ${name} IS NOT NULL AND ${name}::${to}::${from} IS DISTINCT FROM ${name} LIMIT 1`);
|
|
365
|
+
}
|
|
366
|
+
catch (cause) {
|
|
367
|
+
throw new TypeError(`column ${name} does not cast to ${to}: ${errorMessage(cause)}`, { cause });
|
|
368
|
+
}
|
|
369
|
+
if (changes.length > 0)
|
|
370
|
+
throw new TypeError(`column ${name} holds a value that a cast to ${to} would change`);
|
|
371
|
+
}
|
|
372
|
+
// Brings a stored table the stream no longer fits to its shape in place,
|
|
373
|
+
// keeping every row, as Airbyte's Postgres destination does: a column the
|
|
374
|
+
// stream added is added nullable, one it dropped is dropped, a changed type
|
|
375
|
+
// is cast, and NOT NULL and the primary key follow the target's explicit
|
|
376
|
+
// key columns. A value that will not cast exactly, or rows that repeat a
|
|
377
|
+
// new key, fail the load, naming the column, and the transaction leaves the
|
|
378
|
+
// table as it was. Readers wait for the change as they wait for a swap, and
|
|
379
|
+
// the views that read the target, its history or its reader view read it
|
|
380
|
+
// again after the change.
|
|
381
|
+
// A row with a value in a column that the change drops or casts loses or
|
|
382
|
+
// changes the value, so it is a new version, in a commit of the load of its
|
|
383
|
+
// own: the history keeps the row as it was. Its row hash counted the old
|
|
384
|
+
// value, so it becomes NULL, which matches no record until a commit writes
|
|
385
|
+
// the row again. A column that the change adds is NULL in each row, which a
|
|
386
|
+
// row hash does not count, so it makes no new version.
|
|
387
|
+
// A change of type that the history cannot follow by a widening makes the
|
|
388
|
+
// change a commit too, which expires each position before it.
|
|
389
|
+
async #evolve(sql, stage, load) {
|
|
390
|
+
let changedIn;
|
|
391
|
+
const change = async () => (changedIn ??= await load.number(sql));
|
|
392
|
+
await sql.unsafe(`SET LOCAL lock_timeout = '${swapTimeout}'`);
|
|
393
|
+
// Postgres keeps the primary key as a constraint apart from NOT NULL, and
|
|
394
|
+
// refuses to drop NOT NULL from a column it covers.
|
|
395
|
+
const storedKey = await sql.unsafe("SELECT c.conname, a.attname FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = ANY (c.conkey) WHERE c.conrelid = to_regclass($1) AND c.contype = 'p'", [this.qualifiedName]);
|
|
396
|
+
const key = this.table.columns.filter((column) => column.isPrimaryKey);
|
|
397
|
+
const rekeyed = !isDeepStrictEqual(storedKey.map(({ attname }) => attname).toSorted(), key.map(({ name }) => name).toSorted());
|
|
398
|
+
const stored = new Map((await storedColumns(sql, this.qualifiedName)).map((column) => [
|
|
399
|
+
column.name,
|
|
400
|
+
column,
|
|
401
|
+
]));
|
|
402
|
+
const types = new Map((await storedColumns(sql, `pg_temp.${stage}`)).map(({ name, type }) => [
|
|
403
|
+
name,
|
|
404
|
+
type,
|
|
405
|
+
]));
|
|
406
|
+
const kept = new Set(['loaded_at', 'committed_in', '_elt_row_hash']);
|
|
407
|
+
const declared = new Set(this.table.columns.map(({ name }) => name));
|
|
408
|
+
const changed = [...stored.values()]
|
|
409
|
+
.filter(({ name, type }) => !kept.has(name) && (!declared.has(name) || types.get(name) !== type))
|
|
410
|
+
.map(({ name }) => `${quote(name)} IS NOT NULL`)
|
|
411
|
+
.join(' OR ');
|
|
412
|
+
await rebindDependents(sql, this.#readable, async () => {
|
|
413
|
+
await this.#view?.drop(sql);
|
|
414
|
+
if (changed !== '') {
|
|
415
|
+
const committedIn = await change();
|
|
416
|
+
await this.#history.keep(sql, committedIn, this.qualifiedName, `FROM ${this.qualifiedName} AS "_elt_old" WHERE ${changed}`);
|
|
417
|
+
await sql.unsafe(`UPDATE ${this.qualifiedName} SET "committed_in" = $1, "loaded_at" = $2::text::timestamptz, ${version} = NULL WHERE ${changed}`, [committedIn, load.loadedAt]);
|
|
418
|
+
}
|
|
419
|
+
try {
|
|
420
|
+
await this.#dropRetiredChunks(sql);
|
|
421
|
+
const [constraint] = storedKey;
|
|
422
|
+
if (rekeyed && constraint !== undefined)
|
|
423
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} DROP CONSTRAINT ${quote(constraint.conname)}`);
|
|
424
|
+
for (const column of this.table.columns) {
|
|
425
|
+
kept.add(column.name);
|
|
426
|
+
const type = types.get(column.name);
|
|
427
|
+
const existing = stored.get(column.name);
|
|
428
|
+
if (existing === undefined) {
|
|
429
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD COLUMN ${column.quotedName} ${type}`);
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
432
|
+
if (existing.type !== type) {
|
|
433
|
+
await this.#refuseInexactCast(sql, column, existing.type);
|
|
434
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column.quotedName} TYPE ${type} USING ${column.quotedName}::${type}`);
|
|
435
|
+
}
|
|
436
|
+
if (existing.required !== column.isPrimaryKey)
|
|
437
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column.quotedName} ${column.isPrimaryKey ? 'SET' : 'DROP'} NOT NULL`);
|
|
438
|
+
}
|
|
439
|
+
for (const name of stored.keys())
|
|
440
|
+
if (!kept.has(name))
|
|
441
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} DROP COLUMN ${quote(name)}`);
|
|
442
|
+
if (rekeyed && key.length > 0)
|
|
443
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD PRIMARY KEY (${key.map(({ quotedName }) => quotedName).join(', ')})`);
|
|
444
|
+
}
|
|
445
|
+
catch (cause) {
|
|
446
|
+
throw new TypeError(`The rows stored in ${this.qualifiedName} do not fit the new shape of stream ${this.stream.name}: ${errorMessage(cause)}. Clear the copy to load it again, or change the stream so they fit.`, { cause });
|
|
447
|
+
}
|
|
448
|
+
await this.#describe(sql, change);
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
// The rows of one partition, or every row when the stream is not
|
|
452
|
+
// partitioned, as a WHERE clause over the target's or the stage's columns.
|
|
453
|
+
#scope(partition) {
|
|
454
|
+
if (partition === null)
|
|
455
|
+
return ['', []];
|
|
456
|
+
const [condition, values] = this.#within([partition]);
|
|
457
|
+
return [` WHERE ${condition}`, values];
|
|
458
|
+
}
|
|
459
|
+
// The rows of any of the partitions, as a condition over the target's or
|
|
460
|
+
// the stage's columns, with one JSON parameter for each partition: NULL for
|
|
461
|
+
// a row that does not name its partition, such as a staged DELETE whose
|
|
462
|
+
// key leaves the partition out.
|
|
463
|
+
#within(partitions) {
|
|
464
|
+
const each = partitions.map((partition, index) => this.#partition(partition, `$${index + 1}::text::json`));
|
|
465
|
+
return [
|
|
466
|
+
each.length === 0
|
|
467
|
+
? 'FALSE'
|
|
468
|
+
: each.map(([condition]) => `(${condition})`).join(' OR '),
|
|
469
|
+
each.map(([, value]) => value),
|
|
470
|
+
];
|
|
471
|
+
}
|
|
472
|
+
#partition(partition, parameter) {
|
|
473
|
+
const columns = Object.keys(partition).map((field) => {
|
|
474
|
+
const column = this.table.columns.find((column) => column.field === field);
|
|
475
|
+
if (column === undefined)
|
|
476
|
+
throw new TypeError(`Resetting a partition requires destination column ${field}`);
|
|
477
|
+
return column;
|
|
478
|
+
});
|
|
479
|
+
return [
|
|
480
|
+
columns
|
|
481
|
+
.map((column, index) => `${column.quotedName} = ${column.valueFrom(parameter, index)}`)
|
|
482
|
+
.join(' AND '),
|
|
483
|
+
JSON.stringify(columns.map((column) => column.encode(partition))),
|
|
484
|
+
];
|
|
485
|
+
}
|
|
486
|
+
get #reloads() {
|
|
487
|
+
return `${quote(this.schema)}."_elt_reloads"`;
|
|
488
|
+
}
|
|
489
|
+
// The partitions that an open reload of the target reloads, in the
|
|
490
|
+
// writer's own table beside it, written in the commit that opens or
|
|
491
|
+
// continues the reload, so a run that resumes the reload continues it. A
|
|
492
|
+
// reload of the whole stream has no row.
|
|
493
|
+
async #reloading(sql) {
|
|
494
|
+
const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
|
|
495
|
+
if (made?.exists !== true)
|
|
496
|
+
return wholeStream;
|
|
497
|
+
const [row] = await sql.unsafe(`SELECT "partitions" FROM ${this.#reloads} WHERE "target" = $1`, [this.table.name]);
|
|
498
|
+
return row === undefined
|
|
499
|
+
? wholeStream
|
|
500
|
+
: new SomePartitions(readPartitions(this.stream, JSON.parse(row.partitions)));
|
|
501
|
+
}
|
|
502
|
+
async #reload(sql, reloads) {
|
|
503
|
+
if (!(reloads instanceof SomePartitions)) {
|
|
504
|
+
await this.#reloaded(sql);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
// The first reloads of the schema's tables make the table together.
|
|
508
|
+
await lockSchema(sql, this.schema);
|
|
509
|
+
await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#reloads} ("target" TEXT PRIMARY KEY, "partitions" TEXT NOT NULL)`);
|
|
510
|
+
await sql.unsafe(`INSERT INTO ${this.#reloads} ("target", "partitions") VALUES ($1, $2) ON CONFLICT ("target") DO UPDATE SET "partitions" = excluded."partitions"`, [this.table.name, JSON.stringify(reloads.partitions)]);
|
|
511
|
+
}
|
|
512
|
+
async #reloaded(sql) {
|
|
513
|
+
const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
|
|
514
|
+
if (made?.exists === true)
|
|
515
|
+
await sql.unsafe(`DELETE FROM ${this.#reloads} WHERE "target" = $1`, [
|
|
516
|
+
this.table.name,
|
|
517
|
+
]);
|
|
518
|
+
}
|
|
519
|
+
// One stream's load. Operations wait in a session-private TEMP stage, so no
|
|
520
|
+
// other session sees them and a crash leaves nothing behind. prepare only
|
|
521
|
+
// refuses, evolves a stored table the stream no longer fits unless the load
|
|
522
|
+
// overwrites it, and stages, so a run that fails before a commit leaves no
|
|
523
|
+
// rows behind.
|
|
524
|
+
// Each commit is one transaction that merges the stage, with the result of
|
|
525
|
+
// applying its operations one at a time, into the target, or into a
|
|
526
|
+
// reload's hidden target until complete() gives its rows to the target.
|
|
527
|
+
// The first commit also creates the target, or the hidden target of a new
|
|
528
|
+
// reload.
|
|
529
|
+
async prepare(load, { writer, reloading }) {
|
|
530
|
+
const { sql } = load;
|
|
531
|
+
const stage = quote(`_elt_stage_${this.#hash}`);
|
|
532
|
+
// The rows of a commit that belong to the partitions a reload reloads.
|
|
533
|
+
const reloadStage = quote(`_elt_reload_stage_${this.#hash}`);
|
|
534
|
+
const staging = (table) => `CREATE TEMP TABLE ${table} (${seq} BIGINT PRIMARY KEY, ${op} TEXT NOT NULL, ${this.table.columns.map((column) => `${column.quotedName} ${column.storageType}`).join(', ')}, ${version} TEXT)`;
|
|
535
|
+
// What an open reload reloads; the whole stream until a RESET of a
|
|
536
|
+
// partition opens one.
|
|
537
|
+
let reloads = wholeStream;
|
|
538
|
+
const stores = this.table.columns
|
|
539
|
+
.filter((column) => column.storesFile)
|
|
540
|
+
.map((column) => new PostgresFileStore(this.schema, this.table, column));
|
|
541
|
+
let mode = await load.transaction(async (sql) => {
|
|
542
|
+
await this.#refuse(sql, writer);
|
|
543
|
+
await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${stage}`);
|
|
544
|
+
await sql.unsafe(staging(stage));
|
|
545
|
+
await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${reloadStage}`);
|
|
546
|
+
await sql.unsafe(staging(reloadStage));
|
|
547
|
+
for (const store of stores)
|
|
548
|
+
await store.stage(sql);
|
|
549
|
+
const target = await storedFit(sql, this.table, this.qualifiedName, stage);
|
|
550
|
+
const hidden = await storedFit(sql, this.table, this.#hidden, stage);
|
|
551
|
+
let decided = reloadMode({
|
|
552
|
+
reloading,
|
|
553
|
+
destinationSyncMode: this.configuration.destinationSyncMode,
|
|
554
|
+
target,
|
|
555
|
+
hidden,
|
|
556
|
+
});
|
|
557
|
+
// A hidden target no reload continues is a leftover readers never saw.
|
|
558
|
+
if (decided === 'continue')
|
|
559
|
+
reloads = await this.#reloading(sql);
|
|
560
|
+
else {
|
|
561
|
+
await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
|
|
562
|
+
await this.#reloaded(sql);
|
|
563
|
+
}
|
|
564
|
+
await this.#view?.refuse(sql, target === 'stale');
|
|
565
|
+
if (decided === 'evolve') {
|
|
566
|
+
await this.#evolve(sql, stage, load);
|
|
567
|
+
decided = 'load';
|
|
568
|
+
}
|
|
569
|
+
if (decided === 'load' && !this.replaces)
|
|
570
|
+
await this.inspect(sql);
|
|
571
|
+
return decided;
|
|
572
|
+
});
|
|
573
|
+
const fresh = mode === 'create' || mode === 'reload';
|
|
574
|
+
// The reload as the last commit left it, which a discard goes back to.
|
|
575
|
+
let committed = {
|
|
576
|
+
mode,
|
|
577
|
+
reloads,
|
|
578
|
+
};
|
|
579
|
+
const open = () => mode === 'reload' || mode === 'continue';
|
|
580
|
+
const into = () => (open() ? this.#hidden : this.qualifiedName);
|
|
581
|
+
const tables = () => [
|
|
582
|
+
...(open() ? [this.qualifiedName, this.#hidden] : [this.qualifiedName]),
|
|
583
|
+
this.#history.qualifiedName,
|
|
584
|
+
];
|
|
585
|
+
const { columns } = this.table;
|
|
586
|
+
// One JSON parameter per batch, cast back per column: no bind-parameter
|
|
587
|
+
// limit. Declared text so the driver sends it as given.
|
|
588
|
+
const insert = `INSERT INTO ${stage} (${seq}, ${op}, ${columns.map((column) => column.quotedName).join(', ')}, ${version}) SELECT (row->>0)::bigint, row->>1, ${columns.map((column, index) => column.valueFrom('row', index + 2)).join(', ')}, row->>${columns.length + 2} FROM json_array_elements($1::text::json) AS row`;
|
|
589
|
+
// Each staged row as its JSON, and how many bytes they add up to.
|
|
590
|
+
let pending = [];
|
|
591
|
+
let pendingBytes = 0;
|
|
592
|
+
let next = 0;
|
|
593
|
+
const flush = async () => {
|
|
594
|
+
if (pending.length === 0)
|
|
595
|
+
return;
|
|
596
|
+
const rows = pending;
|
|
597
|
+
pending = [];
|
|
598
|
+
pendingBytes = 0;
|
|
599
|
+
await sql.unsafe(insert, [`[${rows.join(',')}]`]);
|
|
600
|
+
};
|
|
601
|
+
// Partitions the stage dropped, which the next commit empties in the
|
|
602
|
+
// table it merges into.
|
|
603
|
+
let resets = [];
|
|
604
|
+
const drop = async () => {
|
|
605
|
+
pending = [];
|
|
606
|
+
pendingBytes = 0;
|
|
607
|
+
resets = [];
|
|
608
|
+
await sql.unsafe(`TRUNCATE ${stage}`);
|
|
609
|
+
for (const store of stores)
|
|
610
|
+
await store.discard(sql);
|
|
611
|
+
};
|
|
612
|
+
// The first commit claims the target: it owns the hidden target of a
|
|
613
|
+
// reload, or creates or adopts the table, and before its transaction it
|
|
614
|
+
// makes the schema of a new target. Later commits find it claimed. claim
|
|
615
|
+
// gives whether the commit replaces the rows of the target with the
|
|
616
|
+
// stage's.
|
|
617
|
+
const unclaimed = {
|
|
618
|
+
before: async () => {
|
|
619
|
+
if (mode === 'create')
|
|
620
|
+
await makeSchema(load.sql, this.schema);
|
|
621
|
+
},
|
|
622
|
+
claim: async (sql, committedIn) => {
|
|
623
|
+
if (open()) {
|
|
624
|
+
await this.#own(sql, writer);
|
|
625
|
+
return false;
|
|
626
|
+
}
|
|
627
|
+
await this.#create(sql, writer, this.replaces, committedIn);
|
|
628
|
+
return this.replaces;
|
|
629
|
+
},
|
|
630
|
+
};
|
|
631
|
+
const claimed = {
|
|
632
|
+
before: async () => { },
|
|
633
|
+
claim: async () => false,
|
|
634
|
+
};
|
|
635
|
+
let claiming = unclaimed;
|
|
636
|
+
return {
|
|
637
|
+
fresh,
|
|
638
|
+
get reloading() {
|
|
639
|
+
return open();
|
|
640
|
+
},
|
|
641
|
+
values: this.values(sql, () => [
|
|
642
|
+
...(open() ? [this.table.name, this.#hiddenName] : [this.table.name]),
|
|
643
|
+
this.#history.name,
|
|
644
|
+
]),
|
|
645
|
+
apply: async (operation) => {
|
|
646
|
+
if (operation.type === 'RESET') {
|
|
647
|
+
await flush();
|
|
648
|
+
const { partition } = operation;
|
|
649
|
+
if (partition === null) {
|
|
650
|
+
// The whole stream starts over in a new hidden target.
|
|
651
|
+
await drop();
|
|
652
|
+
mode = 'reload';
|
|
653
|
+
reloads = wholeStream;
|
|
654
|
+
return;
|
|
655
|
+
}
|
|
656
|
+
const [rows, values] = this.#scope(partition);
|
|
657
|
+
await sql.unsafe(`DELETE FROM ${stage}${rows}`, values);
|
|
658
|
+
resets.push(partition);
|
|
659
|
+
// The partition starts over in a hidden target, and readers see its
|
|
660
|
+
// rows as they were until the stream ends. An overwrite replaces
|
|
661
|
+
// the whole target anyway.
|
|
662
|
+
if (this.replaces)
|
|
663
|
+
return;
|
|
664
|
+
if (mode === 'load') {
|
|
665
|
+
mode = 'reload';
|
|
666
|
+
reloads = new SomePartitions([partition]);
|
|
667
|
+
}
|
|
668
|
+
else if (open() && reloads instanceof SomePartitions)
|
|
669
|
+
reloads = reloads.with(partition);
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
// Each file is read here, before the source advances past it. The
|
|
673
|
+
// row keeps its id, and the row hash its content: each load gives a
|
|
674
|
+
// file a new id, so a row whose file is the same matches.
|
|
675
|
+
let data = operation.type === 'RECORD' ? operation.data : undefined;
|
|
676
|
+
const contents = new Map();
|
|
677
|
+
for (const store of stores) {
|
|
678
|
+
const content = Reflect.get(Object(data), store.column.field);
|
|
679
|
+
if (content instanceof FileContent) {
|
|
680
|
+
const { file, sha256 } = await store.save(sql, content);
|
|
681
|
+
data = { ...Object(data), [store.column.field]: file };
|
|
682
|
+
contents.set(store.column, sha256);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
const row = operation.type === 'RECORD'
|
|
686
|
+
? this.encode(data)
|
|
687
|
+
: this.deletionRow(operation.key);
|
|
688
|
+
const hashed = operation.type === 'RECORD'
|
|
689
|
+
? rowHash(columns, columns.map((column, index) => contents.get(column) ?? row[index] ?? null))
|
|
690
|
+
: null;
|
|
691
|
+
const text = this.#serialize([next++, operation.type === 'RECORD' ? 'R' : 'D', ...row, hashed], data);
|
|
692
|
+
const bytes = Buffer.byteLength(text);
|
|
693
|
+
if (pendingBytes + bytes > batchBytes)
|
|
694
|
+
await flush();
|
|
695
|
+
pending.push(text);
|
|
696
|
+
pendingBytes += bytes;
|
|
697
|
+
if (pending.length >= batchSize || pendingBytes >= batchBytes)
|
|
698
|
+
await flush();
|
|
699
|
+
},
|
|
700
|
+
commit: async () => {
|
|
701
|
+
await flush();
|
|
702
|
+
await claiming.before();
|
|
703
|
+
await load.commit(async (sql, committedIn) => {
|
|
704
|
+
if (mode === 'reload') {
|
|
705
|
+
await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
|
|
706
|
+
await sql.unsafe(`CREATE TABLE ${this.#tableSQL(this.#hidden)}`);
|
|
707
|
+
await this.build(sql, this.#hidden);
|
|
708
|
+
}
|
|
709
|
+
if (open())
|
|
710
|
+
await this.#reload(sql, reloads);
|
|
711
|
+
const replacing = await claiming.claim(sql, committedIn);
|
|
712
|
+
// The rows of a hidden target were never shown, so only the target
|
|
713
|
+
// keeps the rows a commit replaces.
|
|
714
|
+
const history = open() ? null : this.#history;
|
|
715
|
+
for (const partition of resets) {
|
|
716
|
+
const [rows, values] = this.#scope(partition);
|
|
717
|
+
if (history === null)
|
|
718
|
+
await sql.unsafe(`DELETE FROM ${into()}${rows}`, values);
|
|
719
|
+
else
|
|
720
|
+
await history.remove(sql, committedIn, into(), rows, values);
|
|
721
|
+
}
|
|
722
|
+
for (const store of stores)
|
|
723
|
+
await store.publish(sql);
|
|
724
|
+
const written = { loadedAt: load.loadedAt, committedIn };
|
|
725
|
+
if (replacing)
|
|
726
|
+
await this.#replace(sql, stage, written);
|
|
727
|
+
else if (open() && reloads instanceof SomePartitions) {
|
|
728
|
+
// The rows of the reloaded partitions go to the hidden target,
|
|
729
|
+
// the others to the target. A DELETE whose key does not name
|
|
730
|
+
// its partition goes to both, so the reload cannot bring back a
|
|
731
|
+
// row that the source deleted.
|
|
732
|
+
const [within, values] = this.#within(reloads.partitions);
|
|
733
|
+
await sql.unsafe(`INSERT INTO ${reloadStage} SELECT * FROM ${stage} WHERE (${within}) IS TRUE OR (${op} = 'D' AND (${within}) IS NULL)`, values);
|
|
734
|
+
await sql.unsafe(`DELETE FROM ${stage} WHERE (${within}) IS TRUE`, values);
|
|
735
|
+
await this.merge(sql, stage, written, this.qualifiedName, this.#history);
|
|
736
|
+
await this.merge(sql, reloadStage, written, this.#hidden, null);
|
|
737
|
+
await sql.unsafe(`TRUNCATE ${reloadStage}`);
|
|
738
|
+
}
|
|
739
|
+
else
|
|
740
|
+
await this.merge(sql, stage, written, into(), history);
|
|
741
|
+
await sql.unsafe(`TRUNCATE ${stage}`);
|
|
742
|
+
await this.#history.expire(sql);
|
|
743
|
+
for (const store of stores)
|
|
744
|
+
await store.prune(sql, tables());
|
|
745
|
+
});
|
|
746
|
+
claiming = claimed;
|
|
747
|
+
// Its first commit made the hidden target, so the reload continues.
|
|
748
|
+
if (mode === 'reload')
|
|
749
|
+
mode = 'continue';
|
|
750
|
+
committed = { mode, reloads };
|
|
751
|
+
resets = [];
|
|
752
|
+
},
|
|
753
|
+
complete: async () => {
|
|
754
|
+
if (!open())
|
|
755
|
+
return;
|
|
756
|
+
await load.commit(async (sql, committedIn) => {
|
|
757
|
+
const target = await storedFit(sql, this.table, this.qualifiedName, stage);
|
|
758
|
+
// A reload of partitions keeps the target's shape: prepare evolves
|
|
759
|
+
// a stored target before it loads.
|
|
760
|
+
if (reloads instanceof SomePartitions && target === 'fits')
|
|
761
|
+
await this.#reloadPartitionsInPlace(sql, reloads.partitions, committedIn);
|
|
762
|
+
else if (reloads instanceof SomePartitions)
|
|
763
|
+
throw new TypeError(`${this.qualifiedName} changed its shape while some of its partitions reloaded; clear the copy to load it again`);
|
|
764
|
+
else if (target === 'fits')
|
|
765
|
+
await this.#reloadInPlace(sql, committedIn);
|
|
766
|
+
else
|
|
767
|
+
await this.#swap(sql, committedIn);
|
|
768
|
+
await this.#reloaded(sql);
|
|
769
|
+
await this.#history.expire(sql);
|
|
770
|
+
for (const store of stores)
|
|
771
|
+
await store.prune(sql, [
|
|
772
|
+
this.qualifiedName,
|
|
773
|
+
this.#history.qualifiedName,
|
|
774
|
+
]);
|
|
775
|
+
});
|
|
776
|
+
mode = 'load';
|
|
777
|
+
reloads = wholeStream;
|
|
778
|
+
committed = { mode, reloads };
|
|
779
|
+
},
|
|
780
|
+
// A RESET that no commit took is undone with the operations after it:
|
|
781
|
+
// the source's checkpoint does not know it.
|
|
782
|
+
discard: async () => {
|
|
783
|
+
await drop();
|
|
784
|
+
({ mode, reloads } = committed);
|
|
785
|
+
},
|
|
786
|
+
[Symbol.asyncDispose]: async () => {
|
|
787
|
+
await drop();
|
|
788
|
+
await sql.unsafe(`DROP TABLE pg_temp.${stage}`);
|
|
789
|
+
await sql.unsafe(`DROP TABLE pg_temp.${reloadStage}`);
|
|
790
|
+
for (const store of stores)
|
|
791
|
+
await store.unstage(sql);
|
|
792
|
+
},
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
// What an open reload reloads: the whole stream, or some partitions of it,
|
|
797
|
+
// while the others load into the target as before, as Delta Lake's
|
|
798
|
+
// replaceWhere replaces only the rows that its condition selects.
|
|
799
|
+
class WholeStream {
|
|
800
|
+
}
|
|
801
|
+
class SomePartitions {
|
|
802
|
+
partitions;
|
|
803
|
+
constructor(partitions) {
|
|
804
|
+
this.partitions = Object.freeze([...partitions]);
|
|
805
|
+
}
|
|
806
|
+
// These partitions and partition, which a later RESET reloads too.
|
|
807
|
+
with(partition) {
|
|
808
|
+
return this.partitions.some((other) => isDeepStrictEqual(other, partition))
|
|
809
|
+
? this
|
|
810
|
+
: new SomePartitions([...this.partitions, partition]);
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
const wholeStream = new WholeStream();
|