@eltjs/elt-postgresql 0.0.0-stage → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +668 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
import { FileRead, isCalendarDate, isTimestamp, } from '@eltjs/elt';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
const storageTypes = {
|
|
4
|
+
text: 'TEXT',
|
|
5
|
+
integer: 'BIGINT',
|
|
6
|
+
int64: 'BIGINT',
|
|
7
|
+
real: 'DOUBLE PRECISION',
|
|
8
|
+
decimal: 'NUMERIC',
|
|
9
|
+
boolean: 'BOOLEAN',
|
|
10
|
+
date: 'DATE',
|
|
11
|
+
timestamp: 'TIMESTAMPTZ',
|
|
12
|
+
local_timestamp: 'TIMESTAMP',
|
|
13
|
+
local_time: 'TIME',
|
|
14
|
+
blob: 'BYTEA',
|
|
15
|
+
// An object of scalar fields, only as an item of an array column.
|
|
16
|
+
object: 'JSONB',
|
|
17
|
+
};
|
|
18
|
+
// information_schema.columns.data_type for each storage type.
|
|
19
|
+
const dataTypes = {
|
|
20
|
+
text: 'text',
|
|
21
|
+
integer: 'bigint',
|
|
22
|
+
int64: 'bigint',
|
|
23
|
+
real: 'double precision',
|
|
24
|
+
decimal: 'numeric',
|
|
25
|
+
boolean: 'boolean',
|
|
26
|
+
date: 'date',
|
|
27
|
+
timestamp: 'timestamp with time zone',
|
|
28
|
+
local_timestamp: 'timestamp without time zone',
|
|
29
|
+
local_time: 'time without time zone',
|
|
30
|
+
blob: 'bytea',
|
|
31
|
+
object: 'jsonb',
|
|
32
|
+
};
|
|
33
|
+
// The kind that stores each string format; base64 text loads as its bytes.
|
|
34
|
+
const formatKinds = {
|
|
35
|
+
'date-time': 'timestamp',
|
|
36
|
+
date: 'date',
|
|
37
|
+
'date-time-local': 'local_timestamp',
|
|
38
|
+
'time-local': 'local_time',
|
|
39
|
+
int64: 'int64',
|
|
40
|
+
decimal: 'decimal',
|
|
41
|
+
base64: 'blob',
|
|
42
|
+
};
|
|
43
|
+
const plainKinds = new Set([
|
|
44
|
+
'text',
|
|
45
|
+
'integer',
|
|
46
|
+
'real',
|
|
47
|
+
'blob',
|
|
48
|
+
'boolean',
|
|
49
|
+
'date',
|
|
50
|
+
'timestamp',
|
|
51
|
+
'object',
|
|
52
|
+
]);
|
|
53
|
+
// Postgres keeps time to the microsecond.
|
|
54
|
+
const finestPrecision = 6;
|
|
55
|
+
// ISO years count astronomically, so year 0000 is 1 BC, which Postgres spells
|
|
56
|
+
// as year 0001 with a BC suffix; every other year reads the same in both.
|
|
57
|
+
function postgresYear(value) {
|
|
58
|
+
return value.startsWith('0000-') ? `0001${value.slice(4)} BC` : value;
|
|
59
|
+
}
|
|
60
|
+
export class PostgresColumn {
|
|
61
|
+
// The record field the column holds.
|
|
62
|
+
field;
|
|
63
|
+
// What Postgres calls the column, which the table it belongs to decides.
|
|
64
|
+
name;
|
|
65
|
+
type;
|
|
66
|
+
kind;
|
|
67
|
+
isPrimaryKey;
|
|
68
|
+
nullable;
|
|
69
|
+
optional;
|
|
70
|
+
// An array of kind, stored as a native Postgres array of that type.
|
|
71
|
+
array;
|
|
72
|
+
fileRead;
|
|
73
|
+
constructor(field, type, options) {
|
|
74
|
+
if (typeof field !== 'string')
|
|
75
|
+
throw new TypeError('A column holds a field named by a string');
|
|
76
|
+
if (typeof type === 'string' && !plainKinds.has(type))
|
|
77
|
+
throw new TypeError('Unsupported Postgres column type');
|
|
78
|
+
this.type = type;
|
|
79
|
+
const kind = typeof type === 'string' ? type : formatKinds[type.name];
|
|
80
|
+
this.array = options.array ?? false;
|
|
81
|
+
if (this.array && (kind === 'blob' || options.primaryKey))
|
|
82
|
+
throw new TypeError('Array columns hold scalar values or objects and cannot be keys');
|
|
83
|
+
this.fileRead = options.fileRead;
|
|
84
|
+
if (this.fileRead !== undefined &&
|
|
85
|
+
(!(this.fileRead instanceof FileRead) || this.fileRead.name !== field))
|
|
86
|
+
throw new TypeError('Column file read must match its field');
|
|
87
|
+
this.field = field;
|
|
88
|
+
this.name = options.name ?? field;
|
|
89
|
+
this.kind = kind;
|
|
90
|
+
this.isPrimaryKey = options.primaryKey;
|
|
91
|
+
this.nullable = !options.primaryKey && options.nullable;
|
|
92
|
+
this.optional = !options.primaryKey && options.optional;
|
|
93
|
+
Object.freeze(this);
|
|
94
|
+
}
|
|
95
|
+
primaryKey() {
|
|
96
|
+
return new PostgresColumn(this.field, this.type, {
|
|
97
|
+
nullable: false,
|
|
98
|
+
optional: false,
|
|
99
|
+
primaryKey: true,
|
|
100
|
+
array: this.array,
|
|
101
|
+
fileRead: this.fileRead,
|
|
102
|
+
name: this.name,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
notNull() {
|
|
106
|
+
return new PostgresColumn(this.field, this.type, {
|
|
107
|
+
nullable: false,
|
|
108
|
+
optional: false,
|
|
109
|
+
primaryKey: this.isPrimaryKey,
|
|
110
|
+
array: this.array,
|
|
111
|
+
fileRead: this.fileRead,
|
|
112
|
+
name: this.name,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
from(file) {
|
|
116
|
+
if (this.array || (this.kind !== 'blob' && this.kind !== 'text'))
|
|
117
|
+
throw new TypeError('Files require a BLOB column, parsed TEXT or a stored TEXT reference');
|
|
118
|
+
return new PostgresColumn(this.field, this.kind, {
|
|
119
|
+
nullable: this.nullable,
|
|
120
|
+
optional: this.optional,
|
|
121
|
+
primaryKey: this.isPrimaryKey,
|
|
122
|
+
fileRead: new FileRead(this.field, file, this.fileRead?.parser),
|
|
123
|
+
name: this.name,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
parse(parser) {
|
|
127
|
+
if (this.kind !== 'text')
|
|
128
|
+
throw new TypeError('Document parsing requires a TEXT column');
|
|
129
|
+
if (this.fileRead === undefined)
|
|
130
|
+
throw new TypeError('Select a source file before selecting a parser');
|
|
131
|
+
return new PostgresColumn(this.field, this.kind, {
|
|
132
|
+
nullable: this.nullable,
|
|
133
|
+
optional: this.optional,
|
|
134
|
+
primaryKey: this.isPrimaryKey,
|
|
135
|
+
fileRead: new FileRead(this.field, this.fileRead.file, parser),
|
|
136
|
+
name: this.name,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
// This column as the table names it.
|
|
140
|
+
named(name) {
|
|
141
|
+
return new PostgresColumn(this.field, this.type, {
|
|
142
|
+
nullable: this.nullable,
|
|
143
|
+
optional: this.optional,
|
|
144
|
+
primaryKey: this.isPrimaryKey,
|
|
145
|
+
array: this.array,
|
|
146
|
+
fileRead: this.fileRead,
|
|
147
|
+
name,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
get storesFile() {
|
|
151
|
+
return this.kind === 'blob' && this.fileRead !== undefined;
|
|
152
|
+
}
|
|
153
|
+
get quotedName() {
|
|
154
|
+
return quote(this.name);
|
|
155
|
+
}
|
|
156
|
+
// A time finer than microseconds keeps its exact text, which sorts in time
|
|
157
|
+
// order under the "C" collation, since its values share one width.
|
|
158
|
+
get #exactText() {
|
|
159
|
+
const { type } = this;
|
|
160
|
+
return (typeof type !== 'string' &&
|
|
161
|
+
(type.name === 'date-time' ||
|
|
162
|
+
type.name === 'date-time-local' ||
|
|
163
|
+
type.name === 'time-local') &&
|
|
164
|
+
type.precision > finestPrecision);
|
|
165
|
+
}
|
|
166
|
+
// The type of one value, with the precision and scale a decimal declares.
|
|
167
|
+
get #scalarType() {
|
|
168
|
+
const { type } = this;
|
|
169
|
+
if (this.#exactText)
|
|
170
|
+
return 'TEXT';
|
|
171
|
+
if (typeof type !== 'string' &&
|
|
172
|
+
type.name === 'decimal' &&
|
|
173
|
+
type.precision !== undefined)
|
|
174
|
+
return `NUMERIC(${[type.precision, type.scale ?? []].join(', ')})`;
|
|
175
|
+
return storageTypes[this.kind];
|
|
176
|
+
}
|
|
177
|
+
get storageType() {
|
|
178
|
+
if (this.storesFile)
|
|
179
|
+
return 'UUID';
|
|
180
|
+
return `${this.#scalarType}${this.array ? '[]' : ''}`;
|
|
181
|
+
}
|
|
182
|
+
get dataType() {
|
|
183
|
+
if (this.storesFile)
|
|
184
|
+
return 'uuid';
|
|
185
|
+
if (this.array)
|
|
186
|
+
return 'ARRAY';
|
|
187
|
+
return this.#exactText ? 'text' : dataTypes[this.kind];
|
|
188
|
+
}
|
|
189
|
+
// This column's value read back from element index of a JSON batch row. An
|
|
190
|
+
// array's elements are cast one by one, in order; a JSON null stays NULL.
|
|
191
|
+
valueFrom(row, index) {
|
|
192
|
+
if (!this.array)
|
|
193
|
+
return `(${row}->>${index})::${this.storageType}`;
|
|
194
|
+
return `CASE WHEN json_typeof(${row}->${index}) = 'array' THEN ARRAY(SELECT element.value::${this.#scalarType} FROM json_array_elements_text(${row}->${index}) WITH ORDINALITY AS element (value, position) ORDER BY element.position) END`;
|
|
195
|
+
}
|
|
196
|
+
// A field the stream requires stays nullable in storage: rows loaded before
|
|
197
|
+
// it was added have no value for it. Records are still checked for it when
|
|
198
|
+
// they are written (encode).
|
|
199
|
+
get definition() {
|
|
200
|
+
return `${this.quotedName} ${this.storageType}${this.isPrimaryKey ? ' PRIMARY KEY' : ''}`;
|
|
201
|
+
}
|
|
202
|
+
encode(record) {
|
|
203
|
+
if (record === null || typeof record !== 'object' || Array.isArray(record))
|
|
204
|
+
throw new TypeError(`Record is missing field ${JSON.stringify(this.field)}`);
|
|
205
|
+
if (!Object.hasOwn(record, this.field)) {
|
|
206
|
+
if (this.optional)
|
|
207
|
+
return null;
|
|
208
|
+
throw new TypeError(`Record is missing field ${JSON.stringify(this.field)}`);
|
|
209
|
+
}
|
|
210
|
+
const value = Reflect.get(record, this.field);
|
|
211
|
+
if (value === null && this.nullable)
|
|
212
|
+
return null;
|
|
213
|
+
if (this.array) {
|
|
214
|
+
if (Array.isArray(value)) {
|
|
215
|
+
const elements = value.map((element) => this.#element(element));
|
|
216
|
+
if (elements.every((element) => element !== undefined))
|
|
217
|
+
return elements;
|
|
218
|
+
}
|
|
219
|
+
throw new TypeError(`Field ${JSON.stringify(this.field)} requires an array of ${this.kind}${this.nullable ? ' or null' : ' (not null)'}`);
|
|
220
|
+
}
|
|
221
|
+
const encoded = this.#scalar(value);
|
|
222
|
+
if (encoded !== undefined)
|
|
223
|
+
return encoded;
|
|
224
|
+
throw new TypeError(`Field ${JSON.stringify(this.field)} requires ${this.kind}${this.nullable ? ' or null' : ' (not null)'}`);
|
|
225
|
+
}
|
|
226
|
+
// One element of an array in its batch encoding: an object item as its
|
|
227
|
+
// fields, any other as its scalar; undefined when the kind rejects it.
|
|
228
|
+
#element(value) {
|
|
229
|
+
if (this.kind !== 'object')
|
|
230
|
+
return this.#scalar(value);
|
|
231
|
+
// jsonb holds no NUL character, as text holds none.
|
|
232
|
+
return value !== null &&
|
|
233
|
+
typeof value === 'object' &&
|
|
234
|
+
!Array.isArray(value) &&
|
|
235
|
+
Object.values(value).every((field) => field === null ||
|
|
236
|
+
(typeof field === 'string' && !field.includes('\0')) ||
|
|
237
|
+
typeof field === 'boolean' ||
|
|
238
|
+
(typeof field === 'number' && Number.isFinite(field)))
|
|
239
|
+
? Object.fromEntries(Object.entries(value))
|
|
240
|
+
: undefined;
|
|
241
|
+
}
|
|
242
|
+
// One scalar in its batch encoding, or undefined when the kind rejects it.
|
|
243
|
+
#scalar(value) {
|
|
244
|
+
const { type } = this;
|
|
245
|
+
if (typeof type !== 'string') {
|
|
246
|
+
if (typeof value !== 'string' || !type.accepts(value))
|
|
247
|
+
return undefined;
|
|
248
|
+
switch (type.name) {
|
|
249
|
+
case 'base64':
|
|
250
|
+
return `\\x${Buffer.from(value, 'base64').toHex()}`;
|
|
251
|
+
case 'date':
|
|
252
|
+
case 'date-time':
|
|
253
|
+
case 'date-time-local':
|
|
254
|
+
return this.#exactText ? value : postgresYear(value);
|
|
255
|
+
default:
|
|
256
|
+
return value;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
switch (type) {
|
|
260
|
+
case 'text':
|
|
261
|
+
if (typeof value === 'string' && !value.includes('\0'))
|
|
262
|
+
return value;
|
|
263
|
+
break;
|
|
264
|
+
case 'boolean':
|
|
265
|
+
if (typeof value === 'boolean')
|
|
266
|
+
return value;
|
|
267
|
+
break;
|
|
268
|
+
case 'integer':
|
|
269
|
+
if (typeof value === 'bigint' &&
|
|
270
|
+
value >= -(2n ** 63n) &&
|
|
271
|
+
value < 2n ** 63n)
|
|
272
|
+
return String(value);
|
|
273
|
+
if (typeof value === 'number' && Number.isSafeInteger(value))
|
|
274
|
+
return value;
|
|
275
|
+
break;
|
|
276
|
+
case 'real':
|
|
277
|
+
if (typeof value === 'number' && Number.isFinite(value))
|
|
278
|
+
return value;
|
|
279
|
+
break;
|
|
280
|
+
case 'date':
|
|
281
|
+
if (isCalendarDate(value))
|
|
282
|
+
return postgresYear(value);
|
|
283
|
+
break;
|
|
284
|
+
case 'timestamp':
|
|
285
|
+
if (isTimestamp(value))
|
|
286
|
+
return postgresYear(value);
|
|
287
|
+
break;
|
|
288
|
+
case 'blob':
|
|
289
|
+
if (this.storesFile) {
|
|
290
|
+
if (typeof value === 'string' &&
|
|
291
|
+
/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(value))
|
|
292
|
+
return value;
|
|
293
|
+
}
|
|
294
|
+
else if (value instanceof Uint8Array)
|
|
295
|
+
return `\\x${value.toHex()}`;
|
|
296
|
+
break;
|
|
297
|
+
// An object is only an array's item, which #element encodes.
|
|
298
|
+
case 'object':
|
|
299
|
+
}
|
|
300
|
+
return undefined;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type Stream } from '@eltjs/elt';
|
|
2
|
+
import { PostgresColumn } from './postgres-column.ts';
|
|
3
|
+
export declare class PostgresColumns {
|
|
4
|
+
#private;
|
|
5
|
+
static fromSchema({ properties, required, }: Stream['jsonSchema']): readonly PostgresColumn[];
|
|
6
|
+
text(field: string): PostgresColumn;
|
|
7
|
+
integer(field: string): PostgresColumn;
|
|
8
|
+
real(field: string): PostgresColumn;
|
|
9
|
+
boolean(field: string): PostgresColumn;
|
|
10
|
+
date(field: string): PostgresColumn;
|
|
11
|
+
timestamp(field: string): PostgresColumn;
|
|
12
|
+
blob(field: string): PostgresColumn;
|
|
13
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { declaredFormat } from '@eltjs/elt';
|
|
2
|
+
import { PostgresColumn } from './postgres-column.js';
|
|
3
|
+
function scalarType(name, type, format) {
|
|
4
|
+
switch (type) {
|
|
5
|
+
case 'string':
|
|
6
|
+
return format ?? 'text';
|
|
7
|
+
case 'integer':
|
|
8
|
+
return 'integer';
|
|
9
|
+
case 'number':
|
|
10
|
+
return 'real';
|
|
11
|
+
case 'boolean':
|
|
12
|
+
return 'boolean';
|
|
13
|
+
case 'object':
|
|
14
|
+
return 'object';
|
|
15
|
+
default:
|
|
16
|
+
throw new TypeError(`Unsupported JSON Schema type for field ${name}: ${type}`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
// Postgres column declarations are independent of a source or connection.
|
|
20
|
+
export class PostgresColumns {
|
|
21
|
+
// Scalars and arrays of scalars. Unlike SQLite, the string formats get their
|
|
22
|
+
// own types, so readers can do date arithmetic without parsing text, and an
|
|
23
|
+
// array is a native array of its item type.
|
|
24
|
+
static fromSchema({ properties, required = [], }) {
|
|
25
|
+
if (properties === undefined)
|
|
26
|
+
throw new TypeError('Postgres requires an object schema with explicit properties');
|
|
27
|
+
const requiredFields = new Set(required);
|
|
28
|
+
for (const name of requiredFields)
|
|
29
|
+
if (!Object.hasOwn(properties, name))
|
|
30
|
+
throw new TypeError(`Schema does not describe field ${name}`);
|
|
31
|
+
return Object.entries(properties).map(([name, field]) => {
|
|
32
|
+
const types = typeof field.type === 'string' ? [field.type] : field.type;
|
|
33
|
+
if (new Set(types).size !== types.length)
|
|
34
|
+
throw new TypeError(`Unsupported JSON Schema type for field ${name}`);
|
|
35
|
+
const valueTypes = types.filter((type) => type !== 'null');
|
|
36
|
+
const valueType = valueTypes[0];
|
|
37
|
+
if (valueType === undefined || valueTypes.length !== 1)
|
|
38
|
+
throw new TypeError(`Postgres requires one scalar type for field ${name}`);
|
|
39
|
+
const array = valueType === 'array';
|
|
40
|
+
const items = array ? field.items : undefined;
|
|
41
|
+
if (array && items === undefined)
|
|
42
|
+
throw new TypeError(`Unsupported JSON Schema items for field ${name}`);
|
|
43
|
+
const format = declaredFormat(items ?? field);
|
|
44
|
+
return new PostgresColumn(name, scalarType(name, items?.type ?? valueType, format), {
|
|
45
|
+
nullable: types.includes('null'),
|
|
46
|
+
optional: !requiredFields.has(name),
|
|
47
|
+
primaryKey: false,
|
|
48
|
+
array,
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
text(field) {
|
|
53
|
+
return this.#column(field, 'text');
|
|
54
|
+
}
|
|
55
|
+
integer(field) {
|
|
56
|
+
return this.#column(field, 'integer');
|
|
57
|
+
}
|
|
58
|
+
real(field) {
|
|
59
|
+
return this.#column(field, 'real');
|
|
60
|
+
}
|
|
61
|
+
boolean(field) {
|
|
62
|
+
return this.#column(field, 'boolean');
|
|
63
|
+
}
|
|
64
|
+
date(field) {
|
|
65
|
+
return this.#column(field, 'date');
|
|
66
|
+
}
|
|
67
|
+
timestamp(field) {
|
|
68
|
+
return this.#column(field, 'timestamp');
|
|
69
|
+
}
|
|
70
|
+
blob(field) {
|
|
71
|
+
return this.#column(field, 'blob');
|
|
72
|
+
}
|
|
73
|
+
#column(field, type) {
|
|
74
|
+
return new PostgresColumn(field, type, {
|
|
75
|
+
nullable: true,
|
|
76
|
+
optional: false,
|
|
77
|
+
primaryKey: false,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Transaction } from './postgres-load.ts';
|
|
2
|
+
export declare class TargetComments {
|
|
3
|
+
#private;
|
|
4
|
+
constructor(table: string, columns: Readonly<Record<string, string | null>>);
|
|
5
|
+
write(sql: Transaction, kind: 'TABLE' | 'VIEW', schema: string, name: string): Promise<void>;
|
|
6
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// A target's descriptions as Postgres keeps them: one comment on the table or
|
|
2
|
+
// its view, and one on each column.
|
|
3
|
+
export class TargetComments {
|
|
4
|
+
#table;
|
|
5
|
+
#columns;
|
|
6
|
+
constructor(table, columns) {
|
|
7
|
+
this.#table = table;
|
|
8
|
+
this.#columns = columns;
|
|
9
|
+
Object.freeze(this);
|
|
10
|
+
}
|
|
11
|
+
// COMMENT does not accept bind parameters. Let Postgres quote identifiers
|
|
12
|
+
// and literals, including NULL to clear annotations removed from the schema.
|
|
13
|
+
async write(sql, kind, schema, name) {
|
|
14
|
+
const comments = await sql.unsafe(`SELECT format('COMMENT ON ${kind} %I.%I IS %L', $1::text, $2::text, $3::text) AS statement
|
|
15
|
+
UNION ALL
|
|
16
|
+
SELECT format('COMMENT ON COLUMN %I.%I.%I IS %L', $1::text, $2::text, key, value)
|
|
17
|
+
FROM jsonb_each_text($4::jsonb)`, [schema, name, this.#table, sql.json(this.#columns)]);
|
|
18
|
+
await sql.unsafe(comments.map(({ statement }) => statement).join(';'));
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { CopyConfiguration, Deduplication, KeyValue } from '@eltjs/elt';
|
|
2
|
+
import type { EncodedValue, PostgresColumn } from './postgres-column.ts';
|
|
3
|
+
import { type TargetHistory } from './postgres-history.ts';
|
|
4
|
+
import type { Transaction } from './postgres-load.ts';
|
|
5
|
+
import type { PostgresTable } from './postgres-table.ts';
|
|
6
|
+
import { PostgresWriter, type Written } from './postgres-writer.ts';
|
|
7
|
+
export declare class PostgresDeduplicatingWriter extends PostgresWriter {
|
|
8
|
+
#private;
|
|
9
|
+
readonly deduplication: Deduplication;
|
|
10
|
+
readonly keys: readonly PostgresColumn[];
|
|
11
|
+
readonly cursor?: PostgresColumn;
|
|
12
|
+
constructor(configuration: CopyConfiguration, url: string, schema: string, table: PostgresTable, retention: Temporal.Duration);
|
|
13
|
+
get dedupIndex(): string;
|
|
14
|
+
protected get replaces(): boolean;
|
|
15
|
+
protected initialize(transaction: Transaction, replacing: boolean): Promise<void>;
|
|
16
|
+
protected inspect(sql: Transaction): Promise<void>;
|
|
17
|
+
private dropOtherIndexes;
|
|
18
|
+
private index;
|
|
19
|
+
protected build(sql: Transaction, into: string): Promise<void>;
|
|
20
|
+
protected adopt(sql: Transaction): Promise<void>;
|
|
21
|
+
protected kept(stage: string): string;
|
|
22
|
+
protected merge(sql: Transaction, stage: string, written: Written, into: string, history: TargetHistory | null): Promise<void>;
|
|
23
|
+
protected encode(record: unknown): EncodedValue[];
|
|
24
|
+
protected deletionRow(key: Readonly<Record<string, KeyValue>>): EncodedValue[];
|
|
25
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { hash } from 'node:crypto';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
import { PostgresColumns } from './postgres-columns.js';
|
|
4
|
+
import { version } from './postgres-history.js';
|
|
5
|
+
import { PostgresWriter, op, seq } from './postgres-writer.js';
|
|
6
|
+
export class PostgresDeduplicatingWriter extends PostgresWriter {
|
|
7
|
+
deduplication;
|
|
8
|
+
keys;
|
|
9
|
+
cursor;
|
|
10
|
+
constructor(configuration, url, schema, table, retention) {
|
|
11
|
+
super(configuration, url, schema, table, retention);
|
|
12
|
+
this.deduplication = configuration.deduplication();
|
|
13
|
+
const inferred = PostgresColumns.fromSchema(configuration.stream.jsonSchema);
|
|
14
|
+
const column = (field) => {
|
|
15
|
+
const selected = table.columns.find((column) => column.field === field);
|
|
16
|
+
if (selected === undefined)
|
|
17
|
+
throw new TypeError(`Deduplication requires destination column ${field}`);
|
|
18
|
+
if (selected.storageType !==
|
|
19
|
+
inferred.find((column) => column.field === field)?.storageType)
|
|
20
|
+
throw new TypeError(`Deduplication column ${field} must preserve the source scalar type`);
|
|
21
|
+
return selected;
|
|
22
|
+
};
|
|
23
|
+
this.keys = Object.freeze(this.deduplication.primaryKey.map(column));
|
|
24
|
+
const { cursorField } = this.deduplication;
|
|
25
|
+
this.cursor = cursorField === undefined ? undefined : column(cursorField);
|
|
26
|
+
Object.freeze(this);
|
|
27
|
+
}
|
|
28
|
+
// Named after the table and its key, so a changed key builds a new index
|
|
29
|
+
// and an unchanged one is never rebuilt under readers.
|
|
30
|
+
get dedupIndex() {
|
|
31
|
+
return `_elt_dedup_${hash('sha256', JSON.stringify([
|
|
32
|
+
this.schema,
|
|
33
|
+
this.table.name,
|
|
34
|
+
this.keys.map((key) => key.name),
|
|
35
|
+
])).slice(0, 40)}`;
|
|
36
|
+
}
|
|
37
|
+
get replaces() {
|
|
38
|
+
return this.configuration.destinationSyncMode === 'overwrite_dedup';
|
|
39
|
+
}
|
|
40
|
+
async initialize(transaction, replacing) {
|
|
41
|
+
await transaction.unsafe(this.createTableSQL);
|
|
42
|
+
// A replacing load keeps only the rows its records match: its index is
|
|
43
|
+
// built once the rows that leave are gone, since they may repeat a key.
|
|
44
|
+
if (replacing)
|
|
45
|
+
await this.dropOtherIndexes(transaction);
|
|
46
|
+
else
|
|
47
|
+
await this.index(transaction);
|
|
48
|
+
}
|
|
49
|
+
async inspect(sql) {
|
|
50
|
+
const tracked = this.cursor === undefined ? this.keys : [...this.keys, this.cursor];
|
|
51
|
+
const nulls = await sql.unsafe(`SELECT 1 FROM ${this.qualifiedName} WHERE ${tracked.map((column) => `${column.quotedName} IS NULL`).join(' OR ')} LIMIT 1`);
|
|
52
|
+
if (nulls.length > 0)
|
|
53
|
+
throw new TypeError('Existing deduplication keys and cursors must be non-null');
|
|
54
|
+
}
|
|
55
|
+
async dropOtherIndexes(transaction) {
|
|
56
|
+
for (const index of await this.dedupIndexes(transaction))
|
|
57
|
+
if (index !== this.dedupIndex)
|
|
58
|
+
await transaction.unsafe(`DROP INDEX ${quote(this.schema)}.${quote(index)}`);
|
|
59
|
+
}
|
|
60
|
+
async index(transaction) {
|
|
61
|
+
await this.dropOtherIndexes(transaction);
|
|
62
|
+
await transaction.unsafe(`CREATE UNIQUE INDEX IF NOT EXISTS ${quote(this.dedupIndex)} ON ${this.qualifiedName} (${this.keys.map((column) => column.quotedName).join(', ')})`);
|
|
63
|
+
}
|
|
64
|
+
// A rebuilt target's key index, created while it is empty and renamed with
|
|
65
|
+
// the target, since index names are unique in a schema.
|
|
66
|
+
get #provisionalIndex() {
|
|
67
|
+
return `${this.dedupIndex.slice(0, 50)}_next`;
|
|
68
|
+
}
|
|
69
|
+
async build(sql, into) {
|
|
70
|
+
await sql.unsafe(`CREATE UNIQUE INDEX ${quote(this.#provisionalIndex)} ON ${into} (${this.keys.map((column) => column.quotedName).join(', ')})`);
|
|
71
|
+
}
|
|
72
|
+
async adopt(sql) {
|
|
73
|
+
await sql.unsafe(`ALTER INDEX ${quote(this.schema)}.${quote(this.#provisionalIndex)} RENAME TO ${quote(this.dedupIndex)}`);
|
|
74
|
+
}
|
|
75
|
+
get #guarded() {
|
|
76
|
+
return this.configuration.dedupPolicy === 'replace'
|
|
77
|
+
? undefined
|
|
78
|
+
: this.cursor;
|
|
79
|
+
}
|
|
80
|
+
// A text cursor compares by bytes, as SQLite's BINARY does.
|
|
81
|
+
get #collate() {
|
|
82
|
+
return this.cursor?.dataType === 'text' ? ' COLLATE "C"' : '';
|
|
83
|
+
}
|
|
84
|
+
#same(left, right) {
|
|
85
|
+
return this.keys
|
|
86
|
+
.map(({ quotedName }) => `${left}.${quotedName} = ${right}.${quotedName}`)
|
|
87
|
+
.join(' AND ');
|
|
88
|
+
}
|
|
89
|
+
// The staged records that the result of applying the operations one at a
|
|
90
|
+
// time keeps, as a SELECT of their seq: a staged DELETE removes its key,
|
|
91
|
+
// and only records after a key's last DELETE count. replace keeps the
|
|
92
|
+
// newest extraction, so a restated fact overwrites the loaded one;
|
|
93
|
+
// cursor_newer keeps the greatest cursor, the first on ties.
|
|
94
|
+
#accepted(stage) {
|
|
95
|
+
const keys = this.keys.map((column) => column.quotedName);
|
|
96
|
+
const guard = this.#guarded;
|
|
97
|
+
const order = guard === undefined
|
|
98
|
+
? `"staged".${seq} DESC`
|
|
99
|
+
: `"staged".${guard.quotedName}${this.#collate} DESC, "staged".${seq}`;
|
|
100
|
+
return `SELECT "ranked".${seq} FROM (SELECT "staged".${seq}, row_number() OVER (PARTITION BY ${keys.map((key) => `"staged".${key}`).join(', ')} ORDER BY ${order}) AS "_elt_rank" FROM ${stage} AS "staged" LEFT JOIN (SELECT ${keys.join(', ')}, max(${seq}) AS "_elt_last" FROM ${stage} WHERE ${op} = 'D' GROUP BY ${keys.join(', ')}) AS "deleted" ON ${this.#same('"deleted"', '"staged"')} WHERE "staged".${op} = 'R' AND ("deleted"."_elt_last" IS NULL OR "staged".${seq} > "deleted"."_elt_last")) AS "ranked" WHERE "ranked"."_elt_rank" = 1`;
|
|
101
|
+
}
|
|
102
|
+
kept(stage) {
|
|
103
|
+
return `${seq} IN (${this.#accepted(stage)})`;
|
|
104
|
+
}
|
|
105
|
+
// Whether a staged record replaces the stored row: its values differ, and
|
|
106
|
+
// the cursor guard, if any, lets it in, which rejects out-of-order replay.
|
|
107
|
+
#replaces(record, row) {
|
|
108
|
+
const guard = this.#guarded;
|
|
109
|
+
return `${record}.${version} IS DISTINCT FROM ${row}.${version}${guard === undefined ? '' : ` AND ${record}.${guard.quotedName}${this.#collate} > ${row}.${guard.quotedName}${this.#collate}`}`;
|
|
110
|
+
}
|
|
111
|
+
// Applies the staged operations as if one at a time. The history keeps
|
|
112
|
+
// each row that a DELETE removes or an accepted record replaces; a record
|
|
113
|
+
// with the stored row's values leaves the row, its commit and its
|
|
114
|
+
// loaded_at as they are.
|
|
115
|
+
async merge(sql, stage, written, into, history) {
|
|
116
|
+
const keys = this.keys.map((column) => column.quotedName);
|
|
117
|
+
const deleted = `FROM ${into} AS "_elt_old" WHERE (${keys.map((key) => `"_elt_old".${key}`).join(', ')}) IN (SELECT ${keys.join(', ')} FROM ${stage} WHERE ${op} = 'D')`;
|
|
118
|
+
await history?.keep(sql, written.committedIn, into, deleted);
|
|
119
|
+
await sql.unsafe(`DELETE FROM ${into} AS "_elt_target" USING (SELECT DISTINCT ${keys.join(', ')} FROM ${stage} WHERE ${op} = 'D') AS "deleted" WHERE ${this.#same('"_elt_target"', '"deleted"')}`);
|
|
120
|
+
const accepted = this.#accepted(stage);
|
|
121
|
+
await history?.keep(sql, written.committedIn, into, `FROM ${stage} AS "_elt_new" JOIN ${into} AS "_elt_old" ON ${this.#same('"_elt_new"', '"_elt_old"')} WHERE "_elt_new".${seq} IN (${accepted}) AND ${this.#replaces('"_elt_new"', '"_elt_old"')}`);
|
|
122
|
+
await sql.unsafe(`INSERT INTO ${into} AS "_elt_target" (${this.written.join(', ')}) SELECT ${this.staged} FROM ${stage} WHERE ${seq} IN (${accepted}) ORDER BY ${seq} ` +
|
|
123
|
+
`ON CONFLICT (${keys.join(', ')}) DO UPDATE SET ${this.written.map((field) => `${field} = excluded.${field}`).join(', ')} WHERE ${this.#replaces('excluded', '"_elt_target"')}`, [written.loadedAt, written.committedIn]);
|
|
124
|
+
}
|
|
125
|
+
encode(record) {
|
|
126
|
+
this.deduplication.key(record);
|
|
127
|
+
if (this.cursor !== undefined)
|
|
128
|
+
this.deduplication.cursor(record);
|
|
129
|
+
return super.encode(record);
|
|
130
|
+
}
|
|
131
|
+
deletionRow(key) {
|
|
132
|
+
this.deduplication.key(key);
|
|
133
|
+
return this.table.columns.map((column) => this.keys.includes(column) ? column.encode(key) : null);
|
|
134
|
+
}
|
|
135
|
+
}
|