@eltjs/elt-postgresql 0.0.0-stage → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +813 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
import { hash } from 'node:crypto';
|
|
2
|
+
import { Lock } from '@zukhruf/async';
|
|
3
|
+
import { DerivedDataset, } from '@eltjs/elt';
|
|
4
|
+
import { identifier, quote } from './identifier.js';
|
|
5
|
+
import { TargetComments } from './postgres-comments.js';
|
|
6
|
+
import { rebindDependents } from './postgres-dependents.js';
|
|
7
|
+
import { PostgresDestination } from './postgres-destination.js';
|
|
8
|
+
import { storedColumns } from './postgres-fit.js';
|
|
9
|
+
import { TargetHistory, rowHash } from './postgres-history.js';
|
|
10
|
+
import { PostgresLoad } from './postgres-load.js';
|
|
11
|
+
import { TargetReaderView } from './postgres-reader-view.js';
|
|
12
|
+
import { makeSchema } from './postgres-session.js';
|
|
13
|
+
import { View, made } from './view.js';
|
|
14
|
+
// The columns the dataset keeps on every output row beside the rule's own.
|
|
15
|
+
const engineColumns = {
|
|
16
|
+
input_hash: 'SHA-256 of the input the row was made from; the row is made again when its input changes.',
|
|
17
|
+
rule: 'What made the row, such as the parser of its format or the model; the row is made again when that changes. unsupported: the rule does not handle the input.',
|
|
18
|
+
status: 'computed: the rule made the row. empty: the rule found nothing to keep, such as a photo without words. failed: the rule failed, error says why, and the refresh after the passes tries again, at most once for each pass.',
|
|
19
|
+
error: 'Why the rule failed. NULL unless status is failed.',
|
|
20
|
+
};
|
|
21
|
+
// The names a rule column cannot take: the dataset's own, and the history's.
|
|
22
|
+
const ownColumns = [
|
|
23
|
+
...Object.keys(engineColumns),
|
|
24
|
+
'committed_in',
|
|
25
|
+
'_elt_row_hash',
|
|
26
|
+
'replaced_in',
|
|
27
|
+
];
|
|
28
|
+
// Data computed on write from what the pipeline's copies load into one
|
|
29
|
+
// Postgres database: a rule makes one output row for each input, such as the
|
|
30
|
+
// text of each stored file, as pgai's vectorizer keeps an embedding beside
|
|
31
|
+
// each row. The pipeline refreshes it as its copies commit; each refresh
|
|
32
|
+
// makes the rows whose input or rule changed and deletes the rows whose
|
|
33
|
+
// input is gone, and a refresh after the passes also tries failed rows
|
|
34
|
+
// again; the passes try each input at most once, so a failing rule is asked
|
|
35
|
+
// once a pass.
|
|
36
|
+
// A rule's failure on one input is that row's; it never fails the load.
|
|
37
|
+
// Readers query the view name, in the schema of into, over the table
|
|
38
|
+
// raw_<name>, which a refresh makes again when its declaration changes,
|
|
39
|
+
// since it is rebuilt from the inputs.
|
|
40
|
+
// A refresh is a load of the database: each of its transactions that changes
|
|
41
|
+
// a row is a numbered commit, each row names the commit that last wrote it,
|
|
42
|
+
// and the history keeps each row a commit replaced or removed, so the view
|
|
43
|
+
// reads as of a position as the copies' reader views do.
|
|
44
|
+
export class ComputedDataset extends DerivedDataset {
|
|
45
|
+
name;
|
|
46
|
+
// The database and schema the dataset lives in.
|
|
47
|
+
into;
|
|
48
|
+
#from;
|
|
49
|
+
#rule;
|
|
50
|
+
#table;
|
|
51
|
+
#history;
|
|
52
|
+
#view;
|
|
53
|
+
// Every column of an output row with its type and its constraint, in table
|
|
54
|
+
// order.
|
|
55
|
+
#declared;
|
|
56
|
+
constructor({ into, name, description, from, compute, }) {
|
|
57
|
+
super();
|
|
58
|
+
this.name = identifier(name, 'computed dataset name');
|
|
59
|
+
this.into = into;
|
|
60
|
+
this.#from = Object.freeze([...from]);
|
|
61
|
+
this.#rule = compute;
|
|
62
|
+
this.#table = identifier(`raw_${name}`, 'computed dataset table name');
|
|
63
|
+
const taken = new Set([compute.key.name, ...ownColumns]);
|
|
64
|
+
for (const column of Object.keys(compute.output)) {
|
|
65
|
+
identifier(column, 'rule column name');
|
|
66
|
+
if (taken.has(column))
|
|
67
|
+
throw new TypeError(`Computed dataset ${name} cannot have the column ${column}: the dataset already has the column ${column}`);
|
|
68
|
+
taken.add(column);
|
|
69
|
+
}
|
|
70
|
+
this.#declared = Object.freeze([
|
|
71
|
+
{
|
|
72
|
+
name: identifier(compute.key.name, 'rule key name'),
|
|
73
|
+
type: 'text',
|
|
74
|
+
constraint: ' PRIMARY KEY',
|
|
75
|
+
},
|
|
76
|
+
...Object.keys(engineColumns).map((column) => ({
|
|
77
|
+
name: column,
|
|
78
|
+
type: 'text',
|
|
79
|
+
constraint: column === 'error' ? '' : ' NOT NULL',
|
|
80
|
+
})),
|
|
81
|
+
...Object.entries(compute.output).map(([column, { type }]) => ({
|
|
82
|
+
name: column,
|
|
83
|
+
type: type === 'blob' ? 'bytea' : 'text',
|
|
84
|
+
constraint: '',
|
|
85
|
+
})),
|
|
86
|
+
{ name: 'committed_in', type: 'integer', constraint: ' NOT NULL' },
|
|
87
|
+
{ name: '_elt_row_hash', type: 'text', constraint: ' NOT NULL' },
|
|
88
|
+
]);
|
|
89
|
+
const table = this.#qualified(this.#table);
|
|
90
|
+
this.#history = new TargetHistory(into.schema, hash('sha256', JSON.stringify([into.schema, this.#table])).slice(0, 40));
|
|
91
|
+
this.#view = new TargetReaderView({ schema: into.schema, name }, table, this.#history.qualifiedName, this.#columns, new TargetComments(description, {
|
|
92
|
+
[compute.key.name]: compute.key.description,
|
|
93
|
+
...engineColumns,
|
|
94
|
+
...Object.fromEntries(Object.entries(compute.output).map(([column, { description }]) => [
|
|
95
|
+
column,
|
|
96
|
+
description,
|
|
97
|
+
])),
|
|
98
|
+
}));
|
|
99
|
+
Object.freeze(this);
|
|
100
|
+
}
|
|
101
|
+
// The view readers query.
|
|
102
|
+
get qualifiedName() {
|
|
103
|
+
return `${this.into.schema}.${this.name}`;
|
|
104
|
+
}
|
|
105
|
+
// The key, the engine columns and the rule's, which readers see.
|
|
106
|
+
get #columns() {
|
|
107
|
+
return this.#declared
|
|
108
|
+
.map(({ name }) => name)
|
|
109
|
+
.filter((name) => name !== 'committed_in' && name !== '_elt_row_hash');
|
|
110
|
+
}
|
|
111
|
+
validate(connections) {
|
|
112
|
+
for (const target of this.#from)
|
|
113
|
+
if (!(target instanceof View) &&
|
|
114
|
+
this.#source(connections, target) === undefined)
|
|
115
|
+
throw new TypeError(`Computed dataset ${this.name} reads ${target.name}, which no copy of the pipeline loads`);
|
|
116
|
+
const relations = made(connections, this.into);
|
|
117
|
+
for (const name of [this.name, this.#table])
|
|
118
|
+
if (relations.has(`${this.into.schema}.${name}`))
|
|
119
|
+
throw new TypeError(`Computed dataset ${this.name} needs ${this.into.schema}.${name}, which a copy of the pipeline makes`);
|
|
120
|
+
}
|
|
121
|
+
async refresh({ connections, clock, signal, trigger, }) {
|
|
122
|
+
// Open from the start, so it closes a refresh that a process ended; with
|
|
123
|
+
// no commit it is no span, and leaves the positions as they are.
|
|
124
|
+
await using load = await PostgresLoad.open(this.into.url, { dataset: this.name }, { clock, retention: this.into.retention });
|
|
125
|
+
const inputs = await this.#inputs(load.sql, connections);
|
|
126
|
+
if (inputs === null)
|
|
127
|
+
return;
|
|
128
|
+
const key = quote(this.#rule.key.name);
|
|
129
|
+
const table = this.#qualified(this.#table);
|
|
130
|
+
// The number of a transaction's commit, taken at its first change, so a
|
|
131
|
+
// transaction that changes no row takes none.
|
|
132
|
+
const commits = (sql) => {
|
|
133
|
+
let number;
|
|
134
|
+
return () => (number ??= load.number(sql));
|
|
135
|
+
};
|
|
136
|
+
await makeSchema(load.sql, this.into.schema);
|
|
137
|
+
const stored = await load.transaction(async (sql) => {
|
|
138
|
+
const commit = commits(sql);
|
|
139
|
+
await this.#declare(sql, commit);
|
|
140
|
+
const rows = new Map();
|
|
141
|
+
const gone = [];
|
|
142
|
+
for (const row of await sql.unsafe(`SELECT ${key} AS "key", "input_hash", "rule", "status" FROM ${table}`))
|
|
143
|
+
if (inputs.has(row.key))
|
|
144
|
+
rows.set(row.key, {
|
|
145
|
+
inputHash: row.input_hash,
|
|
146
|
+
rule: row.rule,
|
|
147
|
+
status: row.status,
|
|
148
|
+
});
|
|
149
|
+
else
|
|
150
|
+
gone.push(row.key);
|
|
151
|
+
if (gone.length > 0)
|
|
152
|
+
await this.#history.remove(sql, await commit(), table, ` WHERE "_elt_old".${key} IN (SELECT jsonb_array_elements_text($1::text::jsonb))`, [JSON.stringify(gone)]);
|
|
153
|
+
await this.#history.expire(sql);
|
|
154
|
+
return rows;
|
|
155
|
+
});
|
|
156
|
+
const written = this.#declared.map(({ name }) => quote(name));
|
|
157
|
+
const hashed = this.#columns.map((name) => ({ name }));
|
|
158
|
+
// Writes a row in the commit that commit numbers, unless the stored row
|
|
159
|
+
// has the same values: then it keeps its commit. The history keeps the
|
|
160
|
+
// row it replaces.
|
|
161
|
+
const store = async (sql, commit, id, inputHash, rule, status, error, values) => {
|
|
162
|
+
const row = [
|
|
163
|
+
id,
|
|
164
|
+
inputHash,
|
|
165
|
+
rule,
|
|
166
|
+
status,
|
|
167
|
+
error,
|
|
168
|
+
...Object.keys(this.#rule.output).map((column) => values[column] ?? null),
|
|
169
|
+
];
|
|
170
|
+
const version = rowHash(hashed, row);
|
|
171
|
+
const [before] = await sql.unsafe(`SELECT "_elt_row_hash" AS "hash" FROM ${table} WHERE ${key} = $1`, [id]);
|
|
172
|
+
if (before?.hash === version)
|
|
173
|
+
return;
|
|
174
|
+
const number = await commit();
|
|
175
|
+
if (before !== undefined)
|
|
176
|
+
await this.#history.keep(sql, number, table, `FROM ${table} AS "_elt_old" WHERE "_elt_old".${key} = $1`, [id]);
|
|
177
|
+
await sql.unsafe(`INSERT INTO ${table} (${written.join(', ')}) VALUES (${written.map((_, index) => `$${index + 1}`).join(', ')}) ON CONFLICT (${key}) DO UPDATE SET ${written
|
|
178
|
+
.slice(1)
|
|
179
|
+
.map((column) => `${column} = excluded.${column}`)
|
|
180
|
+
.join(', ')}`, [...row.map(parameter), number, version]);
|
|
181
|
+
};
|
|
182
|
+
// The load has one session, so the commits of the calls take turns.
|
|
183
|
+
const session = new Lock();
|
|
184
|
+
const settle = (work) => session.run(() => load.transaction(work), { signal });
|
|
185
|
+
const due = [];
|
|
186
|
+
for (const [id, input] of inputs) {
|
|
187
|
+
if (signal?.aborted)
|
|
188
|
+
return;
|
|
189
|
+
const inputHash = hash('sha256', JSON.stringify(input));
|
|
190
|
+
const rule = await this.#rule.identity(input);
|
|
191
|
+
const before = stored.get(id);
|
|
192
|
+
// A rule that does not handle the input keeps what it made before.
|
|
193
|
+
if (rule === null) {
|
|
194
|
+
if (before?.status === 'computed')
|
|
195
|
+
continue;
|
|
196
|
+
if (before?.rule === 'unsupported' && before.inputHash === inputHash)
|
|
197
|
+
continue;
|
|
198
|
+
await settle((sql) => store(sql, commits(sql), id, inputHash, 'unsupported', 'empty', null, {}));
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (before !== undefined &&
|
|
202
|
+
before.inputHash === inputHash &&
|
|
203
|
+
before.rule === rule) {
|
|
204
|
+
// A failed input waits for the refresh after the passes.
|
|
205
|
+
if (before.status !== 'failed' || trigger.kind === 'commit')
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
// The passes try each input, as it is now, at most once: the refresh
|
|
209
|
+
// after them leaves what a commit refresh of the same passes tried.
|
|
210
|
+
const attempt = `${id}\0${inputHash}\0${rule}`;
|
|
211
|
+
if (trigger.tried.has(attempt))
|
|
212
|
+
continue;
|
|
213
|
+
if (trigger.kind === 'commit')
|
|
214
|
+
trigger.tried.add(attempt);
|
|
215
|
+
due.push({ id, input, inputHash, rule });
|
|
216
|
+
}
|
|
217
|
+
// A pool of calls, each taking the next batch once its own committed, so
|
|
218
|
+
// a slow call holds only its place, as a worker pool keeps its other
|
|
219
|
+
// workers busy. A call that fails stops the pool taking more.
|
|
220
|
+
const failures = [];
|
|
221
|
+
let next = 0;
|
|
222
|
+
await Promise.all(Array.from({ length: this.#rule.concurrency }, async () => {
|
|
223
|
+
while (failures.length === 0 && signal?.aborted !== true) {
|
|
224
|
+
const batch = due.slice(next, next + this.#rule.batch);
|
|
225
|
+
next += this.#rule.batch;
|
|
226
|
+
if (batch.length === 0)
|
|
227
|
+
return;
|
|
228
|
+
let results;
|
|
229
|
+
try {
|
|
230
|
+
results = await this.#rule.compute(batch.map(({ input }) => input), signal);
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
failures.push(error);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
// Each call commits on its own, so readers see rows as they land.
|
|
237
|
+
await settle(async (sql) => {
|
|
238
|
+
const commit = commits(sql);
|
|
239
|
+
for (const [index, { id, inputHash, rule }] of batch.entries()) {
|
|
240
|
+
const result = results[index];
|
|
241
|
+
if (result === undefined || 'gone' in result)
|
|
242
|
+
continue;
|
|
243
|
+
if ('value' in result)
|
|
244
|
+
await store(sql, commit, id, inputHash, rule, 'computed', null, result.value);
|
|
245
|
+
else if ('error' in result)
|
|
246
|
+
await store(sql, commit, id, inputHash, rule, 'failed', result.error, {});
|
|
247
|
+
else
|
|
248
|
+
await store(sql, commit, id, inputHash, rule, 'empty', null, {});
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}));
|
|
253
|
+
if (failures.length > 0)
|
|
254
|
+
throw failures[0];
|
|
255
|
+
}
|
|
256
|
+
// Makes the table, its history and its view, or makes the table again when
|
|
257
|
+
// its declaration changed: the history keeps each row of the old table, in
|
|
258
|
+
// a commit, and the views over the old table read the new one.
|
|
259
|
+
async #declare(sql, commit) {
|
|
260
|
+
const table = this.#qualified(this.#table);
|
|
261
|
+
const stored = await storedColumns(sql, table);
|
|
262
|
+
const create = `CREATE TABLE ${table} (${this.#declared
|
|
263
|
+
.map(({ name, type, constraint }) => `${quote(name)} ${type}${constraint}`)
|
|
264
|
+
.join(', ')}, CHECK ("status" IN ('computed', 'empty', 'failed')))`;
|
|
265
|
+
const fits = stored.length === this.#declared.length &&
|
|
266
|
+
stored.every(({ name, type }, index) => this.#declared[index]?.name === name &&
|
|
267
|
+
this.#declared[index]?.type === type);
|
|
268
|
+
if (stored.length === 0)
|
|
269
|
+
await sql.unsafe(create);
|
|
270
|
+
else if (!fits) {
|
|
271
|
+
const [kept] = await sql.unsafe(`SELECT 1 FROM ${table} LIMIT 1`);
|
|
272
|
+
if (kept !== undefined) {
|
|
273
|
+
await this.#history.create(sql, table, commit);
|
|
274
|
+
await this.#history.keep(sql, await commit(), table, `FROM ${table} AS "_elt_old"`);
|
|
275
|
+
}
|
|
276
|
+
await rebindDependents(sql, [table, this.#history.qualifiedName, this.#view.qualifiedName], async () => {
|
|
277
|
+
await this.#view.drop(sql);
|
|
278
|
+
await sql.unsafe(`DROP TABLE ${table}`);
|
|
279
|
+
await sql.unsafe(create);
|
|
280
|
+
await this.#history.create(sql, table, commit);
|
|
281
|
+
await this.#view.describe(sql);
|
|
282
|
+
});
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
await this.#history.create(sql, table, commit);
|
|
286
|
+
await this.#view.describe(sql);
|
|
287
|
+
}
|
|
288
|
+
// Each input by key, from the rows of each relation of from that exists
|
|
289
|
+
// yet: what the copies loaded, through its reader view if it has one, or
|
|
290
|
+
// what a view before this dataset gives. A key read twice keeps its first
|
|
291
|
+
// input.
|
|
292
|
+
// null while a relation reads a table that is gone, such as one a person
|
|
293
|
+
// dropped by hand: its copy loads it again, and the dataset waits with what
|
|
294
|
+
// it made, as a view waits. Read as no rows, it would delete every output,
|
|
295
|
+
// and the reload would pay the rule for each again.
|
|
296
|
+
async #inputs(sql, connections) {
|
|
297
|
+
const inputs = new Map();
|
|
298
|
+
for (const target of this.#from) {
|
|
299
|
+
const relation = target instanceof View
|
|
300
|
+
? this.#qualified(target.name, target.into.schema)
|
|
301
|
+
: this.#loaded(connections, target);
|
|
302
|
+
if (relation === undefined)
|
|
303
|
+
continue;
|
|
304
|
+
const [exists] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [relation]);
|
|
305
|
+
if (exists?.exists !== true) {
|
|
306
|
+
if (target instanceof View)
|
|
307
|
+
continue;
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
for await (const rows of sql
|
|
311
|
+
.unsafe(`SELECT * FROM ${relation}`)
|
|
312
|
+
.cursor(1000))
|
|
313
|
+
for (const row of rows) {
|
|
314
|
+
const read = this.#rule.read(row);
|
|
315
|
+
if (read !== null && !inputs.has(read.key))
|
|
316
|
+
inputs.set(read.key, read.input);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return inputs;
|
|
320
|
+
}
|
|
321
|
+
// The relation that readers read of a target a copy of the pipeline loads:
|
|
322
|
+
// its reader view, or else its table. undefined until it is first made,
|
|
323
|
+
// since a table that a copy never made yet has no rows to read.
|
|
324
|
+
#loaded(connections, target) {
|
|
325
|
+
const destination = this.#source(connections, target);
|
|
326
|
+
if (destination === undefined)
|
|
327
|
+
return undefined;
|
|
328
|
+
return target.readerView === undefined
|
|
329
|
+
? this.#qualified(target.name, destination.schema)
|
|
330
|
+
: this.#qualified(target.readerView.name, target.readerView.schema);
|
|
331
|
+
}
|
|
332
|
+
// The destination of the copy of the pipeline that loads target.
|
|
333
|
+
#source(connections, target) {
|
|
334
|
+
const connection = connections.find(({ steps }) => steps.some(({ to }) => to === target));
|
|
335
|
+
return connection?.destination instanceof PostgresDestination
|
|
336
|
+
? connection.destination
|
|
337
|
+
: undefined;
|
|
338
|
+
}
|
|
339
|
+
#qualified(name, schema = this.into.schema) {
|
|
340
|
+
return `${quote(schema)}.${quote(name)}`;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
// A value as the driver sends it: text as text, a file's bytes as bytea.
|
|
344
|
+
function parameter(value) {
|
|
345
|
+
return value instanceof Uint8Array ? Buffer.from(value) : value;
|
|
346
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// A schema or published view is named by the caller's own SQL, so a name
|
|
2
|
+
// Postgres cannot hold, which it would silently cut to 63 bytes, is refused
|
|
3
|
+
// rather than renamed.
|
|
4
|
+
export function identifier(name, what) {
|
|
5
|
+
if (typeof name !== 'string' ||
|
|
6
|
+
!name ||
|
|
7
|
+
name.includes('\0') ||
|
|
8
|
+
Buffer.byteLength(name) > 63)
|
|
9
|
+
throw new TypeError(`Invalid ${what}: ${JSON.stringify(name)}`);
|
|
10
|
+
return name;
|
|
11
|
+
}
|
|
12
|
+
export function quote(name) {
|
|
13
|
+
return `"${name.replaceAll('"', '""')}"`;
|
|
14
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { ComputedDataset } from './computed-dataset.ts';
|
|
2
|
+
export type { PostgresReading } from './postgres-as-of.ts';
|
|
3
|
+
export { PostgresCheckpointStore } from './postgres-checkpoint-store.ts';
|
|
4
|
+
export { PostgresColumn } from './postgres-column.ts';
|
|
5
|
+
export { PostgresColumns } from './postgres-columns.ts';
|
|
6
|
+
export { PostgresDestination } from './postgres-destination.ts';
|
|
7
|
+
export { PostgresSyncHistory } from './postgres-sync-history.ts';
|
|
8
|
+
export { PostgresTable } from './postgres-table.ts';
|
|
9
|
+
export { View } from './view.ts';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { ComputedDataset } from './computed-dataset.js';
|
|
2
|
+
export { PostgresCheckpointStore } from './postgres-checkpoint-store.js';
|
|
3
|
+
export { PostgresColumn } from './postgres-column.js';
|
|
4
|
+
export { PostgresColumns } from './postgres-columns.js';
|
|
5
|
+
export { PostgresDestination } from './postgres-destination.js';
|
|
6
|
+
export { PostgresSyncHistory } from './postgres-sync-history.js';
|
|
7
|
+
export { PostgresTable } from './postgres-table.js';
|
|
8
|
+
export { View } from './view.js';
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { quote } from './identifier.js';
|
|
2
|
+
import { PostgresWriter } from './postgres-writer.js';
|
|
3
|
+
export class PostgresAppendWriter extends PostgresWriter {
|
|
4
|
+
async initialize(transaction) {
|
|
5
|
+
await transaction.unsafe(this.createTableSQL);
|
|
6
|
+
// A mode change releases the deduplication index. Dropping only one that
|
|
7
|
+
// exists avoids an exclusive lock that would block readers every run.
|
|
8
|
+
for (const index of await this.dedupIndexes(transaction))
|
|
9
|
+
await transaction.unsafe(`DROP INDEX ${quote(this.schema)}.${quote(index)}`);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type postgres from 'postgres';
|
|
2
|
+
/**
|
|
3
|
+
* The database as of one position: `sql` reads in one read-only transaction
|
|
4
|
+
* of its own session, and each reader view, and each view over one, shows
|
|
5
|
+
* its target as it was at that position. Dispose it to end the session.
|
|
6
|
+
*/
|
|
7
|
+
export type PostgresReading = {
|
|
8
|
+
readonly sql: postgres.Sql;
|
|
9
|
+
readonly position: string;
|
|
10
|
+
readonly at: Temporal.Instant;
|
|
11
|
+
} & AsyncDisposable;
|
|
12
|
+
export declare function readAsOf(url: string, position: string | undefined): Promise<PostgresReading | {
|
|
13
|
+
readonly expired: string;
|
|
14
|
+
}>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { committedAt, expiry, numberingIn, parsePosition, settledMark, } from './postgres-numbering.js';
|
|
2
|
+
import { PostgresSession } from './postgres-session.js';
|
|
3
|
+
// Reads the database as of the position, or as of its newest position. The
|
|
4
|
+
// high-water mark is taken before the snapshot, so the snapshot sees each
|
|
5
|
+
// commit at or below it; one REPEATABLE READ transaction then holds the
|
|
6
|
+
// snapshot that the numbering and every later read see, so a commit after
|
|
7
|
+
// the reading begins shows in none of them. Below the newest commit the
|
|
8
|
+
// snapshot sees, the transaction sets elt.position, which each reader view
|
|
9
|
+
// reads; no read makes or changes an object.
|
|
10
|
+
export async function readAsOf(url, position) {
|
|
11
|
+
const asked = position === undefined ? undefined : parsePosition(position);
|
|
12
|
+
await using stack = new AsyncDisposableStack();
|
|
13
|
+
const { sql } = stack.use(new PostgresSession(url, 'elt-reading'));
|
|
14
|
+
const settled = await settledMark(sql);
|
|
15
|
+
if (settled === null)
|
|
16
|
+
return { expired: 'No load has committed yet' };
|
|
17
|
+
await sql.unsafe('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY');
|
|
18
|
+
stack.defer(async () => {
|
|
19
|
+
await sql.unsafe('ROLLBACK');
|
|
20
|
+
});
|
|
21
|
+
const numbered = await numberingIn(sql, settled);
|
|
22
|
+
if (asked !== undefined) {
|
|
23
|
+
const expired = expiry(numbered, asked);
|
|
24
|
+
if (expired !== undefined)
|
|
25
|
+
return { expired };
|
|
26
|
+
refuseInsidePass(numbered, asked);
|
|
27
|
+
}
|
|
28
|
+
const number = asked?.number ?? numbered?.ends.at(-1);
|
|
29
|
+
const commit = number === undefined ? undefined : await committedAt(sql, number);
|
|
30
|
+
if (numbered === null || number === undefined || commit === undefined)
|
|
31
|
+
return {
|
|
32
|
+
expired: position === undefined
|
|
33
|
+
? 'The database has no position yet'
|
|
34
|
+
: `Position ${position} names no commit of the database`,
|
|
35
|
+
};
|
|
36
|
+
if (number < commit.newest)
|
|
37
|
+
await sql `SELECT set_config('elt.position', ${String(number)}, true)`;
|
|
38
|
+
const owned = stack.move();
|
|
39
|
+
return {
|
|
40
|
+
sql,
|
|
41
|
+
position: `${numbered.epoch}:${number}`,
|
|
42
|
+
at: Temporal.Instant.fromEpochMilliseconds(commit.at.getTime()),
|
|
43
|
+
[Symbol.asyncDispose]: () => owned.disposeAsync(),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
// No later load can straddle a position, so one that a reader was given
|
|
47
|
+
// stays one; a commit that is not a position stays inside its pass.
|
|
48
|
+
function refuseInsidePass(numbered, { epoch, number }) {
|
|
49
|
+
if (!numbered?.ends.includes(number))
|
|
50
|
+
throw new TypeError(`${epoch}:${number} is not a position of the database: a reading of a commit inside a pass would show a part of that pass. Read a position that positions() lists.`);
|
|
51
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type CheckpointSession, CheckpointStore } from '@eltjs/elt';
|
|
2
|
+
export declare class PostgresCheckpointStore extends CheckpointStore {
|
|
3
|
+
#private;
|
|
4
|
+
readonly schema: string;
|
|
5
|
+
constructor({ url, schema }: {
|
|
6
|
+
url: string;
|
|
7
|
+
schema: string;
|
|
8
|
+
});
|
|
9
|
+
protected session<T>(ids: readonly string[], work: (session: CheckpointSession) => Promise<T>): Promise<T>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { CheckpointStore } from '@eltjs/elt';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
import { PostgresSession, connectionUrl, lockSchema, schemaName, } from './postgres-session.js';
|
|
4
|
+
// Checkpoints kept beside the data they describe, in `_elt_checkpoints`
|
|
5
|
+
// of one schema, so dropping the schema resets both. Each acknowledged
|
|
6
|
+
// checkpoint is saved as soon as the load commits it; a crash in between
|
|
7
|
+
// replays that unit, which deduplication absorbs.
|
|
8
|
+
export class PostgresCheckpointStore extends CheckpointStore {
|
|
9
|
+
schema;
|
|
10
|
+
#url;
|
|
11
|
+
constructor({ url, schema }) {
|
|
12
|
+
super();
|
|
13
|
+
this.#url = connectionUrl(url);
|
|
14
|
+
this.schema = schemaName(schema);
|
|
15
|
+
Object.freeze(this);
|
|
16
|
+
}
|
|
17
|
+
get #table() {
|
|
18
|
+
return `${quote(this.schema)}."_elt_checkpoints"`;
|
|
19
|
+
}
|
|
20
|
+
// Each replication's lock is a session lock on the run's own connection, so
|
|
21
|
+
// no transaction stays open while the load runs; each save autocommits.
|
|
22
|
+
async session(ids, work) {
|
|
23
|
+
await using connection = new PostgresSession(this.#url, 'elt-checkpoints');
|
|
24
|
+
const { sql } = connection;
|
|
25
|
+
await this.#create(sql);
|
|
26
|
+
// Two-key locks never collide with the one-key schema lock of DDL. A
|
|
27
|
+
// replication already running fails the run fast; others run in parallel.
|
|
28
|
+
// Taken in sorted order, so two runs never wait on each other.
|
|
29
|
+
const namespace = `elt-checkpoints:${this.schema}`;
|
|
30
|
+
const held = [];
|
|
31
|
+
try {
|
|
32
|
+
for (const id of ids) {
|
|
33
|
+
const [lock] = await sql.unsafe('SELECT pg_try_advisory_lock(hashtext($1), hashtext($2)) AS "locked"', [namespace, id]);
|
|
34
|
+
if (lock?.locked !== true)
|
|
35
|
+
throw new TypeError(`Checkpoint ${id} is in use by another run`);
|
|
36
|
+
held.push(id);
|
|
37
|
+
}
|
|
38
|
+
return await work({
|
|
39
|
+
read: async (id) => {
|
|
40
|
+
const [saved] = await sql.unsafe(`SELECT "binding"::text AS "binding", "reloading" FROM ${this.#table} WHERE "id" = $1`, [id]);
|
|
41
|
+
return saved === undefined
|
|
42
|
+
? undefined
|
|
43
|
+
: {
|
|
44
|
+
binding: String(saved.binding),
|
|
45
|
+
reloading: saved.reloading === true,
|
|
46
|
+
};
|
|
47
|
+
},
|
|
48
|
+
state: async (id) => {
|
|
49
|
+
const [saved] = await sql.unsafe(`SELECT "state"::text AS "state" FROM ${this.#table} WHERE "id" = $1`, [id]);
|
|
50
|
+
return saved === undefined ? undefined : String(saved.state);
|
|
51
|
+
},
|
|
52
|
+
save: async (id, { binding, reloading, state }) => {
|
|
53
|
+
await sql.unsafe(`INSERT INTO ${this.#table} ("id", "binding", "reloading", "state") VALUES ($1, $2::text::json, $3, $4::text::json) ON CONFLICT ("id") DO UPDATE SET "binding" = excluded."binding", "reloading" = excluded."reloading", "state" = excluded."state"`, [id, binding, reloading, state]);
|
|
54
|
+
},
|
|
55
|
+
reloaded: async (id) => {
|
|
56
|
+
await sql.unsafe(`UPDATE ${this.#table} SET "reloading" = false WHERE "id" = $1`, [id]);
|
|
57
|
+
},
|
|
58
|
+
remove: async (id) => {
|
|
59
|
+
await sql.unsafe(`DELETE FROM ${this.#table} WHERE "id" = $1`, [id]);
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
for (const id of held)
|
|
65
|
+
await sql.unsafe('SELECT pg_advisory_unlock(hashtext($1), hashtext($2))', [namespace, id]);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
// Two runs of one connection create the schema at once, and Postgres
|
|
69
|
+
// refuses the second CREATE SCHEMA with a duplicate key; the schema's lock
|
|
70
|
+
// makes them take turns.
|
|
71
|
+
async #create(sql) {
|
|
72
|
+
const [found] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#table]);
|
|
73
|
+
if (found?.exists === true)
|
|
74
|
+
return;
|
|
75
|
+
await sql.begin(async (transaction) => {
|
|
76
|
+
await lockSchema(transaction, this.schema);
|
|
77
|
+
await transaction.unsafe(`CREATE SCHEMA IF NOT EXISTS ${quote(this.schema)}`);
|
|
78
|
+
// JSON keeps the exact text: JSONB refuses \u0000 and lone surrogates,
|
|
79
|
+
// which checkpoint state may carry.
|
|
80
|
+
await transaction.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#table} ("id" TEXT PRIMARY KEY, "binding" JSON NOT NULL, "reloading" BOOLEAN NOT NULL, "state" JSON NOT NULL)`);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { type DeclaredFormat, type DocumentParser, FileRead, type FileReference } from '@eltjs/elt';
|
|
2
|
+
declare const storageTypes: {
|
|
3
|
+
readonly text: 'TEXT';
|
|
4
|
+
readonly integer: 'BIGINT';
|
|
5
|
+
readonly int64: 'BIGINT';
|
|
6
|
+
readonly real: 'DOUBLE PRECISION';
|
|
7
|
+
readonly decimal: 'NUMERIC';
|
|
8
|
+
readonly boolean: 'BOOLEAN';
|
|
9
|
+
readonly date: 'DATE';
|
|
10
|
+
readonly timestamp: 'TIMESTAMPTZ';
|
|
11
|
+
readonly local_timestamp: 'TIMESTAMP';
|
|
12
|
+
readonly local_time: 'TIME';
|
|
13
|
+
readonly blob: 'BYTEA';
|
|
14
|
+
readonly object: 'JSONB';
|
|
15
|
+
};
|
|
16
|
+
type Kind = keyof typeof storageTypes;
|
|
17
|
+
type PlainKind = 'text' | 'integer' | 'real' | 'blob' | 'boolean' | 'date' | 'timestamp' | 'object';
|
|
18
|
+
export type PostgresColumnType = PlainKind | DeclaredFormat;
|
|
19
|
+
type EncodedObject = Readonly<Record<string, string | number | boolean | null>>;
|
|
20
|
+
export type EncodedValue = string | number | boolean | null | readonly (string | number | boolean | EncodedObject)[];
|
|
21
|
+
export declare class PostgresColumn {
|
|
22
|
+
#private;
|
|
23
|
+
readonly field: string;
|
|
24
|
+
readonly name: string;
|
|
25
|
+
readonly type: PostgresColumnType;
|
|
26
|
+
readonly kind: Kind;
|
|
27
|
+
readonly isPrimaryKey: boolean;
|
|
28
|
+
readonly nullable: boolean;
|
|
29
|
+
readonly optional: boolean;
|
|
30
|
+
readonly array: boolean;
|
|
31
|
+
readonly fileRead?: FileRead;
|
|
32
|
+
constructor(field: string, type: PostgresColumnType, options: {
|
|
33
|
+
nullable: boolean;
|
|
34
|
+
optional: boolean;
|
|
35
|
+
primaryKey: boolean;
|
|
36
|
+
array?: boolean;
|
|
37
|
+
fileRead?: FileRead;
|
|
38
|
+
name?: string;
|
|
39
|
+
});
|
|
40
|
+
primaryKey(): PostgresColumn;
|
|
41
|
+
notNull(): PostgresColumn;
|
|
42
|
+
from(file: FileReference): PostgresColumn;
|
|
43
|
+
parse(parser: DocumentParser): PostgresColumn;
|
|
44
|
+
named(name: string): PostgresColumn;
|
|
45
|
+
get storesFile(): boolean;
|
|
46
|
+
get quotedName(): string;
|
|
47
|
+
get storageType(): string;
|
|
48
|
+
get dataType(): string;
|
|
49
|
+
valueFrom(row: string, index: number): string;
|
|
50
|
+
get definition(): string;
|
|
51
|
+
encode(record: unknown): EncodedValue;
|
|
52
|
+
}
|
|
53
|
+
export {};
|