@eltjs/elt-postgresql 0.0.0-stage → 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +359 -2
  3. package/dist/computed-dataset.d.ts +24 -0
  4. package/dist/computed-dataset.js +346 -0
  5. package/dist/identifier.d.ts +2 -0
  6. package/dist/identifier.js +14 -0
  7. package/dist/index.d.ts +9 -0
  8. package/dist/index.js +8 -0
  9. package/dist/postgres-append-writer.d.ts +5 -0
  10. package/dist/postgres-append-writer.js +11 -0
  11. package/dist/postgres-as-of.d.ts +14 -0
  12. package/dist/postgres-as-of.js +51 -0
  13. package/dist/postgres-checkpoint-store.d.ts +10 -0
  14. package/dist/postgres-checkpoint-store.js +83 -0
  15. package/dist/postgres-column.d.ts +53 -0
  16. package/dist/postgres-column.js +302 -0
  17. package/dist/postgres-columns.d.ts +13 -0
  18. package/dist/postgres-columns.js +80 -0
  19. package/dist/postgres-comments.d.ts +6 -0
  20. package/dist/postgres-comments.js +20 -0
  21. package/dist/postgres-deduplicating-writer.d.ts +25 -0
  22. package/dist/postgres-deduplicating-writer.js +135 -0
  23. package/dist/postgres-dependents.d.ts +2 -0
  24. package/dist/postgres-dependents.js +105 -0
  25. package/dist/postgres-destination.d.ts +31 -0
  26. package/dist/postgres-destination.js +87 -0
  27. package/dist/postgres-file-store.d.ts +23 -0
  28. package/dist/postgres-file-store.js +80 -0
  29. package/dist/postgres-fit.d.ts +9 -0
  30. package/dist/postgres-fit.js +25 -0
  31. package/dist/postgres-history.d.ts +17 -0
  32. package/dist/postgres-history.js +141 -0
  33. package/dist/postgres-identifiers.d.ts +8 -0
  34. package/dist/postgres-identifiers.js +26 -0
  35. package/dist/postgres-load.d.ts +20 -0
  36. package/dist/postgres-load.js +73 -0
  37. package/dist/postgres-numbering.d.ts +42 -0
  38. package/dist/postgres-numbering.js +311 -0
  39. package/dist/postgres-overwrite-writer.d.ts +4 -0
  40. package/dist/postgres-overwrite-writer.js +8 -0
  41. package/dist/postgres-privileges.d.ts +7 -0
  42. package/dist/postgres-privileges.js +50 -0
  43. package/dist/postgres-reader-view.d.ts +11 -0
  44. package/dist/postgres-reader-view.js +64 -0
  45. package/dist/postgres-session.d.ts +11 -0
  46. package/dist/postgres-session.js +61 -0
  47. package/dist/postgres-sync-history-schema.d.ts +31 -0
  48. package/dist/postgres-sync-history-schema.js +100 -0
  49. package/dist/postgres-sync-history.d.ts +13 -0
  50. package/dist/postgres-sync-history.js +152 -0
  51. package/dist/postgres-table.d.ts +15 -0
  52. package/dist/postgres-table.js +73 -0
  53. package/dist/postgres-views.d.ts +10 -0
  54. package/dist/postgres-views.js +119 -0
  55. package/dist/postgres-writer.d.ts +39 -0
  56. package/dist/postgres-writer.js +668 -0
  57. package/dist/view.d.ts +22 -0
  58. package/dist/view.js +81 -0
  59. package/package.json +47 -4
@@ -0,0 +1,42 @@
1
+ import type postgres from 'postgres';
2
+ type Queries = postgres.Sql | postgres.TransactionSql;
3
+ export declare function installNumbering(sql: postgres.Sql): Promise<void>;
4
+ export type LoadOwner = {
5
+ readonly connector: string;
6
+ } | {
7
+ readonly dataset: string;
8
+ };
9
+ export declare function grantReading(sql: Queries, role: string): Promise<void>;
10
+ export declare function openLoad(sql: postgres.Sql, owner: LoadOwner, openedAt: Date): Promise<string>;
11
+ export declare function closeLoad(sql: Queries, load: string, closedAt: Date): Promise<void>;
12
+ export declare function commitNumber(sql: Queries, { load, owner }: {
13
+ load: string;
14
+ owner: LoadOwner;
15
+ }, committedAt: Date, retention: Temporal.Duration): Promise<number>;
16
+ export declare function minValid(sql: Pick<postgres.Sql, 'unsafe'>): Promise<number>;
17
+ export declare function releaseGate(sql: postgres.Sql): Promise<void>;
18
+ export declare function commitOnce(sql: Queries, connector: string, at: Date, retention: Temporal.Duration): Promise<number>;
19
+ export declare function expireBefore(sql: Pick<postgres.Sql, 'unsafe'>, changedIn: number, by: 'type' | 'clear'): Promise<void>;
20
+ export type Numbering = {
21
+ readonly epoch: string;
22
+ readonly minValid: number;
23
+ readonly expiredBy: 'retention' | 'type' | 'clear' | null;
24
+ readonly settled: number;
25
+ readonly ends: readonly number[];
26
+ };
27
+ export declare function settledMark(sql: postgres.Sql): Promise<number | null>;
28
+ export declare function numberingIn(sql: Queries, settled: number): Promise<Numbering | null>;
29
+ export declare function committedAt(sql: Queries, number: number): Promise<{
30
+ readonly at: Date;
31
+ readonly newest: number;
32
+ } | undefined>;
33
+ export declare function positionsAfter(sql: postgres.Sql, after: string | undefined): Promise<readonly string[] | {
34
+ readonly expired: string;
35
+ }>;
36
+ export type Position = {
37
+ readonly epoch: string;
38
+ readonly number: number;
39
+ };
40
+ export declare function expiry(numbered: Numbering | null, { epoch, number }: Position): string | undefined;
41
+ export declare function parsePosition(position: string): Position;
42
+ export {};
@@ -0,0 +1,311 @@
1
+ import { quote } from './identifier.js';
2
+ import { lockSchema } from './postgres-session.js';
3
+ // The schema that every destination of the database shares, beside the sync
4
+ // history, so one numbering orders the commits of all its schemas.
5
+ const shared = '_warehouse';
6
+ const epochs = `${shared}."_elt_epoch"`;
7
+ const loads = `${shared}."_elt_loads"`;
8
+ const commits = `${shared}."_elt_commits"`;
9
+ const numbers = `${shared}."_elt_commit_numbers"`;
10
+ // How the database numbers its commits. The epoch names the numbering: it is
11
+ // made with the tables, so a database made again starts a numbering of its
12
+ // own. min_valid is the oldest commit that a reading can read as of: a
13
+ // change of the type of a column that is not a widening expires each
14
+ // position before its commit. A number is an integer, so the sequence fails
15
+ // at its end rather than wrap to a number that names another commit.
16
+ const tables = [
17
+ `CREATE SCHEMA IF NOT EXISTS ${shared}`,
18
+ `CREATE TABLE IF NOT EXISTS ${epochs} ("id" integer PRIMARY KEY CHECK ("id" = 1), "epoch" text NOT NULL, "min_valid" integer NOT NULL DEFAULT 0, "expired_by" text CHECK ("expired_by" IN ('retention', 'type', 'clear')))`,
19
+ `CREATE TABLE IF NOT EXISTS ${loads} ("id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, "connector" text, "dataset" text, "opened_at" timestamptz NOT NULL, "closed_at" timestamptz, CHECK (("connector" IS NULL) <> ("dataset" IS NULL)))`,
20
+ `CREATE INDEX IF NOT EXISTS "_elt_loads_open" ON ${loads} ("connector", "dataset") WHERE "closed_at" IS NULL`,
21
+ `CREATE SEQUENCE IF NOT EXISTS ${numbers} AS integer`,
22
+ `CREATE TABLE IF NOT EXISTS ${commits} ("number" integer PRIMARY KEY, "load" bigint NOT NULL REFERENCES ${loads} ("id"), "committed_at" timestamptz NOT NULL)`,
23
+ `CREATE INDEX IF NOT EXISTS "_elt_commits_load" ON ${commits} ("load", "number")`,
24
+ `INSERT INTO ${epochs} ("id", "epoch") VALUES (1, gen_random_uuid()::text) ON CONFLICT DO NOTHING`,
25
+ ];
26
+ // A commit holds a transaction lock on its number from when it takes it
27
+ // until it ends, in a key space of its own. Postgres releases the locks of a
28
+ // transaction only after its commit is visible (CommitTransaction in xact.c),
29
+ // so a number whose lock is gone is a commit that a new snapshot sees, or one
30
+ // that rolled back.
31
+ const numberLock = `hashtext('elt:commit')`;
32
+ // Taking a number and locking it are two steps, which a reader must not see
33
+ // between: each commit holds this lock shared across them, and a reader holds
34
+ // it alone while it reads the numbers taken and the locks held. It is the
35
+ // readers-writer lock of Courtois, Heymans and Parnas (1971) turned around,
36
+ // with the commits as the many readers, and a commit holds it only for those
37
+ // two steps, so no commit waits for another.
38
+ const gate = `hashtextextended('elt:commit-gate', 0)`;
39
+ // Makes the numbering if the database has none, under the lock of the shared
40
+ // schema: Postgres refuses to make one schema in two sessions at once (ADR
41
+ // 0003). One transaction makes all of it, so its epoch tells that it is
42
+ // there, and the DDL runs only while it is not: CREATE INDEX IF NOT EXISTS
43
+ // takes a share lock on its table even when the index exists, which would
44
+ // wait for every commit in flight. A session that waited for the lock finds
45
+ // the objects another one made, which IF NOT EXISTS skips.
46
+ export async function installNumbering(sql) {
47
+ const [numbering] = await sql `SELECT to_regclass(${epochs}) IS NOT NULL AS "made"`;
48
+ if (numbering?.made === true)
49
+ return;
50
+ await sql.begin(async (transaction) => {
51
+ await lockSchema(transaction, shared);
52
+ for (const statement of tables)
53
+ await transaction.unsafe(statement);
54
+ });
55
+ }
56
+ // The column of _elt_loads that names owner, and its name there.
57
+ function owned(owner) {
58
+ return 'connector' in owner
59
+ ? ['connector', owner.connector]
60
+ : ['dataset', owner.dataset];
61
+ }
62
+ // Lets role read the positions and the readings of the database with its own
63
+ // session, as a reader that did not load does: the numbering, and nothing
64
+ // else of _warehouse. Its object names stay this module's own.
65
+ export async function grantReading(sql, role) {
66
+ await sql.unsafe(`GRANT USAGE ON SCHEMA ${shared} TO ${quote(role)}`);
67
+ await sql.unsafe(`GRANT SELECT ON ${epochs}, ${commits}, ${loads}, ${numbers} TO ${quote(role)}`);
68
+ }
69
+ // Records a load of owner that opens at openedAt, and gives its id. A
70
+ // connection runs one pass at a time, and a dataset one refresh, so a load of
71
+ // it that is still open is one whose process ended during it: the new load
72
+ // closes it, as the sync history of a sole writer fails such a pass.
73
+ export async function openLoad(sql, owner, openedAt) {
74
+ const [column, name] = owned(owner);
75
+ return sql.begin(async (transaction) => {
76
+ const [load] = await transaction `
77
+ INSERT INTO ${transaction.unsafe(loads)} (${transaction(column)}, "opened_at")
78
+ VALUES (${name}, ${openedAt}) RETURNING "id"`;
79
+ if (load === undefined)
80
+ throw new Error('The load was not recorded');
81
+ await transaction `
82
+ UPDATE ${transaction.unsafe(loads)} SET "closed_at" = ${openedAt}
83
+ WHERE ${transaction(column)} = ${name} AND "closed_at" IS NULL AND "id" <> ${load.id}`;
84
+ return load.id;
85
+ });
86
+ }
87
+ export async function closeLoad(sql, load, closedAt) {
88
+ await sql `
89
+ UPDATE ${sql.unsafe(loads)} SET "closed_at" = ${closedAt}
90
+ WHERE "id" = ${load} AND "closed_at" IS NULL`;
91
+ }
92
+ // Takes the next number inside the transaction of a commit of load, and
93
+ // records the commit. A load that a later load of its owner closed has
94
+ // ended, and a commit after its end would move a position that readers were
95
+ // already given: the commit locks the row of its load, so a close that runs
96
+ // at the same time waits for the commit or makes it fail.
97
+ export async function commitNumber(sql, { load, owner }, committedAt, retention) {
98
+ await sql `SELECT pg_advisory_lock_shared(${sql.unsafe(gate)})`;
99
+ const [taken] = await sql `
100
+ SELECT "number", pg_advisory_xact_lock(${sql.unsafe(numberLock)}, "number")
101
+ FROM (SELECT nextval(${numbers})::integer AS "number") AS "next"`;
102
+ await sql `SELECT pg_advisory_unlock_shared(${sql.unsafe(gate)})`;
103
+ if (taken === undefined)
104
+ throw new Error('No commit number was taken');
105
+ const [open] = await sql `
106
+ SELECT 1 FROM ${sql.unsafe(loads)}
107
+ WHERE "id" = ${load} AND "closed_at" IS NULL FOR SHARE`;
108
+ if (open === undefined)
109
+ throw new Error('connector' in owner
110
+ ? `A later load of connection ${owner.connector} closed this load, so it cannot commit; a connection runs one pass at a time`
111
+ : `A later refresh of dataset ${owner.dataset} closed this one, so it cannot commit; a dataset refreshes one at a time`);
112
+ await sql `
113
+ INSERT INTO ${sql.unsafe(commits)} ("number", "load", "committed_at")
114
+ VALUES (${taken.number}, ${load}, ${committedAt})`;
115
+ await retain(sql, committedAt, retention);
116
+ return taken.number;
117
+ }
118
+ // Applies the retention inside a commit's transaction, as Delta Lake expires
119
+ // its log at a checkpoint: the newest position whose commit is older than the
120
+ // window becomes min_valid. It is a position, so the newest position stays
121
+ // readable while a pass stays open longer than the window. Each writer then
122
+ // deletes the rows of its own target's history that only readings before it
123
+ // read. One numbering serves every destination of the database, so the
124
+ // shortest retention among them decides.
125
+ //
126
+ // The numbering then keeps no commit before min_valid and no closed load left
127
+ // with none, so it holds only what a reading still reads: min_valid is a
128
+ // position, so each load that has a commit before it ends at or before it,
129
+ // and keeps min_valid's own commit, whose committed_at a reading there reads.
130
+ // A committed_in of a row only compares with numbers, and needs no commit
131
+ // row. An open load keeps its row with or without commits. Commits of other
132
+ // connectors run at the same time, so min_valid only moves forward: the row
133
+ // of the epoch is locked until the commit that moved it ends, and a commit
134
+ // that read an older min_valid before that finds it moved and deletes nothing.
135
+ async function retain(sql, at, retention) {
136
+ const cutoff = new Date(at.getTime() - retention.total('milliseconds'));
137
+ const [row] = await sql `
138
+ SELECT (SELECT coalesce(max("number"), 0) FROM ${sql.unsafe(commits)} WHERE "committed_at" < ${cutoff}) AS "older", "min_valid"
139
+ FROM ${sql.unsafe(epochs)}`;
140
+ if (row === undefined || row.older <= row.min_valid)
141
+ return;
142
+ const numbered = await numberingIn(sql, row.older);
143
+ const expired = numbered?.ends.at(-1);
144
+ if (expired === undefined || expired <= row.min_valid)
145
+ return;
146
+ const moved = await sql `
147
+ UPDATE ${sql.unsafe(epochs)} SET "min_valid" = ${expired}, "expired_by" = 'retention'
148
+ WHERE "min_valid" < ${expired} RETURNING 1`;
149
+ if (moved.length === 0)
150
+ return;
151
+ await sql `DELETE FROM ${sql.unsafe(commits)} WHERE "number" < ${expired}`;
152
+ await sql `
153
+ DELETE FROM ${sql.unsafe(loads)} AS l
154
+ WHERE l."closed_at" IS NOT NULL
155
+ AND NOT EXISTS (SELECT 1 FROM ${sql.unsafe(commits)} AS c WHERE c."load" = l."id")`;
156
+ }
157
+ // The oldest commit that a reading can read as of, inside the transaction
158
+ // sql runs in.
159
+ export async function minValid(sql) {
160
+ const [row] = await sql.unsafe(`SELECT "min_valid" FROM ${epochs}`);
161
+ return row?.min_valid ?? 0;
162
+ }
163
+ // Lets go of the gate that a commit holds while it takes its number. A commit
164
+ // that failed between taking the gate and letting it go left it held by the
165
+ // session, which outlives the transaction; letting go of a lock the session
166
+ // does not hold only warns.
167
+ export async function releaseGate(sql) {
168
+ await sql `SELECT pg_advisory_unlock_shared(${sql.unsafe(gate)})`;
169
+ }
170
+ // The number of a change that is a load of one commit, such as a clear,
171
+ // taken inside the caller's transaction: the load opens, commits and closes
172
+ // in it. It is not a pass, so it leaves the open loads of its connector open.
173
+ export async function commitOnce(sql, connector, at, retention) {
174
+ const [load] = await sql `
175
+ INSERT INTO ${sql.unsafe(loads)} ("connector", "opened_at")
176
+ VALUES (${connector}, ${at}) RETURNING "id"`;
177
+ if (load === undefined)
178
+ throw new Error('The load was not recorded');
179
+ const number = await commitNumber(sql, { load: load.id, owner: { connector } }, at, retention);
180
+ await closeLoad(sql, load.id, at);
181
+ return number;
182
+ }
183
+ // Makes each position before the commit changedIn expire, inside that
184
+ // commit, for by: a change of type that is not a widening, or a clear that
185
+ // forgets the rows it removes.
186
+ export async function expireBefore(sql, changedIn, by) {
187
+ await sql.unsafe(`UPDATE ${epochs} SET "min_valid" = $1, "expired_by" = $2 WHERE "min_valid" < $1`, [changedIn, by]);
188
+ }
189
+ // The high-water mark of the numbering, or null when the database has none.
190
+ // pg_locks is not read in a snapshot, so a reader takes the mark first and
191
+ // its snapshot after: each commit at or below the mark has ended by then, so
192
+ // the snapshot sees it.
193
+ export async function settledMark(sql) {
194
+ try {
195
+ return await sql.begin(async (transaction) => {
196
+ await transaction `SELECT pg_advisory_xact_lock(${transaction.unsafe(gate)})`;
197
+ const [mark] = await transaction `
198
+ SELECT CASE WHEN "is_called" THEN "last_value" ELSE 0 END::integer AS "taken",
199
+ (SELECT min("objid"::bigint)::integer FROM pg_locks
200
+ WHERE "locktype" = 'advisory' AND "objsubid" = 2
201
+ AND "database" = (SELECT "oid" FROM pg_database WHERE "datname" = current_database())
202
+ AND "classid" = (${transaction.unsafe(numberLock)}::bigint & 4294967295)::oid) AS "held"
203
+ FROM ${transaction.unsafe(numbers)}`;
204
+ if (mark === undefined)
205
+ throw new Error('The sequence has no row');
206
+ return mark.held === null ? mark.taken : mark.held - 1;
207
+ });
208
+ }
209
+ catch (error) {
210
+ if (Object(error).code === '42P01')
211
+ return null;
212
+ throw error;
213
+ }
214
+ }
215
+ // The numbering as the snapshot of the transaction sql runs in sees it, with
216
+ // settled taken before the snapshot. A position is a commit that ends a
217
+ // closed load where no load straddles it, so a reader there sees no part of
218
+ // a pass. Commits past the mark that the snapshot sees still reach across
219
+ // the ends before them.
220
+ export async function numberingIn(sql, settled) {
221
+ const [row] = await sql `SELECT "epoch", "min_valid", "expired_by" FROM ${sql.unsafe(epochs)}`;
222
+ if (row === undefined)
223
+ return null;
224
+ const spans = (await sql `
225
+ SELECT min(c."number") AS "first",
226
+ CASE WHEN l."closed_at" IS NULL THEN NULL ELSE max(c."number") END AS "last"
227
+ FROM ${sql.unsafe(commits)} AS c
228
+ JOIN ${sql.unsafe(loads)} AS l ON l."id" = c."load"
229
+ GROUP BY l."id" ORDER BY "first"`).map(({ first, last }) => ({ first, last: last ?? Infinity }));
230
+ return {
231
+ epoch: row.epoch,
232
+ minValid: row.min_valid,
233
+ expiredBy: row.expired_by,
234
+ settled,
235
+ ends: ends(spans, settled).filter((end) => end >= row.min_valid),
236
+ };
237
+ }
238
+ // When the commit numbered number was made, and the newest number that the
239
+ // snapshot of the transaction sql runs in sees.
240
+ export async function committedAt(sql, number) {
241
+ const [row] = await sql `
242
+ SELECT "committed_at" AS "at", (SELECT max("number") FROM ${sql.unsafe(commits)}) AS "newest"
243
+ FROM ${sql.unsafe(commits)} WHERE "number" = ${number}`;
244
+ return row;
245
+ }
246
+ async function readNumbering(sql) {
247
+ const settled = await settledMark(sql);
248
+ if (settled === null)
249
+ return null;
250
+ return sql.begin('ISOLATION LEVEL REPEATABLE READ READ ONLY', (transaction) => numberingIn(transaction, settled));
251
+ }
252
+ // A sweep line over the loads in order of their first commits: an end is a
253
+ // position when no load that began at or before it ends after it. An open
254
+ // load has no end yet, so it reaches past every later commit.
255
+ function ends(spans, settled) {
256
+ const found = [];
257
+ const pending = spans.values();
258
+ let span = pending.next();
259
+ let reach = 0;
260
+ for (const end of spans
261
+ .map(({ last }) => last)
262
+ .filter((last) => last <= settled)
263
+ .sort((left, right) => left - right)) {
264
+ for (; !span.done && span.value.first <= end; span = pending.next())
265
+ reach = Math.max(reach, span.value.last);
266
+ if (reach === end)
267
+ found.push(end);
268
+ }
269
+ return found;
270
+ }
271
+ // The positions of the database after `after`, ascending, or all of them. A
272
+ // position of another epoch, or past the newest settled commit, is expired.
273
+ export async function positionsAfter(sql, after) {
274
+ const since = after === undefined ? undefined : parsePosition(after);
275
+ const numbered = await readNumbering(sql);
276
+ if (since === undefined)
277
+ return numbered === null
278
+ ? []
279
+ : numbered.ends.map((end) => `${numbered.epoch}:${end}`);
280
+ const expired = expiry(numbered, since);
281
+ if (expired !== undefined)
282
+ return { expired };
283
+ return (numbered?.ends ?? [])
284
+ .filter((end) => end > since.number)
285
+ .map((end) => `${since.epoch}:${end}`);
286
+ }
287
+ export function expiry(numbered, { epoch, number }) {
288
+ const position = `${epoch}:${number}`;
289
+ if (numbered === null)
290
+ return `Position ${position} is of commits this database does not have`;
291
+ if (epoch !== numbered.epoch)
292
+ return `Position ${position} is of an earlier numbering: the commits of this database were numbered again since`;
293
+ if (number > numbered.settled)
294
+ return `Position ${position} is past the newest commit of this database, ${numbered.settled}`;
295
+ if (number < numbered.minValid)
296
+ switch (numbered.expiredBy) {
297
+ case 'type':
298
+ return `Position ${position} is before commit ${numbered.minValid}, which changed the type of a column to one that does not hold each earlier value: the history keeps no row that only a reading before that commit reads`;
299
+ case 'clear':
300
+ return `Position ${position} is before commit ${numbered.minValid}, a clear that forgot the rows of its table: the history keeps no row that only a reading before it reads`;
301
+ default:
302
+ return `Position ${position} is before commit ${numbered.minValid}, the oldest position that the retention of the database keeps readable: the history keeps no row that only a reading before it reads`;
303
+ }
304
+ return undefined;
305
+ }
306
+ export function parsePosition(position) {
307
+ const groups = /^(?<epoch>[^:]+):(?<number>0|[1-9]\d*)$/.exec(position)?.groups;
308
+ if (groups?.epoch === undefined || groups.number === undefined)
309
+ throw new TypeError(`${position} is not a position of a Postgres database`);
310
+ return { epoch: groups.epoch, number: Number(groups.number) };
311
+ }
@@ -0,0 +1,4 @@
1
+ import { PostgresAppendWriter } from './postgres-append-writer.ts';
2
+ export declare class PostgresOverwriteWriter extends PostgresAppendWriter {
3
+ protected get replaces(): boolean;
4
+ }
@@ -0,0 +1,8 @@
1
+ import { PostgresAppendWriter } from './postgres-append-writer.js';
2
+ // Replaces the target at its commit with DELETE, not TRUNCATE: TRUNCATE's
3
+ // exclusive lock would block readers for the rest of the run.
4
+ export class PostgresOverwriteWriter extends PostgresAppendWriter {
5
+ get replaces() {
6
+ return true;
7
+ }
8
+ }
@@ -0,0 +1,7 @@
1
+ import type { Queries } from './postgres-session.ts';
2
+ export declare class Privileges {
3
+ #private;
4
+ private constructor();
5
+ static of(sql: Queries, relation: string): Promise<Privileges>;
6
+ apply(sql: Queries, relation: string): Promise<void>;
7
+ }
@@ -0,0 +1,50 @@
1
+ import { quote } from './identifier.js';
2
+ // Each privilege on the relation and on its columns, the owner's own
3
+ // included, as aclexplode gives them: a relation with no ACL holds its
4
+ // owner's defaults.
5
+ const grantsOf = `
6
+ SELECT NULL::text AS "column",
7
+ CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END AS "grantee",
8
+ g.privilege_type AS "privilege", g.is_grantable AS "grantable"
9
+ FROM pg_class c, aclexplode(coalesce(c.relacl, acldefault('r', c.relowner))) g
10
+ WHERE c.oid = to_regclass($1)
11
+ UNION ALL
12
+ SELECT a.attname::text,
13
+ CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END,
14
+ g.privilege_type, g.is_grantable
15
+ FROM pg_attribute a, aclexplode(a.attacl) g
16
+ WHERE a.attrelid = to_regclass($1) AND a.attnum > 0 AND NOT a.attisdropped`;
17
+ const key = ({ column, grantee, privilege, grantable }) => JSON.stringify([column, grantee, privilege, grantable]);
18
+ // The privileges of a relation, kept so that the relation that takes its
19
+ // name, a view made again or a table swapped in, holds exactly them, as
20
+ // Snowflake's COPY GRANTS keeps them across CREATE OR REPLACE. Applying them
21
+ // is a difference, as dbt applies its grants: it revokes each privilege the
22
+ // new relation has that the old one did not, such as one that default
23
+ // privileges gave it, and grants each that it lacks.
24
+ export class Privileges {
25
+ #grants;
26
+ constructor(grants) {
27
+ this.#grants = grants;
28
+ }
29
+ static async of(sql, relation) {
30
+ return new Privileges(await sql.unsafe(grantsOf, [relation]));
31
+ }
32
+ async apply(sql, relation) {
33
+ const keptKeys = new Set(this.#grants.map(key));
34
+ const held = await sql.unsafe(grantsOf, [relation]);
35
+ const heldKeys = new Set(held.map(key));
36
+ for (const grant of held)
37
+ if (!keptKeys.has(key(grant)))
38
+ await sql.unsafe(statement('REVOKE', grant, relation));
39
+ for (const grant of this.#grants)
40
+ if (!heldKeys.has(key(grant)))
41
+ await sql.unsafe(statement('GRANT', grant, relation));
42
+ }
43
+ }
44
+ function statement(verb, { column, grantee, privilege, grantable }, relation) {
45
+ const on = column === null ? '' : ` (${quote(column)})`;
46
+ const role = grantee === 'PUBLIC' ? 'PUBLIC' : quote(grantee);
47
+ return verb === 'GRANT'
48
+ ? `GRANT ${privilege}${on} ON ${relation} TO ${role}${grantable ? ' WITH GRANT OPTION' : ''}`
49
+ : `REVOKE ${privilege}${on} ON ${relation} FROM ${role} CASCADE`;
50
+ }
@@ -0,0 +1,11 @@
1
+ import type { TargetComments } from './postgres-comments.ts';
2
+ import type { Transaction } from './postgres-load.ts';
3
+ import type { PostgresReaderView } from './postgres-table.ts';
4
+ export declare class TargetReaderView {
5
+ #private;
6
+ constructor(view: PostgresReaderView, target: string, history: string, columns: readonly string[], comments: TargetComments);
7
+ get qualifiedName(): string;
8
+ refuse(sql: Transaction, stale: boolean): Promise<void>;
9
+ describe(sql: Transaction): Promise<void>;
10
+ drop(sql: Transaction): Promise<void>;
11
+ }
@@ -0,0 +1,64 @@
1
+ import { quote } from './identifier.js';
2
+ // The position a reader reads at: the one a reading set for its
3
+ // transaction, or, for a session that set none, past every commit, so the
4
+ // view shows the target as it is now and no row of its history.
5
+ const position = `coalesce(nullif(current_setting('elt.position', true), '')::integer, 2147483647)`;
6
+ // The view readers see a target through: exactly the stream's columns and
7
+ // loaded_at, none of the internal ones, created with the target and
8
+ // described by the comments of those columns. It reads the target as of the
9
+ // position its reader set: the rows that the target had then, and that it
10
+ // still has or its history kept.
11
+ export class TargetReaderView {
12
+ #view;
13
+ // The target's and its history's qualified names, and the columns the
14
+ // view lists, with their comments.
15
+ #target;
16
+ #history;
17
+ #columns;
18
+ #comments;
19
+ constructor(view, target, history, columns, comments) {
20
+ this.#view = view;
21
+ this.#target = target;
22
+ this.#history = history;
23
+ this.#columns = columns;
24
+ this.#comments = comments;
25
+ Object.freeze(this);
26
+ }
27
+ get qualifiedName() {
28
+ return `${quote(this.#view.schema)}.${quote(this.#view.name)}`;
29
+ }
30
+ // Refuses, before anything is read, a reader view this load did not make:
31
+ // not a view, or a view of other columns or another table. A stale
32
+ // target's view still shows the stored columns, which evolving it replaces.
33
+ async refuse(sql, stale) {
34
+ const view = this.qualifiedName;
35
+ const [existing] = await sql.unsafe(`SELECT c.relkind,
36
+ ARRAY(SELECT attname::text FROM pg_attribute
37
+ WHERE attrelid = c.oid AND attnum > 0 AND NOT attisdropped ORDER BY attnum) AS columns,
38
+ EXISTS (SELECT 1 FROM pg_rewrite r JOIN pg_depend d
39
+ ON d.classid = 'pg_rewrite'::regclass AND d.objid = r.oid
40
+ WHERE r.ev_class = c.oid AND d.refobjid = to_regclass($2)) AS reads
41
+ FROM pg_class c WHERE c.oid = to_regclass($1)`, [view, this.#target]);
42
+ if (existing === undefined)
43
+ return;
44
+ if (existing.relkind !== 'v' ||
45
+ !existing.reads ||
46
+ (!stale && existing.columns.join('\0') !== this.#columns.join('\0')))
47
+ throw new TypeError(`${view} is not a view of exactly ${this.#target}; drop it or reset the warehouse`);
48
+ }
49
+ // Created only when absent and never replaced by a commit: replacing a view
50
+ // readers can see would lock them out. prepare refused any view not this
51
+ // load's own.
52
+ async describe(sql) {
53
+ const view = this.qualifiedName;
54
+ const [existing] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [view]);
55
+ if (existing?.exists !== true) {
56
+ const columns = this.#columns.map(quote);
57
+ await sql.unsafe(`CREATE VIEW ${view} AS SELECT ${columns.join(', ')} FROM ${this.#target} WHERE "committed_in" <= ${position} UNION ALL SELECT ${columns.join(', ')} FROM ${this.#history} AS "_elt_old" WHERE "_elt_old"."committed_in" <= ${position} AND "_elt_old"."replaced_in" > ${position}`);
58
+ }
59
+ await this.#comments.write(sql, 'VIEW', this.#view.schema, this.#view.name);
60
+ }
61
+ async drop(sql) {
62
+ await sql.unsafe(`DROP VIEW IF EXISTS ${this.qualifiedName}`);
63
+ }
64
+ }
@@ -0,0 +1,11 @@
1
+ import postgres from 'postgres';
2
+ export type Queries = Pick<postgres.Sql, 'unsafe'>;
3
+ export declare class PostgresSession implements AsyncDisposable {
4
+ readonly sql: postgres.Sql;
5
+ constructor(url: string, applicationName: string);
6
+ [Symbol.asyncDispose](): Promise<void>;
7
+ }
8
+ export declare function connectionUrl(url: string): string;
9
+ export declare function schemaName(schema: string): string;
10
+ export declare function lockSchema(sql: postgres.ISql, schema: string): Promise<void>;
11
+ export declare function makeSchema(sql: postgres.Sql, schema: string): Promise<void>;
@@ -0,0 +1,61 @@
1
+ import postgres from 'postgres';
2
+ import { identifier, quote } from './identifier.js';
3
+ // One session, closed however the work using it ends.
4
+ export class PostgresSession {
5
+ sql;
6
+ constructor(url, applicationName) {
7
+ this.sql = postgres(url, {
8
+ max: 1,
9
+ // postgres.js replaces a connection after 30 to 60 minutes by default,
10
+ // which silently drops a session's TEMP stages and session locks.
11
+ max_lifetime: null,
12
+ onnotice: () => { },
13
+ connection: { application_name: applicationName },
14
+ });
15
+ }
16
+ async [Symbol.asyncDispose]() {
17
+ await this.sql.end({ timeout: 5 });
18
+ }
19
+ }
20
+ export function connectionUrl(url) {
21
+ const parsed = URL.parse(url);
22
+ if (parsed === null ||
23
+ (parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:'))
24
+ throw new TypeError('Postgres requires a postgres:// connection URL');
25
+ return url;
26
+ }
27
+ export function schemaName(schema) {
28
+ identifier(schema, 'schema name');
29
+ if (/^pg_/i.test(schema))
30
+ throw new TypeError('Schema names starting with pg_ are reserved');
31
+ return schema;
32
+ }
33
+ // The advisory lock of a schema's DDL, as hashtextextended keys it.
34
+ const schemaKey = (schema) => `elt:${schema}`;
35
+ // Serializes, until the transaction ends, the DDL that makes a schema and the
36
+ // objects its loads, views and checkpoints share: Postgres refuses a second
37
+ // session's CREATE SCHEMA or CREATE TABLE of the same name with 23505.
38
+ export async function lockSchema(sql, schema) {
39
+ await sql `SELECT pg_advisory_xact_lock(hashtextextended(${schemaKey(schema)}, 0))`;
40
+ }
41
+ // Makes the schema in a transaction of its own, which begins only once the
42
+ // schema's lock is held. A session refreshes its catalog cache when a
43
+ // transaction begins and when it locks a relation, not when it takes an
44
+ // advisory lock, so a transaction that waited for the lock could still miss
45
+ // the schema that the holder made, and its CREATE SCHEMA IF NOT EXISTS would
46
+ // fail with 23505. A schema that this session already sees needs no lock; one
47
+ // that it misses, also by a stale cache, is made under the lock. sql is a
48
+ // session outside any transaction.
49
+ export async function makeSchema(sql, schema) {
50
+ const [made] = await sql `SELECT to_regnamespace(${quote(schema)}) IS NOT NULL AS "exists"`;
51
+ if (made?.exists === true)
52
+ return;
53
+ const key = schemaKey(schema);
54
+ await sql `SELECT pg_advisory_lock(hashtextextended(${key}, 0))`;
55
+ try {
56
+ await sql.begin((transaction) => transaction.unsafe(`CREATE SCHEMA IF NOT EXISTS ${quote(schema)}`));
57
+ }
58
+ finally {
59
+ await sql `SELECT pg_advisory_unlock(hashtextextended(${key}, 0))`;
60
+ }
61
+ }
@@ -0,0 +1,31 @@
1
+ import type { PostgresView } from './postgres-views.ts';
2
+ export declare const syncHistoryViews: readonly PostgresView[];
3
+ export declare const syncHistoryTables: readonly ['CREATE SCHEMA IF NOT EXISTS _warehouse', `CREATE TABLE IF NOT EXISTS _warehouse.sync_attempts (
4
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
5
+ connector text NOT NULL CHECK (btrim(connector) <> ''), source text NOT NULL,
6
+ started_at timestamptz NOT NULL,
7
+ completed_at timestamptz,
8
+ status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
9
+ error text,
10
+ failure_type text CHECK (failure_type IN ('config', 'system')),
11
+ CHECK ((status = 'running') = (completed_at IS NULL)),
12
+ CHECK ((status IN ('partial', 'failed')) = (failure_type IS NOT NULL))
13
+ )`, `CREATE OR REPLACE FUNCTION _warehouse._elt_unversioned() RETURNS boolean LANGUAGE plpgsql STABLE AS $$
14
+ BEGIN
15
+ IF coalesce(current_setting('elt.position', true), '') <> '' THEN
16
+ RAISE EXCEPTION 'The sync history cannot be read as of a position: it keeps no row versions';
17
+ END IF;
18
+ RETURN true;
19
+ END $$`, 'CREATE INDEX IF NOT EXISTS sync_attempts_connector ON _warehouse.sync_attempts (connector, id DESC)', `CREATE TABLE IF NOT EXISTS _warehouse.extraction_coverage (
20
+ attempt_id bigint NOT NULL REFERENCES _warehouse.sync_attempts(id),
21
+ stream text NOT NULL, target_schema text NOT NULL, target_table text NOT NULL,
22
+ sync_mode text NOT NULL, destination_sync_mode text NOT NULL,
23
+ description text NOT NULL CHECK (btrim(description) <> ''), selection jsonb NOT NULL,
24
+ status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
25
+ written_count bigint CHECK (written_count >= 0), deleted_count bigint CHECK (deleted_count >= 0),
26
+ failures jsonb NOT NULL DEFAULT '[]', upstream_current_as_of timestamptz,
27
+ PRIMARY KEY (attempt_id, stream)
28
+ )`, `CREATE TABLE IF NOT EXISTS _warehouse.stream_checks (
29
+ connector text NOT NULL, stream text NOT NULL, checked_at timestamptz NOT NULL,
30
+ PRIMARY KEY (connector, stream)
31
+ )`];