@eltjs/elt-postgresql 0.0.0-stage → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +813 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type postgres from 'postgres';
|
|
2
|
+
type Queries = postgres.Sql | postgres.TransactionSql;
|
|
3
|
+
export declare function installNumbering(sql: postgres.Sql): Promise<void>;
|
|
4
|
+
export type LoadOwner = {
|
|
5
|
+
readonly connector: string;
|
|
6
|
+
} | {
|
|
7
|
+
readonly dataset: string;
|
|
8
|
+
};
|
|
9
|
+
export declare function grantReading(sql: Queries, role: string): Promise<void>;
|
|
10
|
+
export declare function openLoad(sql: postgres.Sql, owner: LoadOwner, openedAt: Date): Promise<string>;
|
|
11
|
+
export declare function closeLoad(sql: Queries, load: string, closedAt: Date): Promise<void>;
|
|
12
|
+
export declare function commitNumber(sql: Queries, { load, owner }: {
|
|
13
|
+
load: string;
|
|
14
|
+
owner: LoadOwner;
|
|
15
|
+
}, committedAt: Date, retention: Temporal.Duration): Promise<number>;
|
|
16
|
+
export declare function minValid(sql: Pick<postgres.Sql, 'unsafe'>): Promise<number>;
|
|
17
|
+
export declare function releaseGate(sql: postgres.Sql): Promise<void>;
|
|
18
|
+
export declare function commitOnce(sql: Queries, connector: string, at: Date, retention: Temporal.Duration): Promise<number>;
|
|
19
|
+
export declare function expireBefore(sql: Pick<postgres.Sql, 'unsafe'>, changedIn: number, by: 'type' | 'clear'): Promise<void>;
|
|
20
|
+
export type Numbering = {
|
|
21
|
+
readonly epoch: string;
|
|
22
|
+
readonly minValid: number;
|
|
23
|
+
readonly expiredBy: 'retention' | 'type' | 'clear' | null;
|
|
24
|
+
readonly settled: number;
|
|
25
|
+
readonly ends: readonly number[];
|
|
26
|
+
};
|
|
27
|
+
export declare function settledMark(sql: postgres.Sql): Promise<number | null>;
|
|
28
|
+
export declare function numberingIn(sql: Queries, settled: number): Promise<Numbering | null>;
|
|
29
|
+
export declare function committedAt(sql: Queries, number: number): Promise<{
|
|
30
|
+
readonly at: Date;
|
|
31
|
+
readonly newest: number;
|
|
32
|
+
} | undefined>;
|
|
33
|
+
export declare function positionsAfter(sql: postgres.Sql, after: string | undefined): Promise<readonly string[] | {
|
|
34
|
+
readonly expired: string;
|
|
35
|
+
}>;
|
|
36
|
+
export type Position = {
|
|
37
|
+
readonly epoch: string;
|
|
38
|
+
readonly number: number;
|
|
39
|
+
};
|
|
40
|
+
export declare function expiry(numbered: Numbering | null, { epoch, number }: Position): string | undefined;
|
|
41
|
+
export declare function parsePosition(position: string): Position;
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import { quote } from './identifier.js';
|
|
2
|
+
import { lockSchema } from './postgres-session.js';
|
|
3
|
+
// The schema that every destination of the database shares, beside the sync
|
|
4
|
+
// history, so one numbering orders the commits of all its schemas.
|
|
5
|
+
const shared = '_warehouse';
|
|
6
|
+
const epochs = `${shared}."_elt_epoch"`;
|
|
7
|
+
const loads = `${shared}."_elt_loads"`;
|
|
8
|
+
const commits = `${shared}."_elt_commits"`;
|
|
9
|
+
const numbers = `${shared}."_elt_commit_numbers"`;
|
|
10
|
+
// How the database numbers its commits. The epoch names the numbering: it is
|
|
11
|
+
// made with the tables, so a database made again starts a numbering of its
|
|
12
|
+
// own. min_valid is the oldest commit that a reading can read as of: a
|
|
13
|
+
// change of the type of a column that is not a widening expires each
|
|
14
|
+
// position before its commit. A number is an integer, so the sequence fails
|
|
15
|
+
// at its end rather than wrap to a number that names another commit.
|
|
16
|
+
const tables = [
|
|
17
|
+
`CREATE SCHEMA IF NOT EXISTS ${shared}`,
|
|
18
|
+
`CREATE TABLE IF NOT EXISTS ${epochs} ("id" integer PRIMARY KEY CHECK ("id" = 1), "epoch" text NOT NULL, "min_valid" integer NOT NULL DEFAULT 0, "expired_by" text CHECK ("expired_by" IN ('retention', 'type', 'clear')))`,
|
|
19
|
+
`CREATE TABLE IF NOT EXISTS ${loads} ("id" bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, "connector" text, "dataset" text, "opened_at" timestamptz NOT NULL, "closed_at" timestamptz, CHECK (("connector" IS NULL) <> ("dataset" IS NULL)))`,
|
|
20
|
+
`CREATE INDEX IF NOT EXISTS "_elt_loads_open" ON ${loads} ("connector", "dataset") WHERE "closed_at" IS NULL`,
|
|
21
|
+
`CREATE SEQUENCE IF NOT EXISTS ${numbers} AS integer`,
|
|
22
|
+
`CREATE TABLE IF NOT EXISTS ${commits} ("number" integer PRIMARY KEY, "load" bigint NOT NULL REFERENCES ${loads} ("id"), "committed_at" timestamptz NOT NULL)`,
|
|
23
|
+
`CREATE INDEX IF NOT EXISTS "_elt_commits_load" ON ${commits} ("load", "number")`,
|
|
24
|
+
`INSERT INTO ${epochs} ("id", "epoch") VALUES (1, gen_random_uuid()::text) ON CONFLICT DO NOTHING`,
|
|
25
|
+
];
|
|
26
|
+
// A commit holds a transaction lock on its number from when it takes it
|
|
27
|
+
// until it ends, in a key space of its own. Postgres releases the locks of a
|
|
28
|
+
// transaction only after its commit is visible (CommitTransaction in xact.c),
|
|
29
|
+
// so a number whose lock is gone is a commit that a new snapshot sees, or one
|
|
30
|
+
// that rolled back.
|
|
31
|
+
const numberLock = `hashtext('elt:commit')`;
|
|
32
|
+
// Taking a number and locking it are two steps, which a reader must not see
|
|
33
|
+
// between: each commit holds this lock shared across them, and a reader holds
|
|
34
|
+
// it alone while it reads the numbers taken and the locks held. It is the
|
|
35
|
+
// readers-writer lock of Courtois, Heymans and Parnas (1971) turned around,
|
|
36
|
+
// with the commits as the many readers, and a commit holds it only for those
|
|
37
|
+
// two steps, so no commit waits for another.
|
|
38
|
+
const gate = `hashtextextended('elt:commit-gate', 0)`;
|
|
39
|
+
// Makes the numbering if the database has none, under the lock of the shared
|
|
40
|
+
// schema: Postgres refuses to make one schema in two sessions at once (ADR
|
|
41
|
+
// 0003). One transaction makes all of it, so its epoch tells that it is
|
|
42
|
+
// there, and the DDL runs only while it is not: CREATE INDEX IF NOT EXISTS
|
|
43
|
+
// takes a share lock on its table even when the index exists, which would
|
|
44
|
+
// wait for every commit in flight. A session that waited for the lock finds
|
|
45
|
+
// the objects another one made, which IF NOT EXISTS skips.
|
|
46
|
+
export async function installNumbering(sql) {
|
|
47
|
+
const [numbering] = await sql `SELECT to_regclass(${epochs}) IS NOT NULL AS "made"`;
|
|
48
|
+
if (numbering?.made === true)
|
|
49
|
+
return;
|
|
50
|
+
await sql.begin(async (transaction) => {
|
|
51
|
+
await lockSchema(transaction, shared);
|
|
52
|
+
for (const statement of tables)
|
|
53
|
+
await transaction.unsafe(statement);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
// The column of _elt_loads that names owner, and its name there.
|
|
57
|
+
function owned(owner) {
|
|
58
|
+
return 'connector' in owner
|
|
59
|
+
? ['connector', owner.connector]
|
|
60
|
+
: ['dataset', owner.dataset];
|
|
61
|
+
}
|
|
62
|
+
// Lets role read the positions and the readings of the database with its own
|
|
63
|
+
// session, as a reader that did not load does: the numbering, and nothing
|
|
64
|
+
// else of _warehouse. Its object names stay this module's own.
|
|
65
|
+
export async function grantReading(sql, role) {
|
|
66
|
+
await sql.unsafe(`GRANT USAGE ON SCHEMA ${shared} TO ${quote(role)}`);
|
|
67
|
+
await sql.unsafe(`GRANT SELECT ON ${epochs}, ${commits}, ${loads}, ${numbers} TO ${quote(role)}`);
|
|
68
|
+
}
|
|
69
|
+
// Records a load of owner that opens at openedAt, and gives its id. A
|
|
70
|
+
// connection runs one pass at a time, and a dataset one refresh, so a load of
|
|
71
|
+
// it that is still open is one whose process ended during it: the new load
|
|
72
|
+
// closes it, as the sync history of a sole writer fails such a pass.
|
|
73
|
+
export async function openLoad(sql, owner, openedAt) {
|
|
74
|
+
const [column, name] = owned(owner);
|
|
75
|
+
return sql.begin(async (transaction) => {
|
|
76
|
+
const [load] = await transaction `
|
|
77
|
+
INSERT INTO ${transaction.unsafe(loads)} (${transaction(column)}, "opened_at")
|
|
78
|
+
VALUES (${name}, ${openedAt}) RETURNING "id"`;
|
|
79
|
+
if (load === undefined)
|
|
80
|
+
throw new Error('The load was not recorded');
|
|
81
|
+
await transaction `
|
|
82
|
+
UPDATE ${transaction.unsafe(loads)} SET "closed_at" = ${openedAt}
|
|
83
|
+
WHERE ${transaction(column)} = ${name} AND "closed_at" IS NULL AND "id" <> ${load.id}`;
|
|
84
|
+
return load.id;
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
export async function closeLoad(sql, load, closedAt) {
|
|
88
|
+
await sql `
|
|
89
|
+
UPDATE ${sql.unsafe(loads)} SET "closed_at" = ${closedAt}
|
|
90
|
+
WHERE "id" = ${load} AND "closed_at" IS NULL`;
|
|
91
|
+
}
|
|
92
|
+
// Takes the next number inside the transaction of a commit of load, and
|
|
93
|
+
// records the commit. A load that a later load of its owner closed has
|
|
94
|
+
// ended, and a commit after its end would move a position that readers were
|
|
95
|
+
// already given: the commit locks the row of its load, so a close that runs
|
|
96
|
+
// at the same time waits for the commit or makes it fail.
|
|
97
|
+
export async function commitNumber(sql, { load, owner }, committedAt, retention) {
|
|
98
|
+
await sql `SELECT pg_advisory_lock_shared(${sql.unsafe(gate)})`;
|
|
99
|
+
const [taken] = await sql `
|
|
100
|
+
SELECT "number", pg_advisory_xact_lock(${sql.unsafe(numberLock)}, "number")
|
|
101
|
+
FROM (SELECT nextval(${numbers})::integer AS "number") AS "next"`;
|
|
102
|
+
await sql `SELECT pg_advisory_unlock_shared(${sql.unsafe(gate)})`;
|
|
103
|
+
if (taken === undefined)
|
|
104
|
+
throw new Error('No commit number was taken');
|
|
105
|
+
const [open] = await sql `
|
|
106
|
+
SELECT 1 FROM ${sql.unsafe(loads)}
|
|
107
|
+
WHERE "id" = ${load} AND "closed_at" IS NULL FOR SHARE`;
|
|
108
|
+
if (open === undefined)
|
|
109
|
+
throw new Error('connector' in owner
|
|
110
|
+
? `A later load of connection ${owner.connector} closed this load, so it cannot commit; a connection runs one pass at a time`
|
|
111
|
+
: `A later refresh of dataset ${owner.dataset} closed this one, so it cannot commit; a dataset refreshes one at a time`);
|
|
112
|
+
await sql `
|
|
113
|
+
INSERT INTO ${sql.unsafe(commits)} ("number", "load", "committed_at")
|
|
114
|
+
VALUES (${taken.number}, ${load}, ${committedAt})`;
|
|
115
|
+
await retain(sql, committedAt, retention);
|
|
116
|
+
return taken.number;
|
|
117
|
+
}
|
|
118
|
+
// Applies the retention inside a commit's transaction, as Delta Lake expires
|
|
119
|
+
// its log at a checkpoint: the newest position whose commit is older than the
|
|
120
|
+
// window becomes min_valid. It is a position, so the newest position stays
|
|
121
|
+
// readable while a pass stays open longer than the window. Each writer then
|
|
122
|
+
// deletes the rows of its own target's history that only readings before it
|
|
123
|
+
// read. One numbering serves every destination of the database, so the
|
|
124
|
+
// shortest retention among them decides.
|
|
125
|
+
//
|
|
126
|
+
// The numbering then keeps no commit before min_valid and no closed load left
|
|
127
|
+
// with none, so it holds only what a reading still reads: min_valid is a
|
|
128
|
+
// position, so each load that has a commit before it ends at or before it,
|
|
129
|
+
// and keeps min_valid's own commit, whose committed_at a reading there reads.
|
|
130
|
+
// A committed_in of a row only compares with numbers, and needs no commit
|
|
131
|
+
// row. An open load keeps its row with or without commits. Commits of other
|
|
132
|
+
// connectors run at the same time, so min_valid only moves forward: the row
|
|
133
|
+
// of the epoch is locked until the commit that moved it ends, and a commit
|
|
134
|
+
// that read an older min_valid before that finds it moved and deletes nothing.
|
|
135
|
+
async function retain(sql, at, retention) {
|
|
136
|
+
const cutoff = new Date(at.getTime() - retention.total('milliseconds'));
|
|
137
|
+
const [row] = await sql `
|
|
138
|
+
SELECT (SELECT coalesce(max("number"), 0) FROM ${sql.unsafe(commits)} WHERE "committed_at" < ${cutoff}) AS "older", "min_valid"
|
|
139
|
+
FROM ${sql.unsafe(epochs)}`;
|
|
140
|
+
if (row === undefined || row.older <= row.min_valid)
|
|
141
|
+
return;
|
|
142
|
+
const numbered = await numberingIn(sql, row.older);
|
|
143
|
+
const expired = numbered?.ends.at(-1);
|
|
144
|
+
if (expired === undefined || expired <= row.min_valid)
|
|
145
|
+
return;
|
|
146
|
+
const moved = await sql `
|
|
147
|
+
UPDATE ${sql.unsafe(epochs)} SET "min_valid" = ${expired}, "expired_by" = 'retention'
|
|
148
|
+
WHERE "min_valid" < ${expired} RETURNING 1`;
|
|
149
|
+
if (moved.length === 0)
|
|
150
|
+
return;
|
|
151
|
+
await sql `DELETE FROM ${sql.unsafe(commits)} WHERE "number" < ${expired}`;
|
|
152
|
+
await sql `
|
|
153
|
+
DELETE FROM ${sql.unsafe(loads)} AS l
|
|
154
|
+
WHERE l."closed_at" IS NOT NULL
|
|
155
|
+
AND NOT EXISTS (SELECT 1 FROM ${sql.unsafe(commits)} AS c WHERE c."load" = l."id")`;
|
|
156
|
+
}
|
|
157
|
+
// The oldest commit that a reading can read as of, inside the transaction
|
|
158
|
+
// sql runs in.
|
|
159
|
+
export async function minValid(sql) {
|
|
160
|
+
const [row] = await sql.unsafe(`SELECT "min_valid" FROM ${epochs}`);
|
|
161
|
+
return row?.min_valid ?? 0;
|
|
162
|
+
}
|
|
163
|
+
// Lets go of the gate that a commit holds while it takes its number. A commit
|
|
164
|
+
// that failed between taking the gate and letting it go left it held by the
|
|
165
|
+
// session, which outlives the transaction; letting go of a lock the session
|
|
166
|
+
// does not hold only warns.
|
|
167
|
+
export async function releaseGate(sql) {
|
|
168
|
+
await sql `SELECT pg_advisory_unlock_shared(${sql.unsafe(gate)})`;
|
|
169
|
+
}
|
|
170
|
+
// The number of a change that is a load of one commit, such as a clear,
|
|
171
|
+
// taken inside the caller's transaction: the load opens, commits and closes
|
|
172
|
+
// in it. It is not a pass, so it leaves the open loads of its connector open.
|
|
173
|
+
export async function commitOnce(sql, connector, at, retention) {
|
|
174
|
+
const [load] = await sql `
|
|
175
|
+
INSERT INTO ${sql.unsafe(loads)} ("connector", "opened_at")
|
|
176
|
+
VALUES (${connector}, ${at}) RETURNING "id"`;
|
|
177
|
+
if (load === undefined)
|
|
178
|
+
throw new Error('The load was not recorded');
|
|
179
|
+
const number = await commitNumber(sql, { load: load.id, owner: { connector } }, at, retention);
|
|
180
|
+
await closeLoad(sql, load.id, at);
|
|
181
|
+
return number;
|
|
182
|
+
}
|
|
183
|
+
// Makes each position before the commit changedIn expire, inside that
|
|
184
|
+
// commit, for by: a change of type that is not a widening, or a clear that
|
|
185
|
+
// forgets the rows it removes.
|
|
186
|
+
export async function expireBefore(sql, changedIn, by) {
|
|
187
|
+
await sql.unsafe(`UPDATE ${epochs} SET "min_valid" = $1, "expired_by" = $2 WHERE "min_valid" < $1`, [changedIn, by]);
|
|
188
|
+
}
|
|
189
|
+
// The high-water mark of the numbering, or null when the database has none.
|
|
190
|
+
// pg_locks is not read in a snapshot, so a reader takes the mark first and
|
|
191
|
+
// its snapshot after: each commit at or below the mark has ended by then, so
|
|
192
|
+
// the snapshot sees it.
|
|
193
|
+
export async function settledMark(sql) {
|
|
194
|
+
try {
|
|
195
|
+
return await sql.begin(async (transaction) => {
|
|
196
|
+
await transaction `SELECT pg_advisory_xact_lock(${transaction.unsafe(gate)})`;
|
|
197
|
+
const [mark] = await transaction `
|
|
198
|
+
SELECT CASE WHEN "is_called" THEN "last_value" ELSE 0 END::integer AS "taken",
|
|
199
|
+
(SELECT min("objid"::bigint)::integer FROM pg_locks
|
|
200
|
+
WHERE "locktype" = 'advisory' AND "objsubid" = 2
|
|
201
|
+
AND "database" = (SELECT "oid" FROM pg_database WHERE "datname" = current_database())
|
|
202
|
+
AND "classid" = (${transaction.unsafe(numberLock)}::bigint & 4294967295)::oid) AS "held"
|
|
203
|
+
FROM ${transaction.unsafe(numbers)}`;
|
|
204
|
+
if (mark === undefined)
|
|
205
|
+
throw new Error('The sequence has no row');
|
|
206
|
+
return mark.held === null ? mark.taken : mark.held - 1;
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
if (Object(error).code === '42P01')
|
|
211
|
+
return null;
|
|
212
|
+
throw error;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// The numbering as the snapshot of the transaction sql runs in sees it, with
|
|
216
|
+
// settled taken before the snapshot. A position is a commit that ends a
|
|
217
|
+
// closed load where no load straddles it, so a reader there sees no part of
|
|
218
|
+
// a pass. Commits past the mark that the snapshot sees still reach across
|
|
219
|
+
// the ends before them.
|
|
220
|
+
export async function numberingIn(sql, settled) {
|
|
221
|
+
const [row] = await sql `SELECT "epoch", "min_valid", "expired_by" FROM ${sql.unsafe(epochs)}`;
|
|
222
|
+
if (row === undefined)
|
|
223
|
+
return null;
|
|
224
|
+
const spans = (await sql `
|
|
225
|
+
SELECT min(c."number") AS "first",
|
|
226
|
+
CASE WHEN l."closed_at" IS NULL THEN NULL ELSE max(c."number") END AS "last"
|
|
227
|
+
FROM ${sql.unsafe(commits)} AS c
|
|
228
|
+
JOIN ${sql.unsafe(loads)} AS l ON l."id" = c."load"
|
|
229
|
+
GROUP BY l."id" ORDER BY "first"`).map(({ first, last }) => ({ first, last: last ?? Infinity }));
|
|
230
|
+
return {
|
|
231
|
+
epoch: row.epoch,
|
|
232
|
+
minValid: row.min_valid,
|
|
233
|
+
expiredBy: row.expired_by,
|
|
234
|
+
settled,
|
|
235
|
+
ends: ends(spans, settled).filter((end) => end >= row.min_valid),
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
// When the commit numbered number was made, and the newest number that the
|
|
239
|
+
// snapshot of the transaction sql runs in sees.
|
|
240
|
+
export async function committedAt(sql, number) {
|
|
241
|
+
const [row] = await sql `
|
|
242
|
+
SELECT "committed_at" AS "at", (SELECT max("number") FROM ${sql.unsafe(commits)}) AS "newest"
|
|
243
|
+
FROM ${sql.unsafe(commits)} WHERE "number" = ${number}`;
|
|
244
|
+
return row;
|
|
245
|
+
}
|
|
246
|
+
async function readNumbering(sql) {
|
|
247
|
+
const settled = await settledMark(sql);
|
|
248
|
+
if (settled === null)
|
|
249
|
+
return null;
|
|
250
|
+
return sql.begin('ISOLATION LEVEL REPEATABLE READ READ ONLY', (transaction) => numberingIn(transaction, settled));
|
|
251
|
+
}
|
|
252
|
+
// A sweep line over the loads in order of their first commits: an end is a
|
|
253
|
+
// position when no load that began at or before it ends after it. An open
|
|
254
|
+
// load has no end yet, so it reaches past every later commit.
|
|
255
|
+
function ends(spans, settled) {
|
|
256
|
+
const found = [];
|
|
257
|
+
const pending = spans.values();
|
|
258
|
+
let span = pending.next();
|
|
259
|
+
let reach = 0;
|
|
260
|
+
for (const end of spans
|
|
261
|
+
.map(({ last }) => last)
|
|
262
|
+
.filter((last) => last <= settled)
|
|
263
|
+
.sort((left, right) => left - right)) {
|
|
264
|
+
for (; !span.done && span.value.first <= end; span = pending.next())
|
|
265
|
+
reach = Math.max(reach, span.value.last);
|
|
266
|
+
if (reach === end)
|
|
267
|
+
found.push(end);
|
|
268
|
+
}
|
|
269
|
+
return found;
|
|
270
|
+
}
|
|
271
|
+
// The positions of the database after `after`, ascending, or all of them. A
|
|
272
|
+
// position of another epoch, or past the newest settled commit, is expired.
|
|
273
|
+
export async function positionsAfter(sql, after) {
|
|
274
|
+
const since = after === undefined ? undefined : parsePosition(after);
|
|
275
|
+
const numbered = await readNumbering(sql);
|
|
276
|
+
if (since === undefined)
|
|
277
|
+
return numbered === null
|
|
278
|
+
? []
|
|
279
|
+
: numbered.ends.map((end) => `${numbered.epoch}:${end}`);
|
|
280
|
+
const expired = expiry(numbered, since);
|
|
281
|
+
if (expired !== undefined)
|
|
282
|
+
return { expired };
|
|
283
|
+
return (numbered?.ends ?? [])
|
|
284
|
+
.filter((end) => end > since.number)
|
|
285
|
+
.map((end) => `${since.epoch}:${end}`);
|
|
286
|
+
}
|
|
287
|
+
export function expiry(numbered, { epoch, number }) {
|
|
288
|
+
const position = `${epoch}:${number}`;
|
|
289
|
+
if (numbered === null)
|
|
290
|
+
return `Position ${position} is of commits this database does not have`;
|
|
291
|
+
if (epoch !== numbered.epoch)
|
|
292
|
+
return `Position ${position} is of an earlier numbering: the commits of this database were numbered again since`;
|
|
293
|
+
if (number > numbered.settled)
|
|
294
|
+
return `Position ${position} is past the newest commit of this database, ${numbered.settled}`;
|
|
295
|
+
if (number < numbered.minValid)
|
|
296
|
+
switch (numbered.expiredBy) {
|
|
297
|
+
case 'type':
|
|
298
|
+
return `Position ${position} is before commit ${numbered.minValid}, which changed the type of a column to one that does not hold each earlier value: the history keeps no row that only a reading before that commit reads`;
|
|
299
|
+
case 'clear':
|
|
300
|
+
return `Position ${position} is before commit ${numbered.minValid}, a clear that forgot the rows of its table: the history keeps no row that only a reading before it reads`;
|
|
301
|
+
default:
|
|
302
|
+
return `Position ${position} is before commit ${numbered.minValid}, the oldest position that the retention of the database keeps readable: the history keeps no row that only a reading before it reads`;
|
|
303
|
+
}
|
|
304
|
+
return undefined;
|
|
305
|
+
}
|
|
306
|
+
export function parsePosition(position) {
|
|
307
|
+
const groups = /^(?<epoch>[^:]+):(?<number>0|[1-9]\d*)$/.exec(position)?.groups;
|
|
308
|
+
if (groups?.epoch === undefined || groups.number === undefined)
|
|
309
|
+
throw new TypeError(`${position} is not a position of a Postgres database`);
|
|
310
|
+
return { epoch: groups.epoch, number: Number(groups.number) };
|
|
311
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { PostgresAppendWriter } from './postgres-append-writer.js';
|
|
2
|
+
// Replaces the target at its commit with DELETE, not TRUNCATE: TRUNCATE's
|
|
3
|
+
// exclusive lock would block readers for the rest of the run.
|
|
4
|
+
export class PostgresOverwriteWriter extends PostgresAppendWriter {
|
|
5
|
+
get replaces() {
|
|
6
|
+
return true;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { quote } from './identifier.js';
|
|
2
|
+
// Each privilege on the relation and on its columns, the owner's own
|
|
3
|
+
// included, as aclexplode gives them: a relation with no ACL holds its
|
|
4
|
+
// owner's defaults.
|
|
5
|
+
const grantsOf = `
|
|
6
|
+
SELECT NULL::text AS "column",
|
|
7
|
+
CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END AS "grantee",
|
|
8
|
+
g.privilege_type AS "privilege", g.is_grantable AS "grantable"
|
|
9
|
+
FROM pg_class c, aclexplode(coalesce(c.relacl, acldefault('r', c.relowner))) g
|
|
10
|
+
WHERE c.oid = to_regclass($1)
|
|
11
|
+
UNION ALL
|
|
12
|
+
SELECT a.attname::text,
|
|
13
|
+
CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END,
|
|
14
|
+
g.privilege_type, g.is_grantable
|
|
15
|
+
FROM pg_attribute a, aclexplode(a.attacl) g
|
|
16
|
+
WHERE a.attrelid = to_regclass($1) AND a.attnum > 0 AND NOT a.attisdropped`;
|
|
17
|
+
const key = ({ column, grantee, privilege, grantable }) => JSON.stringify([column, grantee, privilege, grantable]);
|
|
18
|
+
// The privileges of a relation, kept so that the relation that takes its
|
|
19
|
+
// name, a view made again or a table swapped in, holds exactly them, as
|
|
20
|
+
// Snowflake's COPY GRANTS keeps them across CREATE OR REPLACE. Applying them
|
|
21
|
+
// is a difference, as dbt applies its grants: it revokes each privilege the
|
|
22
|
+
// new relation has that the old one did not, such as one that default
|
|
23
|
+
// privileges gave it, and grants each that it lacks.
|
|
24
|
+
export class Privileges {
|
|
25
|
+
#grants;
|
|
26
|
+
constructor(grants) {
|
|
27
|
+
this.#grants = grants;
|
|
28
|
+
}
|
|
29
|
+
static async of(sql, relation) {
|
|
30
|
+
return new Privileges(await sql.unsafe(grantsOf, [relation]));
|
|
31
|
+
}
|
|
32
|
+
async apply(sql, relation) {
|
|
33
|
+
const keptKeys = new Set(this.#grants.map(key));
|
|
34
|
+
const held = await sql.unsafe(grantsOf, [relation]);
|
|
35
|
+
const heldKeys = new Set(held.map(key));
|
|
36
|
+
for (const grant of held)
|
|
37
|
+
if (!keptKeys.has(key(grant)))
|
|
38
|
+
await sql.unsafe(statement('REVOKE', grant, relation));
|
|
39
|
+
for (const grant of this.#grants)
|
|
40
|
+
if (!heldKeys.has(key(grant)))
|
|
41
|
+
await sql.unsafe(statement('GRANT', grant, relation));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function statement(verb, { column, grantee, privilege, grantable }, relation) {
|
|
45
|
+
const on = column === null ? '' : ` (${quote(column)})`;
|
|
46
|
+
const role = grantee === 'PUBLIC' ? 'PUBLIC' : quote(grantee);
|
|
47
|
+
return verb === 'GRANT'
|
|
48
|
+
? `GRANT ${privilege}${on} ON ${relation} TO ${role}${grantable ? ' WITH GRANT OPTION' : ''}`
|
|
49
|
+
: `REVOKE ${privilege}${on} ON ${relation} FROM ${role} CASCADE`;
|
|
50
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { TargetComments } from './postgres-comments.ts';
|
|
2
|
+
import type { Transaction } from './postgres-load.ts';
|
|
3
|
+
import type { PostgresReaderView } from './postgres-table.ts';
|
|
4
|
+
export declare class TargetReaderView {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(view: PostgresReaderView, target: string, history: string, columns: readonly string[], comments: TargetComments);
|
|
7
|
+
get qualifiedName(): string;
|
|
8
|
+
refuse(sql: Transaction, stale: boolean): Promise<void>;
|
|
9
|
+
describe(sql: Transaction): Promise<void>;
|
|
10
|
+
drop(sql: Transaction): Promise<void>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { quote } from './identifier.js';
|
|
2
|
+
// The position a reader reads at: the one a reading set for its
|
|
3
|
+
// transaction, or, for a session that set none, past every commit, so the
|
|
4
|
+
// view shows the target as it is now and no row of its history.
|
|
5
|
+
const position = `coalesce(nullif(current_setting('elt.position', true), '')::integer, 2147483647)`;
|
|
6
|
+
// The view readers see a target through: exactly the stream's columns and
|
|
7
|
+
// loaded_at, none of the internal ones, created with the target and
|
|
8
|
+
// described by the comments of those columns. It reads the target as of the
|
|
9
|
+
// position its reader set: the rows that the target had then, and that it
|
|
10
|
+
// still has or its history kept.
|
|
11
|
+
export class TargetReaderView {
|
|
12
|
+
#view;
|
|
13
|
+
// The target's and its history's qualified names, and the columns the
|
|
14
|
+
// view lists, with their comments.
|
|
15
|
+
#target;
|
|
16
|
+
#history;
|
|
17
|
+
#columns;
|
|
18
|
+
#comments;
|
|
19
|
+
constructor(view, target, history, columns, comments) {
|
|
20
|
+
this.#view = view;
|
|
21
|
+
this.#target = target;
|
|
22
|
+
this.#history = history;
|
|
23
|
+
this.#columns = columns;
|
|
24
|
+
this.#comments = comments;
|
|
25
|
+
Object.freeze(this);
|
|
26
|
+
}
|
|
27
|
+
get qualifiedName() {
|
|
28
|
+
return `${quote(this.#view.schema)}.${quote(this.#view.name)}`;
|
|
29
|
+
}
|
|
30
|
+
// Refuses, before anything is read, a reader view this load did not make:
|
|
31
|
+
// not a view, or a view of other columns or another table. A stale
|
|
32
|
+
// target's view still shows the stored columns, which evolving it replaces.
|
|
33
|
+
async refuse(sql, stale) {
|
|
34
|
+
const view = this.qualifiedName;
|
|
35
|
+
const [existing] = await sql.unsafe(`SELECT c.relkind,
|
|
36
|
+
ARRAY(SELECT attname::text FROM pg_attribute
|
|
37
|
+
WHERE attrelid = c.oid AND attnum > 0 AND NOT attisdropped ORDER BY attnum) AS columns,
|
|
38
|
+
EXISTS (SELECT 1 FROM pg_rewrite r JOIN pg_depend d
|
|
39
|
+
ON d.classid = 'pg_rewrite'::regclass AND d.objid = r.oid
|
|
40
|
+
WHERE r.ev_class = c.oid AND d.refobjid = to_regclass($2)) AS reads
|
|
41
|
+
FROM pg_class c WHERE c.oid = to_regclass($1)`, [view, this.#target]);
|
|
42
|
+
if (existing === undefined)
|
|
43
|
+
return;
|
|
44
|
+
if (existing.relkind !== 'v' ||
|
|
45
|
+
!existing.reads ||
|
|
46
|
+
(!stale && existing.columns.join('\0') !== this.#columns.join('\0')))
|
|
47
|
+
throw new TypeError(`${view} is not a view of exactly ${this.#target}; drop it or reset the warehouse`);
|
|
48
|
+
}
|
|
49
|
+
// Created only when absent and never replaced by a commit: replacing a view
|
|
50
|
+
// readers can see would lock them out. prepare refused any view not this
|
|
51
|
+
// load's own.
|
|
52
|
+
async describe(sql) {
|
|
53
|
+
const view = this.qualifiedName;
|
|
54
|
+
const [existing] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [view]);
|
|
55
|
+
if (existing?.exists !== true) {
|
|
56
|
+
const columns = this.#columns.map(quote);
|
|
57
|
+
await sql.unsafe(`CREATE VIEW ${view} AS SELECT ${columns.join(', ')} FROM ${this.#target} WHERE "committed_in" <= ${position} UNION ALL SELECT ${columns.join(', ')} FROM ${this.#history} AS "_elt_old" WHERE "_elt_old"."committed_in" <= ${position} AND "_elt_old"."replaced_in" > ${position}`);
|
|
58
|
+
}
|
|
59
|
+
await this.#comments.write(sql, 'VIEW', this.#view.schema, this.#view.name);
|
|
60
|
+
}
|
|
61
|
+
async drop(sql) {
|
|
62
|
+
await sql.unsafe(`DROP VIEW IF EXISTS ${this.qualifiedName}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import postgres from 'postgres';
|
|
2
|
+
export type Queries = Pick<postgres.Sql, 'unsafe'>;
|
|
3
|
+
export declare class PostgresSession implements AsyncDisposable {
|
|
4
|
+
readonly sql: postgres.Sql;
|
|
5
|
+
constructor(url: string, applicationName: string);
|
|
6
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
7
|
+
}
|
|
8
|
+
export declare function connectionUrl(url: string): string;
|
|
9
|
+
export declare function schemaName(schema: string): string;
|
|
10
|
+
export declare function lockSchema(sql: postgres.ISql, schema: string): Promise<void>;
|
|
11
|
+
export declare function makeSchema(sql: postgres.Sql, schema: string): Promise<void>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import postgres from 'postgres';
|
|
2
|
+
import { identifier, quote } from './identifier.js';
|
|
3
|
+
// One session, closed however the work using it ends.
|
|
4
|
+
export class PostgresSession {
|
|
5
|
+
sql;
|
|
6
|
+
constructor(url, applicationName) {
|
|
7
|
+
this.sql = postgres(url, {
|
|
8
|
+
max: 1,
|
|
9
|
+
// postgres.js replaces a connection after 30 to 60 minutes by default,
|
|
10
|
+
// which silently drops a session's TEMP stages and session locks.
|
|
11
|
+
max_lifetime: null,
|
|
12
|
+
onnotice: () => { },
|
|
13
|
+
connection: { application_name: applicationName },
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
async [Symbol.asyncDispose]() {
|
|
17
|
+
await this.sql.end({ timeout: 5 });
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export function connectionUrl(url) {
|
|
21
|
+
const parsed = URL.parse(url);
|
|
22
|
+
if (parsed === null ||
|
|
23
|
+
(parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:'))
|
|
24
|
+
throw new TypeError('Postgres requires a postgres:// connection URL');
|
|
25
|
+
return url;
|
|
26
|
+
}
|
|
27
|
+
export function schemaName(schema) {
|
|
28
|
+
identifier(schema, 'schema name');
|
|
29
|
+
if (/^pg_/i.test(schema))
|
|
30
|
+
throw new TypeError('Schema names starting with pg_ are reserved');
|
|
31
|
+
return schema;
|
|
32
|
+
}
|
|
33
|
+
// The advisory lock of a schema's DDL, as hashtextextended keys it.
|
|
34
|
+
const schemaKey = (schema) => `elt:${schema}`;
|
|
35
|
+
// Serializes, until the transaction ends, the DDL that makes a schema and the
|
|
36
|
+
// objects its loads, views and checkpoints share: Postgres refuses a second
|
|
37
|
+
// session's CREATE SCHEMA or CREATE TABLE of the same name with 23505.
|
|
38
|
+
export async function lockSchema(sql, schema) {
|
|
39
|
+
await sql `SELECT pg_advisory_xact_lock(hashtextextended(${schemaKey(schema)}, 0))`;
|
|
40
|
+
}
|
|
41
|
+
// Makes the schema in a transaction of its own, which begins only once the
|
|
42
|
+
// schema's lock is held. A session refreshes its catalog cache when a
|
|
43
|
+
// transaction begins and when it locks a relation, not when it takes an
|
|
44
|
+
// advisory lock, so a transaction that waited for the lock could still miss
|
|
45
|
+
// the schema that the holder made, and its CREATE SCHEMA IF NOT EXISTS would
|
|
46
|
+
// fail with 23505. A schema that this session already sees needs no lock; one
|
|
47
|
+
// that it misses, also by a stale cache, is made under the lock. sql is a
|
|
48
|
+
// session outside any transaction.
|
|
49
|
+
export async function makeSchema(sql, schema) {
|
|
50
|
+
const [made] = await sql `SELECT to_regnamespace(${quote(schema)}) IS NOT NULL AS "exists"`;
|
|
51
|
+
if (made?.exists === true)
|
|
52
|
+
return;
|
|
53
|
+
const key = schemaKey(schema);
|
|
54
|
+
await sql `SELECT pg_advisory_lock(hashtextextended(${key}, 0))`;
|
|
55
|
+
try {
|
|
56
|
+
await sql.begin((transaction) => transaction.unsafe(`CREATE SCHEMA IF NOT EXISTS ${quote(schema)}`));
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
await sql `SELECT pg_advisory_unlock(hashtextextended(${key}, 0))`;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { PostgresView } from './postgres-views.ts';
|
|
2
|
+
export declare const syncHistoryViews: readonly PostgresView[];
|
|
3
|
+
export declare const syncHistoryTables: readonly ['CREATE SCHEMA IF NOT EXISTS _warehouse', `CREATE TABLE IF NOT EXISTS _warehouse.sync_attempts (
|
|
4
|
+
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
5
|
+
connector text NOT NULL CHECK (btrim(connector) <> ''), source text NOT NULL,
|
|
6
|
+
started_at timestamptz NOT NULL,
|
|
7
|
+
completed_at timestamptz,
|
|
8
|
+
status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
|
|
9
|
+
error text,
|
|
10
|
+
failure_type text CHECK (failure_type IN ('config', 'system')),
|
|
11
|
+
CHECK ((status = 'running') = (completed_at IS NULL)),
|
|
12
|
+
CHECK ((status IN ('partial', 'failed')) = (failure_type IS NOT NULL))
|
|
13
|
+
)`, `CREATE OR REPLACE FUNCTION _warehouse._elt_unversioned() RETURNS boolean LANGUAGE plpgsql STABLE AS $$
|
|
14
|
+
BEGIN
|
|
15
|
+
IF coalesce(current_setting('elt.position', true), '') <> '' THEN
|
|
16
|
+
RAISE EXCEPTION 'The sync history cannot be read as of a position: it keeps no row versions';
|
|
17
|
+
END IF;
|
|
18
|
+
RETURN true;
|
|
19
|
+
END $$`, 'CREATE INDEX IF NOT EXISTS sync_attempts_connector ON _warehouse.sync_attempts (connector, id DESC)', `CREATE TABLE IF NOT EXISTS _warehouse.extraction_coverage (
|
|
20
|
+
attempt_id bigint NOT NULL REFERENCES _warehouse.sync_attempts(id),
|
|
21
|
+
stream text NOT NULL, target_schema text NOT NULL, target_table text NOT NULL,
|
|
22
|
+
sync_mode text NOT NULL, destination_sync_mode text NOT NULL,
|
|
23
|
+
description text NOT NULL CHECK (btrim(description) <> ''), selection jsonb NOT NULL,
|
|
24
|
+
status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
|
|
25
|
+
written_count bigint CHECK (written_count >= 0), deleted_count bigint CHECK (deleted_count >= 0),
|
|
26
|
+
failures jsonb NOT NULL DEFAULT '[]', upstream_current_as_of timestamptz,
|
|
27
|
+
PRIMARY KEY (attempt_id, stream)
|
|
28
|
+
)`, `CREATE TABLE IF NOT EXISTS _warehouse.stream_checks (
|
|
29
|
+
connector text NOT NULL, stream text NOT NULL, checked_at timestamptz NOT NULL,
|
|
30
|
+
PRIMARY KEY (connector, stream)
|
|
31
|
+
)`];
|