@eltjs/elt-postgresql 0.0.0-stage → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +813 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { errorMessage } from '@eltjs/elt';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
import { Privileges } from './postgres-privileges.js';
|
|
4
|
+
// Each view that reads one of the roots, directly or through other views,
|
|
5
|
+
// deepest last. A view's own rule depends on the view, which is not a reader
|
|
6
|
+
// of it.
|
|
7
|
+
const dependents = `
|
|
8
|
+
WITH RECURSIVE roots AS (
|
|
9
|
+
SELECT to_regclass(name) AS oid FROM unnest($1::text[]) AS name
|
|
10
|
+
WHERE to_regclass(name) IS NOT NULL
|
|
11
|
+
), readers AS (
|
|
12
|
+
SELECT r.ev_class AS oid, 1 AS depth
|
|
13
|
+
FROM pg_depend d JOIN pg_rewrite r
|
|
14
|
+
ON d.classid = 'pg_rewrite'::regclass AND d.objid = r.oid
|
|
15
|
+
WHERE d.refclassid = 'pg_class'::regclass
|
|
16
|
+
AND d.refobjid IN (SELECT oid FROM roots) AND r.ev_class <> d.refobjid
|
|
17
|
+
UNION ALL
|
|
18
|
+
SELECT r.ev_class, readers.depth + 1
|
|
19
|
+
FROM readers JOIN pg_depend d
|
|
20
|
+
ON d.refclassid = 'pg_class'::regclass AND d.refobjid = readers.oid
|
|
21
|
+
JOIN pg_rewrite r ON d.classid = 'pg_rewrite'::regclass AND d.objid = r.oid
|
|
22
|
+
WHERE r.ev_class <> readers.oid
|
|
23
|
+
)
|
|
24
|
+
SELECT format('%I.%I', n.nspname, c.relname) AS "relation",
|
|
25
|
+
CASE c.relkind WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE 'VIEW' END AS "kind",
|
|
26
|
+
pg_get_viewdef(c.oid) AS "definition",
|
|
27
|
+
array_to_string(c.reloptions, ', ') AS "options",
|
|
28
|
+
c.relispopulated AS "populated",
|
|
29
|
+
pg_get_userbyid(c.relowner) AS "owner",
|
|
30
|
+
obj_description(c.oid, 'pg_class') AS "description",
|
|
31
|
+
coalesce((SELECT json_agg(json_build_object('name', a.attname,
|
|
32
|
+
'description', col_description(c.oid, a.attnum)) ORDER BY a.attnum)
|
|
33
|
+
FROM pg_attribute a
|
|
34
|
+
WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped
|
|
35
|
+
AND col_description(c.oid, a.attnum) IS NOT NULL), '[]') AS "columns",
|
|
36
|
+
coalesce((SELECT json_agg(pg_get_indexdef(i.indexrelid) ORDER BY i.indexrelid)
|
|
37
|
+
FROM pg_index i WHERE i.indrelid = c.oid), '[]') AS "indexes"
|
|
38
|
+
FROM (SELECT oid, max(depth) AS depth FROM readers
|
|
39
|
+
WHERE oid NOT IN (SELECT oid FROM roots) GROUP BY oid) v
|
|
40
|
+
JOIN pg_class c ON c.oid = v.oid
|
|
41
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
42
|
+
ORDER BY v.depth, c.oid`;
|
|
43
|
+
// Runs change, which Postgres would refuse under the views that read roots:
|
|
44
|
+
// it binds a view to the relations it read when it was made, so DROP TABLE,
|
|
45
|
+
// ALTER COLUMN TYPE, DROP COLUMN and DROP VIEW fail while one reads them.
|
|
46
|
+
// The views are saved, dropped, and made again over what the change left,
|
|
47
|
+
// by name, in the same transaction, so a reader never finds one missing:
|
|
48
|
+
// each takes the relation that now has the name it reads, as a view of
|
|
49
|
+
// Snowflake or BigQuery, or a Redshift view WITH NO SCHEMA BINDING, resolves
|
|
50
|
+
// its tables at each query (pg-deps-management's save and restore). A view
|
|
51
|
+
// that does not fit what the change left, such as one that reads a column
|
|
52
|
+
// the change dropped, fails the change and names the view. The roots are the
|
|
53
|
+
// change's own, which it can make again, such as a table it swaps in; each
|
|
54
|
+
// root that existed keeps its privileges in the relation that has its name
|
|
55
|
+
// after the change.
|
|
56
|
+
export async function rebindDependents(sql, roots, change) {
|
|
57
|
+
// Every name qualified, so each view reads the same names again.
|
|
58
|
+
const [{ path } = { path: '' }] = await sql.unsafe("SELECT current_setting('search_path') AS path");
|
|
59
|
+
await sql.unsafe("SELECT set_config('search_path', '', true)");
|
|
60
|
+
const saved = await sql.unsafe(dependents, [roots]);
|
|
61
|
+
const existing = await sql.unsafe('SELECT relation FROM unnest($1::text[]) AS relation WHERE to_regclass(relation) IS NOT NULL', [roots]);
|
|
62
|
+
const privileges = new Map();
|
|
63
|
+
for (const { relation } of [...existing, ...saved])
|
|
64
|
+
privileges.set(relation, await Privileges.of(sql, relation));
|
|
65
|
+
await sql.unsafe("SELECT set_config('search_path', $1, true)", [path]);
|
|
66
|
+
for (const view of saved.toReversed())
|
|
67
|
+
try {
|
|
68
|
+
await sql.unsafe(`DROP ${view.kind} ${view.relation}`);
|
|
69
|
+
}
|
|
70
|
+
catch (cause) {
|
|
71
|
+
throw new TypeError(`${view.relation} reads ${roots[0]}, and an object that elt cannot make again reads ${view.relation}: ${errorMessage(cause)}`, { cause });
|
|
72
|
+
}
|
|
73
|
+
const result = await change();
|
|
74
|
+
await sql.unsafe("SELECT set_config('search_path', '', true)");
|
|
75
|
+
for (const view of saved)
|
|
76
|
+
try {
|
|
77
|
+
await make(sql, view);
|
|
78
|
+
}
|
|
79
|
+
catch (cause) {
|
|
80
|
+
throw new TypeError(`${view.relation} reads ${roots[0]} and does not fit its change: ${errorMessage(cause)}. Change the view, or drop it.`, { cause });
|
|
81
|
+
}
|
|
82
|
+
for (const [relation, kept] of privileges)
|
|
83
|
+
await kept.apply(sql, relation);
|
|
84
|
+
await sql.unsafe("SELECT set_config('search_path', $1, true)", [path]);
|
|
85
|
+
return result;
|
|
86
|
+
}
|
|
87
|
+
async function make(sql, view) {
|
|
88
|
+
const options = view.options === null ? '' : ` WITH (${view.options})`;
|
|
89
|
+
let data = '';
|
|
90
|
+
if (view.kind === 'MATERIALIZED VIEW')
|
|
91
|
+
data = view.populated ? ' WITH DATA' : ' WITH NO DATA';
|
|
92
|
+
await sql.unsafe(`CREATE ${view.kind} ${view.relation}${options} AS ${view.definition.replace(/;\s*$/, '')}${data}`);
|
|
93
|
+
for (const index of view.indexes)
|
|
94
|
+
await sql.unsafe(index);
|
|
95
|
+
const [{ me } = { me: '' }] = await sql.unsafe('SELECT current_user AS me');
|
|
96
|
+
if (view.owner !== me)
|
|
97
|
+
await sql.unsafe(`ALTER ${view.kind} ${view.relation} OWNER TO ${quote(view.owner)}`);
|
|
98
|
+
// COMMENT cannot bind values; Postgres quotes the text itself.
|
|
99
|
+
const comments = await sql.unsafe(`SELECT format('COMMENT ON ${view.kind} %s IS %L', $1::text, $2::text) AS statement WHERE $2::text IS NOT NULL
|
|
100
|
+
UNION ALL
|
|
101
|
+
SELECT format('COMMENT ON COLUMN %s.%I IS %L', $1::text, c->>'name', c->>'description')
|
|
102
|
+
FROM json_array_elements($3::text::json) AS c`, [view.relation, view.description, JSON.stringify(view.columns)]);
|
|
103
|
+
for (const { statement } of comments)
|
|
104
|
+
await sql.unsafe(statement);
|
|
105
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { type CopyConfiguration, Destination, type Load, type PipelineClock } from '@eltjs/elt';
|
|
2
|
+
import { type PostgresReading } from './postgres-as-of.ts';
|
|
3
|
+
import type { PostgresColumn } from './postgres-column.ts';
|
|
4
|
+
import { PostgresColumns } from './postgres-columns.ts';
|
|
5
|
+
import { PostgresTable } from './postgres-table.ts';
|
|
6
|
+
import type { PostgresWriter } from './postgres-writer.ts';
|
|
7
|
+
export declare class PostgresDestination extends Destination<PostgresTable> {
|
|
8
|
+
#private;
|
|
9
|
+
readonly supportedDestinationSyncModes: readonly ["overwrite", "append", "append_dedup", "overwrite_dedup"];
|
|
10
|
+
readonly schema: string;
|
|
11
|
+
readonly retention: Temporal.Duration;
|
|
12
|
+
constructor({ url, schema, retention, }: {
|
|
13
|
+
url: string;
|
|
14
|
+
schema: string;
|
|
15
|
+
retention?: Temporal.Duration;
|
|
16
|
+
});
|
|
17
|
+
get url(): string;
|
|
18
|
+
location(target: PostgresTable): string;
|
|
19
|
+
load(owner: {
|
|
20
|
+
clock: PipelineClock;
|
|
21
|
+
connection: string;
|
|
22
|
+
}): Promise<Load<PostgresTable>>;
|
|
23
|
+
positions(after?: string): Promise<readonly string[] | {
|
|
24
|
+
readonly expired: string;
|
|
25
|
+
}>;
|
|
26
|
+
reading(position?: string): Promise<PostgresReading | {
|
|
27
|
+
readonly expired: string;
|
|
28
|
+
}>;
|
|
29
|
+
table(name: string, configure?: (columns: PostgresColumns) => readonly PostgresColumn[]): PostgresTable;
|
|
30
|
+
createWriter(configuration: CopyConfiguration, target: PostgresTable): PostgresWriter;
|
|
31
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { Destination, } from '@eltjs/elt';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
import { PostgresAppendWriter } from './postgres-append-writer.js';
|
|
4
|
+
import { readAsOf } from './postgres-as-of.js';
|
|
5
|
+
import { PostgresColumns } from './postgres-columns.js';
|
|
6
|
+
import { PostgresDeduplicatingWriter } from './postgres-deduplicating-writer.js';
|
|
7
|
+
import { PostgresLoad } from './postgres-load.js';
|
|
8
|
+
import { positionsAfter } from './postgres-numbering.js';
|
|
9
|
+
import { PostgresOverwriteWriter } from './postgres-overwrite-writer.js';
|
|
10
|
+
import { PostgresSession, connectionUrl, schemaName, } from './postgres-session.js';
|
|
11
|
+
import { PostgresTable } from './postgres-table.js';
|
|
12
|
+
// Loads into one schema of one database. The URL carries credentials, so it
|
|
13
|
+
// stays private and out of location(), which is persisted with checkpoints.
|
|
14
|
+
export class PostgresDestination extends Destination {
|
|
15
|
+
supportedDestinationSyncModes = Object.freeze([
|
|
16
|
+
'overwrite',
|
|
17
|
+
'append',
|
|
18
|
+
'append_dedup',
|
|
19
|
+
'overwrite_dedup',
|
|
20
|
+
]);
|
|
21
|
+
schema;
|
|
22
|
+
// How long the history keeps a position readable: 30 days unless set, as
|
|
23
|
+
// Delta Lake keeps its log. The destinations of one database share one
|
|
24
|
+
// numbering, so the shortest retention among them decides.
|
|
25
|
+
retention;
|
|
26
|
+
#url;
|
|
27
|
+
constructor({ url, schema, retention = Temporal.Duration.from({ days: 30 }), }) {
|
|
28
|
+
super();
|
|
29
|
+
if (retention.sign < 0)
|
|
30
|
+
throw new RangeError(`Retention ${retention} must not be negative`);
|
|
31
|
+
this.#url = connectionUrl(url);
|
|
32
|
+
this.schema = schemaName(schema);
|
|
33
|
+
this.retention = retention;
|
|
34
|
+
Object.freeze(this);
|
|
35
|
+
}
|
|
36
|
+
// The connection URL, for the views and computed datasets of the
|
|
37
|
+
// destination. A getter, so neither JSON nor util.inspect shows it.
|
|
38
|
+
get url() {
|
|
39
|
+
return this.#url;
|
|
40
|
+
}
|
|
41
|
+
location(target) {
|
|
42
|
+
return `${quote(this.schema)}.${target.quotedName}`;
|
|
43
|
+
}
|
|
44
|
+
// One connection for the whole run; each commit is its own transaction,
|
|
45
|
+
// numbered in the numbering that the destinations of the database share.
|
|
46
|
+
async load(owner) {
|
|
47
|
+
const load = await PostgresLoad.open(this.#url, { connector: owner.connection }, { clock: owner.clock, retention: this.retention });
|
|
48
|
+
return {
|
|
49
|
+
prepare: (configuration, target, binding) => this.createWriter(configuration, target).prepare(load, binding),
|
|
50
|
+
[Symbol.asyncDispose]: () => load[Symbol.asyncDispose](),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
// The positions of the database after a position, ascending, or all of
|
|
54
|
+
// them; each one, `${epoch}:${number}`, names a commit where no pass of any
|
|
55
|
+
// schema of the database is half read. A position of another epoch, or past
|
|
56
|
+
// the newest commit, is expired.
|
|
57
|
+
async positions(after) {
|
|
58
|
+
await using session = new PostgresSession(this.#url, 'elt-positions');
|
|
59
|
+
return await positionsAfter(session.sql, after);
|
|
60
|
+
}
|
|
61
|
+
// The database as of a position, or as of its newest one, read through the
|
|
62
|
+
// reader views: { expired } for a position of another epoch or past the
|
|
63
|
+
// newest commit, and a TypeError for one inside a pass or not a position.
|
|
64
|
+
reading(position) {
|
|
65
|
+
return readAsOf(this.#url, position);
|
|
66
|
+
}
|
|
67
|
+
table(name, configure) {
|
|
68
|
+
return new PostgresTable(name, configure?.(new PostgresColumns()));
|
|
69
|
+
}
|
|
70
|
+
createWriter(configuration, target) {
|
|
71
|
+
this.validateConfiguration(configuration, target);
|
|
72
|
+
if (!(target instanceof PostgresTable))
|
|
73
|
+
throw new TypeError('Postgres requires Postgres table targets');
|
|
74
|
+
const table = target.resolve(configuration.stream);
|
|
75
|
+
switch (configuration.destinationSyncMode) {
|
|
76
|
+
case 'append_dedup':
|
|
77
|
+
case 'overwrite_dedup':
|
|
78
|
+
return new PostgresDeduplicatingWriter(configuration, this.#url, this.schema, table, this.retention);
|
|
79
|
+
case 'append':
|
|
80
|
+
return new PostgresAppendWriter(configuration, this.#url, this.schema, table, this.retention);
|
|
81
|
+
case 'overwrite':
|
|
82
|
+
return new PostgresOverwriteWriter(configuration, this.#url, this.schema, table, this.retention);
|
|
83
|
+
default:
|
|
84
|
+
throw new TypeError(`Destination does not support ${configuration.destinationSyncMode}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type postgres from 'postgres';
|
|
2
|
+
import type { FileContent } from '@eltjs/elt';
|
|
3
|
+
import type { PostgresColumn } from './postgres-column.ts';
|
|
4
|
+
import type { PostgresTable } from './postgres-table.ts';
|
|
5
|
+
export declare class PostgresFileStore {
|
|
6
|
+
#private;
|
|
7
|
+
static readonly chunkSize: number;
|
|
8
|
+
readonly schema: string;
|
|
9
|
+
readonly table: PostgresTable;
|
|
10
|
+
readonly column: PostgresColumn;
|
|
11
|
+
readonly qualifiedName: string;
|
|
12
|
+
constructor(schema: string, table: PostgresTable, column: PostgresColumn);
|
|
13
|
+
static qualifiedName(schema: string, table: PostgresTable, column: string): string;
|
|
14
|
+
stage(sql: postgres.Sql): Promise<void>;
|
|
15
|
+
save(sql: postgres.Sql, content: FileContent): Promise<{
|
|
16
|
+
readonly file: string;
|
|
17
|
+
readonly sha256: Uint8Array;
|
|
18
|
+
}>;
|
|
19
|
+
publish(sql: postgres.Sql): Promise<void>;
|
|
20
|
+
discard(sql: postgres.Sql): Promise<void>;
|
|
21
|
+
unstage(sql: postgres.Sql): Promise<void>;
|
|
22
|
+
prune(sql: Pick<postgres.Sql, 'unsafe'>, tables: readonly string[]): Promise<void>;
|
|
23
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { createHash, hash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
const chunks = '("file" UUID NOT NULL, "n" BIGINT NOT NULL, "bytes" BYTEA NOT NULL, PRIMARY KEY ("file", "n"))';
|
|
4
|
+
// Like SQLite, keep original files in bounded chunks rather than one whole-file
|
|
5
|
+
// value. A file column contains the UUID of its ordered BYTEA chunks.
|
|
6
|
+
export class PostgresFileStore {
|
|
7
|
+
static chunkSize = 4 * 1024 * 1024;
|
|
8
|
+
schema;
|
|
9
|
+
table;
|
|
10
|
+
column;
|
|
11
|
+
qualifiedName;
|
|
12
|
+
// The TEMP table of chunks saved since the last commit, private to the
|
|
13
|
+
// load's session like the stage of their records.
|
|
14
|
+
#staged;
|
|
15
|
+
constructor(schema, table, column) {
|
|
16
|
+
this.schema = schema;
|
|
17
|
+
this.table = table;
|
|
18
|
+
this.column = column;
|
|
19
|
+
this.qualifiedName = PostgresFileStore.qualifiedName(schema, table, column.name);
|
|
20
|
+
this.#staged = quote(`_elt_files_stage_${PostgresFileStore.#key(table, column.name)}`);
|
|
21
|
+
}
|
|
22
|
+
// The chunk table of the column named column, which may no longer be one
|
|
23
|
+
// of the table's file columns.
|
|
24
|
+
static qualifiedName(schema, table, column) {
|
|
25
|
+
return `${quote(schema)}.${quote(`_elt_files_${PostgresFileStore.#key(table, column)}`)}`;
|
|
26
|
+
}
|
|
27
|
+
static #key(table, column) {
|
|
28
|
+
return hash('sha256', JSON.stringify([table.name, column])).slice(0, 40);
|
|
29
|
+
}
|
|
30
|
+
async stage(sql) {
|
|
31
|
+
await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${this.#staged}`);
|
|
32
|
+
await sql.unsafe(`CREATE TEMP TABLE ${this.#staged} ${chunks}`);
|
|
33
|
+
}
|
|
34
|
+
// Stages the file's bytes under a new id, and gives the id and the SHA-256
|
|
35
|
+
// of the bytes, which the row hash counts in place of the id.
|
|
36
|
+
async save(sql, content) {
|
|
37
|
+
const file = randomUUID();
|
|
38
|
+
const sha256 = createHash('sha256');
|
|
39
|
+
let n = 0;
|
|
40
|
+
const insert = (bytes) => sql.unsafe(`INSERT INTO pg_temp.${this.#staged} ("file", "n", "bytes") VALUES ($1, $2, $3)`, [file, n++, Buffer.from(bytes)]);
|
|
41
|
+
for await (const chunk of content.chunks(PostgresFileStore.chunkSize)) {
|
|
42
|
+
sha256.update(chunk);
|
|
43
|
+
await insert(chunk);
|
|
44
|
+
}
|
|
45
|
+
if (n === 0)
|
|
46
|
+
await insert(new Uint8Array());
|
|
47
|
+
return { file, sha256: sha256.digest() };
|
|
48
|
+
}
|
|
49
|
+
// Moves the staged chunks into the store, inside a commit.
|
|
50
|
+
async publish(sql) {
|
|
51
|
+
await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.qualifiedName} ${chunks}`);
|
|
52
|
+
await sql.unsafe(`INSERT INTO ${this.qualifiedName} SELECT * FROM pg_temp.${this.#staged}`);
|
|
53
|
+
await sql.unsafe(`TRUNCATE pg_temp.${this.#staged}`);
|
|
54
|
+
}
|
|
55
|
+
async discard(sql) {
|
|
56
|
+
await sql.unsafe(`TRUNCATE pg_temp.${this.#staged}`);
|
|
57
|
+
}
|
|
58
|
+
async unstage(sql) {
|
|
59
|
+
await sql.unsafe(`DROP TABLE pg_temp.${this.#staged}`);
|
|
60
|
+
}
|
|
61
|
+
// Called inside each commit, after the merge: chunks no row of the tables
|
|
62
|
+
// refers to, such as a deduplication loser's, go. The tables are the target,
|
|
63
|
+
// its history, whose rows keep their files for a reading as of a position
|
|
64
|
+
// before the commit that replaced them, and, while a reload is open, its
|
|
65
|
+
// hidden target. A table can type the column otherwise, such as a target
|
|
66
|
+
// whose reload makes it a file column, so ids compare as text. This store
|
|
67
|
+
// belongs to just one target column and one writer.
|
|
68
|
+
// ponytail: scans this column's chunks; use targeted cleanup if checkpoint cost grows.
|
|
69
|
+
async prune(sql, tables) {
|
|
70
|
+
const referring = [];
|
|
71
|
+
for (const table of tables) {
|
|
72
|
+
const [column] = await sql.unsafe('SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass($1) AND attname = $2 AND NOT attisdropped', [table, this.column.name]);
|
|
73
|
+
if (column !== undefined)
|
|
74
|
+
referring.push(table);
|
|
75
|
+
}
|
|
76
|
+
await sql.unsafe(`DELETE FROM ${this.qualifiedName} AS chunks WHERE ${referring
|
|
77
|
+
.map((table) => `NOT EXISTS (SELECT 1 FROM ${table} AS kept WHERE kept.${this.column.quotedName}::text = chunks."file"::text)`)
|
|
78
|
+
.join(' AND ') || 'true'}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { StoredFit } from '@eltjs/elt';
|
|
2
|
+
import type { Transaction } from './postgres-load.ts';
|
|
3
|
+
import type { PostgresTable } from './postgres-table.ts';
|
|
4
|
+
export declare function storedColumns(sql: Pick<Transaction, 'unsafe'>, relation: string): import("postgres").PendingQuery<{
|
|
5
|
+
name: string;
|
|
6
|
+
type: string;
|
|
7
|
+
required: boolean;
|
|
8
|
+
}[]>;
|
|
9
|
+
export declare function storedFit(sql: Transaction, target: PostgresTable, table: string, stage: string): Promise<StoredFit>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// The columns a relation stores, with each one's type as Postgres spells it
|
|
2
|
+
// and whether it is NOT NULL.
|
|
3
|
+
export function storedColumns(sql, relation) {
|
|
4
|
+
return sql.unsafe('SELECT attname AS name, format_type(atttypid, atttypmod) AS type, attnotnull AS required FROM pg_attribute WHERE attrelid = to_regclass($1) AND attnum > 0 AND NOT attisdropped', [relation]);
|
|
5
|
+
}
|
|
6
|
+
// Whether a stored table has the columns, types and NOT NULL the stream
|
|
7
|
+
// needs. The stage is built from the same column declarations, so Postgres
|
|
8
|
+
// spells both alike.
|
|
9
|
+
export async function storedFit(sql, target, table, stage) {
|
|
10
|
+
const stored = await storedColumns(sql, table);
|
|
11
|
+
if (stored.length === 0)
|
|
12
|
+
return 'missing';
|
|
13
|
+
const types = new Map((await storedColumns(sql, `pg_temp.${stage}`)).map(({ name, type }) => [
|
|
14
|
+
name,
|
|
15
|
+
type,
|
|
16
|
+
]));
|
|
17
|
+
const needed = [
|
|
18
|
+
...target.columns.map((column) => `${column.name} ${types.get(column.name)} ${column.isPrimaryKey}`),
|
|
19
|
+
'loaded_at timestamp with time zone true',
|
|
20
|
+
'committed_in integer true',
|
|
21
|
+
'_elt_row_hash text false',
|
|
22
|
+
];
|
|
23
|
+
const has = stored.map(({ name, type, required }) => `${name} ${type} ${required}`);
|
|
24
|
+
return needed.sort().join('\0') === has.sort().join('\0') ? 'fits' : 'stale';
|
|
25
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { EncodedValue } from './postgres-column.ts';
|
|
2
|
+
import type { Queries } from './postgres-session.ts';
|
|
3
|
+
export declare const version = "\"_elt_row_hash\"";
|
|
4
|
+
export declare class TargetHistory {
|
|
5
|
+
#private;
|
|
6
|
+
readonly schema: string;
|
|
7
|
+
readonly name: string;
|
|
8
|
+
constructor(schema: string, key: string);
|
|
9
|
+
get qualifiedName(): string;
|
|
10
|
+
create(sql: Queries, target: string, change: () => Promise<number>): Promise<void>;
|
|
11
|
+
expire(sql: Queries): Promise<void>;
|
|
12
|
+
keep(sql: Queries, replacedIn: number, target: string, rows: string, params?: readonly string[]): Promise<void>;
|
|
13
|
+
remove(sql: Queries, replacedIn: number, target: string, where: string, params?: readonly string[]): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
export declare function rowHash(columns: readonly {
|
|
16
|
+
readonly name: string;
|
|
17
|
+
}[], values: readonly (EncodedValue | Uint8Array)[]): string;
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
import { storedColumns } from './postgres-fit.js';
|
|
4
|
+
import { expireBefore, minValid } from './postgres-numbering.js';
|
|
5
|
+
export const version = '"_elt_row_hash"';
|
|
6
|
+
// The rows of a target that commits replaced or removed, each with the
|
|
7
|
+
// number of the commit that did, as a SQL Server temporal table keeps them
|
|
8
|
+
// beside its current table. Each column has the type of the target's column
|
|
9
|
+
// of the same name, and changes its type when the target's does; a column
|
|
10
|
+
// that the target drops stays, with its last type.
|
|
11
|
+
export class TargetHistory {
|
|
12
|
+
schema;
|
|
13
|
+
name;
|
|
14
|
+
constructor(schema, key) {
|
|
15
|
+
this.schema = schema;
|
|
16
|
+
this.name = `_elt_history_${key}`;
|
|
17
|
+
}
|
|
18
|
+
get qualifiedName() {
|
|
19
|
+
return `${quote(this.schema)}.${quote(this.name)}`;
|
|
20
|
+
}
|
|
21
|
+
// Made with the target, with the target's columns, none NOT NULL: a
|
|
22
|
+
// column that the target drops is NULL in each row kept after. A history
|
|
23
|
+
// that exists follows the target; change gives the number of the commit
|
|
24
|
+
// that changes the target's shape.
|
|
25
|
+
async create(sql, target, change) {
|
|
26
|
+
const columns = await storedColumns(sql, target);
|
|
27
|
+
await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.qualifiedName} (${[...columns.map(({ name, type }) => `${quote(name)} ${type}`), '"replaced_in" INTEGER NOT NULL'].join(', ')})`);
|
|
28
|
+
await sql.unsafe(`CREATE INDEX IF NOT EXISTS ${quote(`${this.name}_replaced`)} ON ${this.qualifiedName} ("replaced_in")`);
|
|
29
|
+
await this.#follow(sql, target, change);
|
|
30
|
+
}
|
|
31
|
+
// Takes each column of the target that it does not have, and the type of
|
|
32
|
+
// each target column whose type it does not have. A widening casts the
|
|
33
|
+
// kept values, so each position still reads. Any other change of type
|
|
34
|
+
// acts as a rewrite in the commit of the change: each position before it
|
|
35
|
+
// expires, and the rows that only those positions read go before the cast,
|
|
36
|
+
// so no kept value blocks it. Only a change takes a lock, so a commit that
|
|
37
|
+
// changes no shape waits for no reader. Gives the target's columns, and
|
|
38
|
+
// whether the change expired the positions before it.
|
|
39
|
+
async #follow(sql, target, change) {
|
|
40
|
+
const columns = await storedColumns(sql, target);
|
|
41
|
+
const kept = new Map((await storedColumns(sql, this.qualifiedName)).map(({ name, type }) => [
|
|
42
|
+
name,
|
|
43
|
+
type,
|
|
44
|
+
]));
|
|
45
|
+
const rewritten = columns.some(({ name, type }) => {
|
|
46
|
+
const had = kept.get(name);
|
|
47
|
+
return had !== undefined && had !== type && !widens(had, type);
|
|
48
|
+
});
|
|
49
|
+
if (rewritten) {
|
|
50
|
+
const changedIn = await change();
|
|
51
|
+
await expireBefore(sql, changedIn, 'type');
|
|
52
|
+
await sql.unsafe(`DELETE FROM ${this.qualifiedName} WHERE "replaced_in" <= $1`, [changedIn]);
|
|
53
|
+
}
|
|
54
|
+
for (const { name, type } of columns) {
|
|
55
|
+
const column = quote(name);
|
|
56
|
+
const had = kept.get(name);
|
|
57
|
+
if (had === undefined)
|
|
58
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD COLUMN ${column} ${type}`);
|
|
59
|
+
else if (had !== type)
|
|
60
|
+
await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column} TYPE ${type} USING ${column}::${type}`);
|
|
61
|
+
}
|
|
62
|
+
return { columns: columns.map(({ name }) => name), rewritten };
|
|
63
|
+
}
|
|
64
|
+
// Deletes the rows that only a reading before min_valid reads, which the
|
|
65
|
+
// retention or a change of type left, once the history exists.
|
|
66
|
+
async expire(sql) {
|
|
67
|
+
const [history] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
|
|
68
|
+
if (history?.exists !== true)
|
|
69
|
+
return;
|
|
70
|
+
await sql.unsafe(`DELETE FROM ${this.qualifiedName} WHERE "replaced_in" <= $1`, [await minValid(sql)]);
|
|
71
|
+
}
|
|
72
|
+
// Copies the rows of the target that rows selects into the history, as
|
|
73
|
+
// replaced in commit replacedIn. rows is the FROM clause of a SELECT in
|
|
74
|
+
// which "_elt_old" names the target. A commit that expired the positions
|
|
75
|
+
// before it keeps no row, as only those positions would read it.
|
|
76
|
+
async keep(sql, replacedIn, target, rows, params = []) {
|
|
77
|
+
const { columns, rewritten } = await this.#follow(sql, target, async () => replacedIn);
|
|
78
|
+
if (rewritten)
|
|
79
|
+
return;
|
|
80
|
+
const quoted = columns.map(quote);
|
|
81
|
+
await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${quoted.join(', ')}, "replaced_in") SELECT ${quoted.map((column) => `"_elt_old".${column}`).join(', ')}, ${Math.trunc(replacedIn)} ${rows}`, [...params]);
|
|
82
|
+
}
|
|
83
|
+
// Deletes the rows of the target that where selects, in commit replacedIn,
|
|
84
|
+
// and keeps them first. where is a WHERE clause over the target named
|
|
85
|
+
// "_elt_old", or nothing for every row.
|
|
86
|
+
async remove(sql, replacedIn, target, where, params = []) {
|
|
87
|
+
await this.keep(sql, replacedIn, target, `FROM ${target} AS "_elt_old"${where}`, params);
|
|
88
|
+
await sql.unsafe(`DELETE FROM ${target} AS "_elt_old"${where}`, [
|
|
89
|
+
...params,
|
|
90
|
+
]);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// Whether a change of a column from type from to type to maps each old value
|
|
94
|
+
// to one value of the new type exactly, with no loss: a widening. Table
|
|
95
|
+
// formats that change a column's type in place allow only these, and read
|
|
96
|
+
// the past with its own schema, which one Postgres view cannot: Delta Lake's
|
|
97
|
+
// type widening (byte to short to int to long, float to double, decimal
|
|
98
|
+
// precision up, date to timestamp_ntz), Iceberg's promotions (int to long,
|
|
99
|
+
// float to double, decimal precision up), BigQuery's change to a less
|
|
100
|
+
// restrictive type, and Snowflake's longer VARCHAR and larger NUMBER. Every
|
|
101
|
+
// value has an exact text form, so each type widens to text, and an array
|
|
102
|
+
// widens when its item does. bigint does not widen to double precision: past
|
|
103
|
+
// 2^53 it loses digits, and Databricks refuses long to double too.
|
|
104
|
+
function widens(from, to) {
|
|
105
|
+
if (to === 'text')
|
|
106
|
+
return true;
|
|
107
|
+
if (from.endsWith('[]') && to.endsWith('[]'))
|
|
108
|
+
return widens(from.slice(0, -2), to.slice(0, -2));
|
|
109
|
+
return widenings.get(from) === to;
|
|
110
|
+
}
|
|
111
|
+
// The widenings between the types the destination declares, as Postgres
|
|
112
|
+
// spells them, other than to text.
|
|
113
|
+
const widenings = new Map([
|
|
114
|
+
['bigint', 'numeric'],
|
|
115
|
+
['date', 'timestamp without time zone'],
|
|
116
|
+
]);
|
|
117
|
+
// The SHA-256 of the values of a staged row, which names its version: a
|
|
118
|
+
// commit that finds the stored row with the same hash leaves it as it is.
|
|
119
|
+
// Each value that is not null counts with its column's name, so a column the
|
|
120
|
+
// stream adds, null in each stored row, makes no new version. A stored file
|
|
121
|
+
// counts as the SHA-256 of its bytes, not as its id.
|
|
122
|
+
export function rowHash(columns, values) {
|
|
123
|
+
const digest = createHash('sha256');
|
|
124
|
+
const add = (tag, data) => {
|
|
125
|
+
digest.update(`${tag}${Buffer.byteLength(data)}:`);
|
|
126
|
+
digest.update(data);
|
|
127
|
+
};
|
|
128
|
+
columns.forEach((column, index) => {
|
|
129
|
+
const value = values[index];
|
|
130
|
+
if (value === null || value === undefined)
|
|
131
|
+
return;
|
|
132
|
+
add('c', column.name);
|
|
133
|
+
if (value instanceof Uint8Array)
|
|
134
|
+
add('b', value);
|
|
135
|
+
else if (typeof value === 'string')
|
|
136
|
+
add('t', value);
|
|
137
|
+
else
|
|
138
|
+
add('j', JSON.stringify(value));
|
|
139
|
+
});
|
|
140
|
+
return digest.digest('hex');
|
|
141
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Identifiers } from '@eltjs/elt';
|
|
2
|
+
declare class PostgresIdentifiers extends Identifiers {
|
|
3
|
+
protected readonly maxBytes = 63;
|
|
4
|
+
protected readonly ownColumns: string[];
|
|
5
|
+
key(name: string): string;
|
|
6
|
+
}
|
|
7
|
+
export declare const identifiers: Readonly<PostgresIdentifiers>;
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { Identifiers } from '@eltjs/elt';
|
|
2
|
+
// Postgres keeps 63 bytes of a name and silently drops the rest, and refuses
|
|
3
|
+
// a column named like one of its system columns however it is quoted. The
|
|
4
|
+
// destination's own columns are loaded_at, which readers see, and the
|
|
5
|
+
// columns only its own SQL reads: committed_in and _elt_row_hash in each
|
|
6
|
+
// table, and replaced_in in each history. The naming rule already turns a
|
|
7
|
+
// field _elt_row_hash into elt_row_hash, so the list need not hold it.
|
|
8
|
+
class PostgresIdentifiers extends Identifiers {
|
|
9
|
+
maxBytes = 63;
|
|
10
|
+
ownColumns = [
|
|
11
|
+
'loaded_at',
|
|
12
|
+
'committed_in',
|
|
13
|
+
'replaced_in',
|
|
14
|
+
'tableoid',
|
|
15
|
+
'cmax',
|
|
16
|
+
'xmax',
|
|
17
|
+
'cmin',
|
|
18
|
+
'xmin',
|
|
19
|
+
'ctid',
|
|
20
|
+
];
|
|
21
|
+
// Quoted identifiers are case-sensitive.
|
|
22
|
+
key(name) {
|
|
23
|
+
return name;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export const identifiers = Object.freeze(new PostgresIdentifiers());
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type postgres from 'postgres';
|
|
2
|
+
import type { PipelineClock } from '@eltjs/elt';
|
|
3
|
+
import { type LoadOwner } from './postgres-numbering.ts';
|
|
4
|
+
export type Transaction = postgres.Sql;
|
|
5
|
+
export declare class PostgresLoad implements AsyncDisposable {
|
|
6
|
+
#private;
|
|
7
|
+
readonly loadedAt: string;
|
|
8
|
+
private constructor();
|
|
9
|
+
get sql(): Transaction;
|
|
10
|
+
static open(url: string, owner: LoadOwner, retaining: Retaining): Promise<PostgresLoad>;
|
|
11
|
+
transaction<T>(work: (sql: Transaction) => Promise<T>): Promise<T>;
|
|
12
|
+
number(sql: Transaction): Promise<number>;
|
|
13
|
+
commit<T>(work: (sql: Transaction, committedIn: number) => Promise<T>): Promise<T>;
|
|
14
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
type Retaining = {
|
|
17
|
+
readonly clock: PipelineClock;
|
|
18
|
+
readonly retention: Temporal.Duration;
|
|
19
|
+
};
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { closeLoad, commitNumber, installNumbering, openLoad, releaseGate, } from './postgres-numbering.js';
|
|
2
|
+
import { PostgresSession } from './postgres-session.js';
|
|
3
|
+
// A run's one connection to the database, and its record in the numbering of the
|
|
4
|
+
// database: open from the start of the load until it is disposed, with a
|
|
5
|
+
// number for each commit. A connection's pass loads, and so does a refresh
|
|
6
|
+
// of a derived dataset. No transaction stays open between commits, so
|
|
7
|
+
// reading the source holds no locks on the targets and keeps nothing from
|
|
8
|
+
// vacuum.
|
|
9
|
+
export class PostgresLoad {
|
|
10
|
+
#connection;
|
|
11
|
+
#clock;
|
|
12
|
+
#retention;
|
|
13
|
+
#owner;
|
|
14
|
+
#id;
|
|
15
|
+
// When the load began, which each row it writes keeps in loaded_at.
|
|
16
|
+
loadedAt;
|
|
17
|
+
constructor(connection, owner, { clock, retention }, id, loadedAt) {
|
|
18
|
+
this.#connection = connection;
|
|
19
|
+
this.#clock = clock;
|
|
20
|
+
this.#retention = retention;
|
|
21
|
+
this.#owner = owner;
|
|
22
|
+
this.#id = id;
|
|
23
|
+
this.loadedAt = loadedAt.toISOString();
|
|
24
|
+
}
|
|
25
|
+
get sql() {
|
|
26
|
+
return this.#connection.sql;
|
|
27
|
+
}
|
|
28
|
+
static async open(url, owner, retaining) {
|
|
29
|
+
const connection = new PostgresSession(url, 'elt');
|
|
30
|
+
try {
|
|
31
|
+
const loadedAt = new Date(retaining.clock.Date.now());
|
|
32
|
+
await installNumbering(connection.sql);
|
|
33
|
+
const id = await openLoad(connection.sql, owner, loadedAt);
|
|
34
|
+
return new PostgresLoad(connection, owner, retaining, id, loadedAt);
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
await connection[Symbol.asyncDispose]();
|
|
38
|
+
throw error;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
// What work does becomes durable and visible together, or not at all. A
|
|
42
|
+
// failure lets go of the gate a commit number holds while it is taken.
|
|
43
|
+
async transaction(work) {
|
|
44
|
+
await this.sql.unsafe('BEGIN');
|
|
45
|
+
try {
|
|
46
|
+
const result = await work(this.sql);
|
|
47
|
+
await this.sql.unsafe('COMMIT');
|
|
48
|
+
return result;
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
await this.sql.unsafe('ROLLBACK');
|
|
52
|
+
await releaseGate(this.sql);
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// Makes the transaction it runs in a commit of this load, and gives its
|
|
57
|
+
// number.
|
|
58
|
+
number(sql) {
|
|
59
|
+
return commitNumber(sql, { load: this.#id, owner: this.#owner }, new Date(this.#clock.Date.now()), this.#retention);
|
|
60
|
+
}
|
|
61
|
+
// A transaction that is a commit of this load, with its number.
|
|
62
|
+
commit(work) {
|
|
63
|
+
return this.transaction(async (sql) => work(sql, await this.number(sql)));
|
|
64
|
+
}
|
|
65
|
+
async [Symbol.asyncDispose]() {
|
|
66
|
+
try {
|
|
67
|
+
await closeLoad(this.sql, this.#id, new Date(this.#clock.Date.now()));
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
await this.#connection[Symbol.asyncDispose]();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|