@eltjs/elt-postgresql 0.0.0-stage → 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +359 -2
  3. package/dist/computed-dataset.d.ts +24 -0
  4. package/dist/computed-dataset.js +346 -0
  5. package/dist/identifier.d.ts +2 -0
  6. package/dist/identifier.js +14 -0
  7. package/dist/index.d.ts +9 -0
  8. package/dist/index.js +8 -0
  9. package/dist/postgres-append-writer.d.ts +5 -0
  10. package/dist/postgres-append-writer.js +11 -0
  11. package/dist/postgres-as-of.d.ts +14 -0
  12. package/dist/postgres-as-of.js +51 -0
  13. package/dist/postgres-checkpoint-store.d.ts +10 -0
  14. package/dist/postgres-checkpoint-store.js +83 -0
  15. package/dist/postgres-column.d.ts +53 -0
  16. package/dist/postgres-column.js +302 -0
  17. package/dist/postgres-columns.d.ts +13 -0
  18. package/dist/postgres-columns.js +80 -0
  19. package/dist/postgres-comments.d.ts +6 -0
  20. package/dist/postgres-comments.js +20 -0
  21. package/dist/postgres-deduplicating-writer.d.ts +25 -0
  22. package/dist/postgres-deduplicating-writer.js +135 -0
  23. package/dist/postgres-dependents.d.ts +2 -0
  24. package/dist/postgres-dependents.js +105 -0
  25. package/dist/postgres-destination.d.ts +31 -0
  26. package/dist/postgres-destination.js +87 -0
  27. package/dist/postgres-file-store.d.ts +23 -0
  28. package/dist/postgres-file-store.js +80 -0
  29. package/dist/postgres-fit.d.ts +9 -0
  30. package/dist/postgres-fit.js +25 -0
  31. package/dist/postgres-history.d.ts +17 -0
  32. package/dist/postgres-history.js +141 -0
  33. package/dist/postgres-identifiers.d.ts +8 -0
  34. package/dist/postgres-identifiers.js +26 -0
  35. package/dist/postgres-load.d.ts +20 -0
  36. package/dist/postgres-load.js +73 -0
  37. package/dist/postgres-numbering.d.ts +42 -0
  38. package/dist/postgres-numbering.js +311 -0
  39. package/dist/postgres-overwrite-writer.d.ts +4 -0
  40. package/dist/postgres-overwrite-writer.js +8 -0
  41. package/dist/postgres-privileges.d.ts +7 -0
  42. package/dist/postgres-privileges.js +50 -0
  43. package/dist/postgres-reader-view.d.ts +11 -0
  44. package/dist/postgres-reader-view.js +64 -0
  45. package/dist/postgres-session.d.ts +11 -0
  46. package/dist/postgres-session.js +61 -0
  47. package/dist/postgres-sync-history-schema.d.ts +31 -0
  48. package/dist/postgres-sync-history-schema.js +100 -0
  49. package/dist/postgres-sync-history.d.ts +13 -0
  50. package/dist/postgres-sync-history.js +152 -0
  51. package/dist/postgres-table.d.ts +15 -0
  52. package/dist/postgres-table.js +73 -0
  53. package/dist/postgres-views.d.ts +10 -0
  54. package/dist/postgres-views.js +119 -0
  55. package/dist/postgres-writer.d.ts +39 -0
  56. package/dist/postgres-writer.js +668 -0
  57. package/dist/view.d.ts +22 -0
  58. package/dist/view.js +81 -0
  59. package/package.json +47 -4
@@ -0,0 +1,105 @@
1
+ import { errorMessage } from '@eltjs/elt';
2
+ import { quote } from './identifier.js';
3
+ import { Privileges } from './postgres-privileges.js';
4
+ // Each view that reads one of the roots, directly or through other views,
5
+ // deepest last. A view's own rule depends on the view, which is not a reader
6
+ // of it.
7
+ const dependents = `
8
+ WITH RECURSIVE roots AS (
9
+ SELECT to_regclass(name) AS oid FROM unnest($1::text[]) AS name
10
+ WHERE to_regclass(name) IS NOT NULL
11
+ ), readers AS (
12
+ SELECT r.ev_class AS oid, 1 AS depth
13
+ FROM pg_depend d JOIN pg_rewrite r
14
+ ON d.classid = 'pg_rewrite'::regclass AND d.objid = r.oid
15
+ WHERE d.refclassid = 'pg_class'::regclass
16
+ AND d.refobjid IN (SELECT oid FROM roots) AND r.ev_class <> d.refobjid
17
+ UNION ALL
18
+ SELECT r.ev_class, readers.depth + 1
19
+ FROM readers JOIN pg_depend d
20
+ ON d.refclassid = 'pg_class'::regclass AND d.refobjid = readers.oid
21
+ JOIN pg_rewrite r ON d.classid = 'pg_rewrite'::regclass AND d.objid = r.oid
22
+ WHERE r.ev_class <> readers.oid
23
+ )
24
+ SELECT format('%I.%I', n.nspname, c.relname) AS "relation",
25
+ CASE c.relkind WHEN 'm' THEN 'MATERIALIZED VIEW' ELSE 'VIEW' END AS "kind",
26
+ pg_get_viewdef(c.oid) AS "definition",
27
+ array_to_string(c.reloptions, ', ') AS "options",
28
+ c.relispopulated AS "populated",
29
+ pg_get_userbyid(c.relowner) AS "owner",
30
+ obj_description(c.oid, 'pg_class') AS "description",
31
+ coalesce((SELECT json_agg(json_build_object('name', a.attname,
32
+ 'description', col_description(c.oid, a.attnum)) ORDER BY a.attnum)
33
+ FROM pg_attribute a
34
+ WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped
35
+ AND col_description(c.oid, a.attnum) IS NOT NULL), '[]') AS "columns",
36
+ coalesce((SELECT json_agg(pg_get_indexdef(i.indexrelid) ORDER BY i.indexrelid)
37
+ FROM pg_index i WHERE i.indrelid = c.oid), '[]') AS "indexes"
38
+ FROM (SELECT oid, max(depth) AS depth FROM readers
39
+ WHERE oid NOT IN (SELECT oid FROM roots) GROUP BY oid) v
40
+ JOIN pg_class c ON c.oid = v.oid
41
+ JOIN pg_namespace n ON n.oid = c.relnamespace
42
+ ORDER BY v.depth, c.oid`;
43
+ // Runs change, which Postgres would refuse under the views that read roots:
44
+ // it binds a view to the relations it read when it was made, so DROP TABLE,
45
+ // ALTER COLUMN TYPE, DROP COLUMN and DROP VIEW fail while one reads them.
46
+ // The views are saved, dropped, and made again over what the change left,
47
+ // by name, in the same transaction, so a reader never finds one missing:
48
+ // each takes the relation that now has the name it reads, as a view of
49
+ // Snowflake or BigQuery, or a Redshift view WITH NO SCHEMA BINDING, resolves
50
+ // its tables at each query (pg-deps-management's save and restore). A view
51
+ // that does not fit what the change left, such as one that reads a column
52
+ // the change dropped, fails the change and names the view. The roots are the
53
+ // change's own, which it can make again, such as a table it swaps in; each
54
+ // root that existed keeps its privileges in the relation that has its name
55
+ // after the change.
56
+ export async function rebindDependents(sql, roots, change) {
57
+ // Every name qualified, so each view reads the same names again.
58
+ const [{ path } = { path: '' }] = await sql.unsafe("SELECT current_setting('search_path') AS path");
59
+ await sql.unsafe("SELECT set_config('search_path', '', true)");
60
+ const saved = await sql.unsafe(dependents, [roots]);
61
+ const existing = await sql.unsafe('SELECT relation FROM unnest($1::text[]) AS relation WHERE to_regclass(relation) IS NOT NULL', [roots]);
62
+ const privileges = new Map();
63
+ for (const { relation } of [...existing, ...saved])
64
+ privileges.set(relation, await Privileges.of(sql, relation));
65
+ await sql.unsafe("SELECT set_config('search_path', $1, true)", [path]);
66
+ for (const view of saved.toReversed())
67
+ try {
68
+ await sql.unsafe(`DROP ${view.kind} ${view.relation}`);
69
+ }
70
+ catch (cause) {
71
+ throw new TypeError(`${view.relation} reads ${roots[0]}, and an object that elt cannot make again reads ${view.relation}: ${errorMessage(cause)}`, { cause });
72
+ }
73
+ const result = await change();
74
+ await sql.unsafe("SELECT set_config('search_path', '', true)");
75
+ for (const view of saved)
76
+ try {
77
+ await make(sql, view);
78
+ }
79
+ catch (cause) {
80
+ throw new TypeError(`${view.relation} reads ${roots[0]} and does not fit its change: ${errorMessage(cause)}. Change the view, or drop it.`, { cause });
81
+ }
82
+ for (const [relation, kept] of privileges)
83
+ await kept.apply(sql, relation);
84
+ await sql.unsafe("SELECT set_config('search_path', $1, true)", [path]);
85
+ return result;
86
+ }
87
+ async function make(sql, view) {
88
+ const options = view.options === null ? '' : ` WITH (${view.options})`;
89
+ let data = '';
90
+ if (view.kind === 'MATERIALIZED VIEW')
91
+ data = view.populated ? ' WITH DATA' : ' WITH NO DATA';
92
+ await sql.unsafe(`CREATE ${view.kind} ${view.relation}${options} AS ${view.definition.replace(/;\s*$/, '')}${data}`);
93
+ for (const index of view.indexes)
94
+ await sql.unsafe(index);
95
+ const [{ me } = { me: '' }] = await sql.unsafe('SELECT current_user AS me');
96
+ if (view.owner !== me)
97
+ await sql.unsafe(`ALTER ${view.kind} ${view.relation} OWNER TO ${quote(view.owner)}`);
98
+ // COMMENT cannot bind values; Postgres quotes the text itself.
99
+ const comments = await sql.unsafe(`SELECT format('COMMENT ON ${view.kind} %s IS %L', $1::text, $2::text) AS statement WHERE $2::text IS NOT NULL
100
+ UNION ALL
101
+ SELECT format('COMMENT ON COLUMN %s.%I IS %L', $1::text, c->>'name', c->>'description')
102
+ FROM json_array_elements($3::text::json) AS c`, [view.relation, view.description, JSON.stringify(view.columns)]);
103
+ for (const { statement } of comments)
104
+ await sql.unsafe(statement);
105
+ }
@@ -0,0 +1,31 @@
1
+ import { type CopyConfiguration, Destination, type Load, type PipelineClock } from '@eltjs/elt';
2
+ import { type PostgresReading } from './postgres-as-of.ts';
3
+ import type { PostgresColumn } from './postgres-column.ts';
4
+ import { PostgresColumns } from './postgres-columns.ts';
5
+ import { PostgresTable } from './postgres-table.ts';
6
+ import type { PostgresWriter } from './postgres-writer.ts';
7
+ export declare class PostgresDestination extends Destination<PostgresTable> {
8
+ #private;
9
+ readonly supportedDestinationSyncModes: readonly ["overwrite", "append", "append_dedup", "overwrite_dedup"];
10
+ readonly schema: string;
11
+ readonly retention: Temporal.Duration;
12
+ constructor({ url, schema, retention, }: {
13
+ url: string;
14
+ schema: string;
15
+ retention?: Temporal.Duration;
16
+ });
17
+ get url(): string;
18
+ location(target: PostgresTable): string;
19
+ load(owner: {
20
+ clock: PipelineClock;
21
+ connection: string;
22
+ }): Promise<Load<PostgresTable>>;
23
+ positions(after?: string): Promise<readonly string[] | {
24
+ readonly expired: string;
25
+ }>;
26
+ reading(position?: string): Promise<PostgresReading | {
27
+ readonly expired: string;
28
+ }>;
29
+ table(name: string, configure?: (columns: PostgresColumns) => readonly PostgresColumn[]): PostgresTable;
30
+ createWriter(configuration: CopyConfiguration, target: PostgresTable): PostgresWriter;
31
+ }
@@ -0,0 +1,87 @@
1
+ import { Destination, } from '@eltjs/elt';
2
+ import { quote } from './identifier.js';
3
+ import { PostgresAppendWriter } from './postgres-append-writer.js';
4
+ import { readAsOf } from './postgres-as-of.js';
5
+ import { PostgresColumns } from './postgres-columns.js';
6
+ import { PostgresDeduplicatingWriter } from './postgres-deduplicating-writer.js';
7
+ import { PostgresLoad } from './postgres-load.js';
8
+ import { positionsAfter } from './postgres-numbering.js';
9
+ import { PostgresOverwriteWriter } from './postgres-overwrite-writer.js';
10
+ import { PostgresSession, connectionUrl, schemaName, } from './postgres-session.js';
11
+ import { PostgresTable } from './postgres-table.js';
12
+ // Loads into one schema of one database. The URL carries credentials, so it
13
+ // stays private and out of location(), which is persisted with checkpoints.
14
+ export class PostgresDestination extends Destination {
15
+ supportedDestinationSyncModes = Object.freeze([
16
+ 'overwrite',
17
+ 'append',
18
+ 'append_dedup',
19
+ 'overwrite_dedup',
20
+ ]);
21
+ schema;
22
+ // How long the history keeps a position readable: 30 days unless set, as
23
+ // Delta Lake keeps its log. The destinations of one database share one
24
+ // numbering, so the shortest retention among them decides.
25
+ retention;
26
+ #url;
27
+ constructor({ url, schema, retention = Temporal.Duration.from({ days: 30 }), }) {
28
+ super();
29
+ if (retention.sign < 0)
30
+ throw new RangeError(`Retention ${retention} must not be negative`);
31
+ this.#url = connectionUrl(url);
32
+ this.schema = schemaName(schema);
33
+ this.retention = retention;
34
+ Object.freeze(this);
35
+ }
36
+ // The connection URL, for the views and computed datasets of the
37
+ // destination. A getter, so neither JSON nor util.inspect shows it.
38
+ get url() {
39
+ return this.#url;
40
+ }
41
+ location(target) {
42
+ return `${quote(this.schema)}.${target.quotedName}`;
43
+ }
44
+ // One connection for the whole run; each commit is its own transaction,
45
+ // numbered in the numbering that the destinations of the database share.
46
+ async load(owner) {
47
+ const load = await PostgresLoad.open(this.#url, { connector: owner.connection }, { clock: owner.clock, retention: this.retention });
48
+ return {
49
+ prepare: (configuration, target, binding) => this.createWriter(configuration, target).prepare(load, binding),
50
+ [Symbol.asyncDispose]: () => load[Symbol.asyncDispose](),
51
+ };
52
+ }
53
+ // The positions of the database after a position, ascending, or all of
54
+ // them; each one, `${epoch}:${number}`, names a commit where no pass of any
55
+ // schema of the database is half read. A position of another epoch, or past
56
+ // the newest commit, is expired.
57
+ async positions(after) {
58
+ await using session = new PostgresSession(this.#url, 'elt-positions');
59
+ return await positionsAfter(session.sql, after);
60
+ }
61
+ // The database as of a position, or as of its newest one, read through the
62
+ // reader views: { expired } for a position of another epoch or past the
63
+ // newest commit, and a TypeError for one inside a pass or not a position.
64
+ reading(position) {
65
+ return readAsOf(this.#url, position);
66
+ }
67
+ table(name, configure) {
68
+ return new PostgresTable(name, configure?.(new PostgresColumns()));
69
+ }
70
+ createWriter(configuration, target) {
71
+ this.validateConfiguration(configuration, target);
72
+ if (!(target instanceof PostgresTable))
73
+ throw new TypeError('Postgres requires Postgres table targets');
74
+ const table = target.resolve(configuration.stream);
75
+ switch (configuration.destinationSyncMode) {
76
+ case 'append_dedup':
77
+ case 'overwrite_dedup':
78
+ return new PostgresDeduplicatingWriter(configuration, this.#url, this.schema, table, this.retention);
79
+ case 'append':
80
+ return new PostgresAppendWriter(configuration, this.#url, this.schema, table, this.retention);
81
+ case 'overwrite':
82
+ return new PostgresOverwriteWriter(configuration, this.#url, this.schema, table, this.retention);
83
+ default:
84
+ throw new TypeError(`Destination does not support ${configuration.destinationSyncMode}`);
85
+ }
86
+ }
87
+ }
@@ -0,0 +1,23 @@
1
+ import type postgres from 'postgres';
2
+ import type { FileContent } from '@eltjs/elt';
3
+ import type { PostgresColumn } from './postgres-column.ts';
4
+ import type { PostgresTable } from './postgres-table.ts';
5
+ export declare class PostgresFileStore {
6
+ #private;
7
+ static readonly chunkSize: number;
8
+ readonly schema: string;
9
+ readonly table: PostgresTable;
10
+ readonly column: PostgresColumn;
11
+ readonly qualifiedName: string;
12
+ constructor(schema: string, table: PostgresTable, column: PostgresColumn);
13
+ static qualifiedName(schema: string, table: PostgresTable, column: string): string;
14
+ stage(sql: postgres.Sql): Promise<void>;
15
+ save(sql: postgres.Sql, content: FileContent): Promise<{
16
+ readonly file: string;
17
+ readonly sha256: Uint8Array;
18
+ }>;
19
+ publish(sql: postgres.Sql): Promise<void>;
20
+ discard(sql: postgres.Sql): Promise<void>;
21
+ unstage(sql: postgres.Sql): Promise<void>;
22
+ prune(sql: Pick<postgres.Sql, 'unsafe'>, tables: readonly string[]): Promise<void>;
23
+ }
@@ -0,0 +1,80 @@
1
+ import { createHash, hash, randomUUID } from 'node:crypto';
2
+ import { quote } from './identifier.js';
3
+ const chunks = '("file" UUID NOT NULL, "n" BIGINT NOT NULL, "bytes" BYTEA NOT NULL, PRIMARY KEY ("file", "n"))';
4
+ // Like SQLite, keep original files in bounded chunks rather than one whole-file
5
+ // value. A file column contains the UUID of its ordered BYTEA chunks.
6
+ export class PostgresFileStore {
7
+ static chunkSize = 4 * 1024 * 1024;
8
+ schema;
9
+ table;
10
+ column;
11
+ qualifiedName;
12
+ // The TEMP table of chunks saved since the last commit, private to the
13
+ // load's session like the stage of their records.
14
+ #staged;
15
+ constructor(schema, table, column) {
16
+ this.schema = schema;
17
+ this.table = table;
18
+ this.column = column;
19
+ this.qualifiedName = PostgresFileStore.qualifiedName(schema, table, column.name);
20
+ this.#staged = quote(`_elt_files_stage_${PostgresFileStore.#key(table, column.name)}`);
21
+ }
22
+ // The chunk table of the column named column, which may no longer be one
23
+ // of the table's file columns.
24
+ static qualifiedName(schema, table, column) {
25
+ return `${quote(schema)}.${quote(`_elt_files_${PostgresFileStore.#key(table, column)}`)}`;
26
+ }
27
+ static #key(table, column) {
28
+ return hash('sha256', JSON.stringify([table.name, column])).slice(0, 40);
29
+ }
30
+ async stage(sql) {
31
+ await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${this.#staged}`);
32
+ await sql.unsafe(`CREATE TEMP TABLE ${this.#staged} ${chunks}`);
33
+ }
34
+ // Stages the file's bytes under a new id, and gives the id and the SHA-256
35
+ // of the bytes, which the row hash counts in place of the id.
36
+ async save(sql, content) {
37
+ const file = randomUUID();
38
+ const sha256 = createHash('sha256');
39
+ let n = 0;
40
+ const insert = (bytes) => sql.unsafe(`INSERT INTO pg_temp.${this.#staged} ("file", "n", "bytes") VALUES ($1, $2, $3)`, [file, n++, Buffer.from(bytes)]);
41
+ for await (const chunk of content.chunks(PostgresFileStore.chunkSize)) {
42
+ sha256.update(chunk);
43
+ await insert(chunk);
44
+ }
45
+ if (n === 0)
46
+ await insert(new Uint8Array());
47
+ return { file, sha256: sha256.digest() };
48
+ }
49
+ // Moves the staged chunks into the store, inside a commit.
50
+ async publish(sql) {
51
+ await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.qualifiedName} ${chunks}`);
52
+ await sql.unsafe(`INSERT INTO ${this.qualifiedName} SELECT * FROM pg_temp.${this.#staged}`);
53
+ await sql.unsafe(`TRUNCATE pg_temp.${this.#staged}`);
54
+ }
55
+ async discard(sql) {
56
+ await sql.unsafe(`TRUNCATE pg_temp.${this.#staged}`);
57
+ }
58
+ async unstage(sql) {
59
+ await sql.unsafe(`DROP TABLE pg_temp.${this.#staged}`);
60
+ }
61
+ // Called inside each commit, after the merge: chunks no row of the tables
62
+ // refers to, such as a deduplication loser's, go. The tables are the target,
63
+ // its history, whose rows keep their files for a reading as of a position
64
+ // before the commit that replaced them, and, while a reload is open, its
65
+ // hidden target. A table can type the column otherwise, such as a target
66
+ // whose reload makes it a file column, so ids compare as text. This store
67
+ // belongs to just one target column and one writer.
68
+ // ponytail: scans this column's chunks; use targeted cleanup if checkpoint cost grows.
69
+ async prune(sql, tables) {
70
+ const referring = [];
71
+ for (const table of tables) {
72
+ const [column] = await sql.unsafe('SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass($1) AND attname = $2 AND NOT attisdropped', [table, this.column.name]);
73
+ if (column !== undefined)
74
+ referring.push(table);
75
+ }
76
+ await sql.unsafe(`DELETE FROM ${this.qualifiedName} AS chunks WHERE ${referring
77
+ .map((table) => `NOT EXISTS (SELECT 1 FROM ${table} AS kept WHERE kept.${this.column.quotedName}::text = chunks."file"::text)`)
78
+ .join(' AND ') || 'true'}`);
79
+ }
80
+ }
@@ -0,0 +1,9 @@
1
+ import type { StoredFit } from '@eltjs/elt';
2
+ import type { Transaction } from './postgres-load.ts';
3
+ import type { PostgresTable } from './postgres-table.ts';
4
+ export declare function storedColumns(sql: Pick<Transaction, 'unsafe'>, relation: string): import("postgres").PendingQuery<{
5
+ name: string;
6
+ type: string;
7
+ required: boolean;
8
+ }[]>;
9
+ export declare function storedFit(sql: Transaction, target: PostgresTable, table: string, stage: string): Promise<StoredFit>;
@@ -0,0 +1,25 @@
1
+ // The columns a relation stores, with each one's type as Postgres spells it
2
+ // and whether it is NOT NULL.
3
+ export function storedColumns(sql, relation) {
4
+ return sql.unsafe('SELECT attname AS name, format_type(atttypid, atttypmod) AS type, attnotnull AS required FROM pg_attribute WHERE attrelid = to_regclass($1) AND attnum > 0 AND NOT attisdropped', [relation]);
5
+ }
6
+ // Whether a stored table has the columns, types and NOT NULL the stream
7
+ // needs. The stage is built from the same column declarations, so Postgres
8
+ // spells both alike.
9
+ export async function storedFit(sql, target, table, stage) {
10
+ const stored = await storedColumns(sql, table);
11
+ if (stored.length === 0)
12
+ return 'missing';
13
+ const types = new Map((await storedColumns(sql, `pg_temp.${stage}`)).map(({ name, type }) => [
14
+ name,
15
+ type,
16
+ ]));
17
+ const needed = [
18
+ ...target.columns.map((column) => `${column.name} ${types.get(column.name)} ${column.isPrimaryKey}`),
19
+ 'loaded_at timestamp with time zone true',
20
+ 'committed_in integer true',
21
+ '_elt_row_hash text false',
22
+ ];
23
+ const has = stored.map(({ name, type, required }) => `${name} ${type} ${required}`);
24
+ return needed.sort().join('\0') === has.sort().join('\0') ? 'fits' : 'stale';
25
+ }
@@ -0,0 +1,17 @@
1
+ import type { EncodedValue } from './postgres-column.ts';
2
+ import type { Queries } from './postgres-session.ts';
3
+ export declare const version = "\"_elt_row_hash\"";
4
+ export declare class TargetHistory {
5
+ #private;
6
+ readonly schema: string;
7
+ readonly name: string;
8
+ constructor(schema: string, key: string);
9
+ get qualifiedName(): string;
10
+ create(sql: Queries, target: string, change: () => Promise<number>): Promise<void>;
11
+ expire(sql: Queries): Promise<void>;
12
+ keep(sql: Queries, replacedIn: number, target: string, rows: string, params?: readonly string[]): Promise<void>;
13
+ remove(sql: Queries, replacedIn: number, target: string, where: string, params?: readonly string[]): Promise<void>;
14
+ }
15
+ export declare function rowHash(columns: readonly {
16
+ readonly name: string;
17
+ }[], values: readonly (EncodedValue | Uint8Array)[]): string;
@@ -0,0 +1,141 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { quote } from './identifier.js';
3
+ import { storedColumns } from './postgres-fit.js';
4
+ import { expireBefore, minValid } from './postgres-numbering.js';
5
+ export const version = '"_elt_row_hash"';
6
+ // The rows of a target that commits replaced or removed, each with the
7
+ // number of the commit that did, as a SQL Server temporal table keeps them
8
+ // beside its current table. Each column has the type of the target's column
9
+ // of the same name, and changes its type when the target's does; a column
10
+ // that the target drops stays, with its last type.
11
+ export class TargetHistory {
12
+ schema;
13
+ name;
14
+ constructor(schema, key) {
15
+ this.schema = schema;
16
+ this.name = `_elt_history_${key}`;
17
+ }
18
+ get qualifiedName() {
19
+ return `${quote(this.schema)}.${quote(this.name)}`;
20
+ }
21
+ // Made with the target, with the target's columns, none NOT NULL: a
22
+ // column that the target drops is NULL in each row kept after. A history
23
+ // that exists follows the target; change gives the number of the commit
24
+ // that changes the target's shape.
25
+ async create(sql, target, change) {
26
+ const columns = await storedColumns(sql, target);
27
+ await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.qualifiedName} (${[...columns.map(({ name, type }) => `${quote(name)} ${type}`), '"replaced_in" INTEGER NOT NULL'].join(', ')})`);
28
+ await sql.unsafe(`CREATE INDEX IF NOT EXISTS ${quote(`${this.name}_replaced`)} ON ${this.qualifiedName} ("replaced_in")`);
29
+ await this.#follow(sql, target, change);
30
+ }
31
+ // Takes each column of the target that it does not have, and the type of
32
+ // each target column whose type it does not have. A widening casts the
33
+ // kept values, so each position still reads. Any other change of type
34
+ // acts as a rewrite in the commit of the change: each position before it
35
+ // expires, and the rows that only those positions read go before the cast,
36
+ // so no kept value blocks it. Only a change takes a lock, so a commit that
37
+ // changes no shape waits for no reader. Gives the target's columns, and
38
+ // whether the change expired the positions before it.
39
+ async #follow(sql, target, change) {
40
+ const columns = await storedColumns(sql, target);
41
+ const kept = new Map((await storedColumns(sql, this.qualifiedName)).map(({ name, type }) => [
42
+ name,
43
+ type,
44
+ ]));
45
+ const rewritten = columns.some(({ name, type }) => {
46
+ const had = kept.get(name);
47
+ return had !== undefined && had !== type && !widens(had, type);
48
+ });
49
+ if (rewritten) {
50
+ const changedIn = await change();
51
+ await expireBefore(sql, changedIn, 'type');
52
+ await sql.unsafe(`DELETE FROM ${this.qualifiedName} WHERE "replaced_in" <= $1`, [changedIn]);
53
+ }
54
+ for (const { name, type } of columns) {
55
+ const column = quote(name);
56
+ const had = kept.get(name);
57
+ if (had === undefined)
58
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ADD COLUMN ${column} ${type}`);
59
+ else if (had !== type)
60
+ await sql.unsafe(`ALTER TABLE ${this.qualifiedName} ALTER COLUMN ${column} TYPE ${type} USING ${column}::${type}`);
61
+ }
62
+ return { columns: columns.map(({ name }) => name), rewritten };
63
+ }
64
+ // Deletes the rows that only a reading before min_valid reads, which the
65
+ // retention or a change of type left, once the history exists.
66
+ async expire(sql) {
67
+ const [history] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.qualifiedName]);
68
+ if (history?.exists !== true)
69
+ return;
70
+ await sql.unsafe(`DELETE FROM ${this.qualifiedName} WHERE "replaced_in" <= $1`, [await minValid(sql)]);
71
+ }
72
+ // Copies the rows of the target that rows selects into the history, as
73
+ // replaced in commit replacedIn. rows is the FROM clause of a SELECT in
74
+ // which "_elt_old" names the target. A commit that expired the positions
75
+ // before it keeps no row, as only those positions would read it.
76
+ async keep(sql, replacedIn, target, rows, params = []) {
77
+ const { columns, rewritten } = await this.#follow(sql, target, async () => replacedIn);
78
+ if (rewritten)
79
+ return;
80
+ const quoted = columns.map(quote);
81
+ await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${quoted.join(', ')}, "replaced_in") SELECT ${quoted.map((column) => `"_elt_old".${column}`).join(', ')}, ${Math.trunc(replacedIn)} ${rows}`, [...params]);
82
+ }
83
+ // Deletes the rows of the target that where selects, in commit replacedIn,
84
+ // and keeps them first. where is a WHERE clause over the target named
85
+ // "_elt_old", or nothing for every row.
86
+ async remove(sql, replacedIn, target, where, params = []) {
87
+ await this.keep(sql, replacedIn, target, `FROM ${target} AS "_elt_old"${where}`, params);
88
+ await sql.unsafe(`DELETE FROM ${target} AS "_elt_old"${where}`, [
89
+ ...params,
90
+ ]);
91
+ }
92
+ }
93
+ // Whether a change of a column from type from to type to maps each old value
94
+ // to one value of the new type exactly, with no loss: a widening. Table
95
+ // formats that change a column's type in place allow only these, and read
96
+ // the past with its own schema, which one Postgres view cannot: Delta Lake's
97
+ // type widening (byte to short to int to long, float to double, decimal
98
+ // precision up, date to timestamp_ntz), Iceberg's promotions (int to long,
99
+ // float to double, decimal precision up), BigQuery's change to a less
100
+ // restrictive type, and Snowflake's longer VARCHAR and larger NUMBER. Every
101
+ // value has an exact text form, so each type widens to text, and an array
102
+ // widens when its item does. bigint does not widen to double precision: past
103
+ // 2^53 it loses digits, and Databricks refuses long to double too.
104
+ function widens(from, to) {
105
+ if (to === 'text')
106
+ return true;
107
+ if (from.endsWith('[]') && to.endsWith('[]'))
108
+ return widens(from.slice(0, -2), to.slice(0, -2));
109
+ return widenings.get(from) === to;
110
+ }
111
+ // The widenings between the types the destination declares, as Postgres
112
+ // spells them, other than to text.
113
+ const widenings = new Map([
114
+ ['bigint', 'numeric'],
115
+ ['date', 'timestamp without time zone'],
116
+ ]);
117
+ // The SHA-256 of the values of a staged row, which names its version: a
118
+ // commit that finds the stored row with the same hash leaves it as it is.
119
+ // Each value that is not null counts with its column's name, so a column the
120
+ // stream adds, null in each stored row, makes no new version. A stored file
121
+ // counts as the SHA-256 of its bytes, not as its id.
122
+ export function rowHash(columns, values) {
123
+ const digest = createHash('sha256');
124
+ const add = (tag, data) => {
125
+ digest.update(`${tag}${Buffer.byteLength(data)}:`);
126
+ digest.update(data);
127
+ };
128
+ columns.forEach((column, index) => {
129
+ const value = values[index];
130
+ if (value === null || value === undefined)
131
+ return;
132
+ add('c', column.name);
133
+ if (value instanceof Uint8Array)
134
+ add('b', value);
135
+ else if (typeof value === 'string')
136
+ add('t', value);
137
+ else
138
+ add('j', JSON.stringify(value));
139
+ });
140
+ return digest.digest('hex');
141
+ }
@@ -0,0 +1,8 @@
1
+ import { Identifiers } from '@eltjs/elt';
2
+ declare class PostgresIdentifiers extends Identifiers {
3
+ protected readonly maxBytes = 63;
4
+ protected readonly ownColumns: string[];
5
+ key(name: string): string;
6
+ }
7
+ export declare const identifiers: Readonly<PostgresIdentifiers>;
8
+ export {};
@@ -0,0 +1,26 @@
1
+ import { Identifiers } from '@eltjs/elt';
2
+ // Postgres keeps 63 bytes of a name and silently drops the rest, and refuses
3
+ // a column named like one of its system columns however it is quoted. The
4
+ // destination's own columns are loaded_at, which readers see, and the
5
+ // columns only its own SQL reads: committed_in and _elt_row_hash in each
6
+ // table, and replaced_in in each history. The naming rule already turns a
7
+ // field _elt_row_hash into elt_row_hash, so the list need not hold it.
8
+ class PostgresIdentifiers extends Identifiers {
9
+ maxBytes = 63;
10
+ ownColumns = [
11
+ 'loaded_at',
12
+ 'committed_in',
13
+ 'replaced_in',
14
+ 'tableoid',
15
+ 'cmax',
16
+ 'xmax',
17
+ 'cmin',
18
+ 'xmin',
19
+ 'ctid',
20
+ ];
21
+ // Quoted identifiers are case-sensitive.
22
+ key(name) {
23
+ return name;
24
+ }
25
+ }
26
+ export const identifiers = Object.freeze(new PostgresIdentifiers());
@@ -0,0 +1,20 @@
1
+ import type postgres from 'postgres';
2
+ import type { PipelineClock } from '@eltjs/elt';
3
+ import { type LoadOwner } from './postgres-numbering.ts';
4
+ export type Transaction = postgres.Sql;
5
+ export declare class PostgresLoad implements AsyncDisposable {
6
+ #private;
7
+ readonly loadedAt: string;
8
+ private constructor();
9
+ get sql(): Transaction;
10
+ static open(url: string, owner: LoadOwner, retaining: Retaining): Promise<PostgresLoad>;
11
+ transaction<T>(work: (sql: Transaction) => Promise<T>): Promise<T>;
12
+ number(sql: Transaction): Promise<number>;
13
+ commit<T>(work: (sql: Transaction, committedIn: number) => Promise<T>): Promise<T>;
14
+ [Symbol.asyncDispose](): Promise<void>;
15
+ }
16
+ type Retaining = {
17
+ readonly clock: PipelineClock;
18
+ readonly retention: Temporal.Duration;
19
+ };
20
+ export {};
@@ -0,0 +1,73 @@
1
+ import { closeLoad, commitNumber, installNumbering, openLoad, releaseGate, } from './postgres-numbering.js';
2
+ import { PostgresSession } from './postgres-session.js';
3
+ // A run's one connection to the database, and its record in the numbering of the
4
+ // database: open from the start of the load until it is disposed, with a
5
+ // number for each commit. A connection's pass loads, and so does a refresh
6
+ // of a derived dataset. No transaction stays open between commits, so
7
+ // reading the source holds no locks on the targets and keeps nothing from
8
+ // vacuum.
9
+ export class PostgresLoad {
10
+ #connection;
11
+ #clock;
12
+ #retention;
13
+ #owner;
14
+ #id;
15
+ // When the load began, which each row it writes keeps in loaded_at.
16
+ loadedAt;
17
+ constructor(connection, owner, { clock, retention }, id, loadedAt) {
18
+ this.#connection = connection;
19
+ this.#clock = clock;
20
+ this.#retention = retention;
21
+ this.#owner = owner;
22
+ this.#id = id;
23
+ this.loadedAt = loadedAt.toISOString();
24
+ }
25
+ get sql() {
26
+ return this.#connection.sql;
27
+ }
28
+ static async open(url, owner, retaining) {
29
+ const connection = new PostgresSession(url, 'elt');
30
+ try {
31
+ const loadedAt = new Date(retaining.clock.Date.now());
32
+ await installNumbering(connection.sql);
33
+ const id = await openLoad(connection.sql, owner, loadedAt);
34
+ return new PostgresLoad(connection, owner, retaining, id, loadedAt);
35
+ }
36
+ catch (error) {
37
+ await connection[Symbol.asyncDispose]();
38
+ throw error;
39
+ }
40
+ }
41
+ // What work does becomes durable and visible together, or not at all. A
42
+ // failure lets go of the gate a commit number holds while it is taken.
43
+ async transaction(work) {
44
+ await this.sql.unsafe('BEGIN');
45
+ try {
46
+ const result = await work(this.sql);
47
+ await this.sql.unsafe('COMMIT');
48
+ return result;
49
+ }
50
+ catch (error) {
51
+ await this.sql.unsafe('ROLLBACK');
52
+ await releaseGate(this.sql);
53
+ throw error;
54
+ }
55
+ }
56
+ // Makes the transaction it runs in a commit of this load, and gives its
57
+ // number.
58
+ number(sql) {
59
+ return commitNumber(sql, { load: this.#id, owner: this.#owner }, new Date(this.#clock.Date.now()), this.#retention);
60
+ }
61
+ // A transaction that is a commit of this load, with its number.
62
+ commit(work) {
63
+ return this.transaction(async (sql) => work(sql, await this.number(sql)));
64
+ }
65
+ async [Symbol.asyncDispose]() {
66
+ try {
67
+ await closeLoad(this.sql, this.#id, new Date(this.#clock.Date.now()));
68
+ }
69
+ finally {
70
+ await this.#connection[Symbol.asyncDispose]();
71
+ }
72
+ }
73
+ }