@eltjs/elt-postgresql 0.0.0-stage → 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +359 -2
  3. package/dist/computed-dataset.d.ts +24 -0
  4. package/dist/computed-dataset.js +346 -0
  5. package/dist/identifier.d.ts +2 -0
  6. package/dist/identifier.js +14 -0
  7. package/dist/index.d.ts +9 -0
  8. package/dist/index.js +8 -0
  9. package/dist/postgres-append-writer.d.ts +5 -0
  10. package/dist/postgres-append-writer.js +11 -0
  11. package/dist/postgres-as-of.d.ts +14 -0
  12. package/dist/postgres-as-of.js +51 -0
  13. package/dist/postgres-checkpoint-store.d.ts +10 -0
  14. package/dist/postgres-checkpoint-store.js +83 -0
  15. package/dist/postgres-column.d.ts +53 -0
  16. package/dist/postgres-column.js +302 -0
  17. package/dist/postgres-columns.d.ts +13 -0
  18. package/dist/postgres-columns.js +80 -0
  19. package/dist/postgres-comments.d.ts +6 -0
  20. package/dist/postgres-comments.js +20 -0
  21. package/dist/postgres-deduplicating-writer.d.ts +25 -0
  22. package/dist/postgres-deduplicating-writer.js +135 -0
  23. package/dist/postgres-dependents.d.ts +2 -0
  24. package/dist/postgres-dependents.js +105 -0
  25. package/dist/postgres-destination.d.ts +31 -0
  26. package/dist/postgres-destination.js +87 -0
  27. package/dist/postgres-file-store.d.ts +23 -0
  28. package/dist/postgres-file-store.js +80 -0
  29. package/dist/postgres-fit.d.ts +9 -0
  30. package/dist/postgres-fit.js +25 -0
  31. package/dist/postgres-history.d.ts +17 -0
  32. package/dist/postgres-history.js +141 -0
  33. package/dist/postgres-identifiers.d.ts +8 -0
  34. package/dist/postgres-identifiers.js +26 -0
  35. package/dist/postgres-load.d.ts +20 -0
  36. package/dist/postgres-load.js +73 -0
  37. package/dist/postgres-numbering.d.ts +42 -0
  38. package/dist/postgres-numbering.js +311 -0
  39. package/dist/postgres-overwrite-writer.d.ts +4 -0
  40. package/dist/postgres-overwrite-writer.js +8 -0
  41. package/dist/postgres-privileges.d.ts +7 -0
  42. package/dist/postgres-privileges.js +50 -0
  43. package/dist/postgres-reader-view.d.ts +11 -0
  44. package/dist/postgres-reader-view.js +64 -0
  45. package/dist/postgres-session.d.ts +11 -0
  46. package/dist/postgres-session.js +61 -0
  47. package/dist/postgres-sync-history-schema.d.ts +31 -0
  48. package/dist/postgres-sync-history-schema.js +100 -0
  49. package/dist/postgres-sync-history.d.ts +13 -0
  50. package/dist/postgres-sync-history.js +152 -0
  51. package/dist/postgres-table.d.ts +15 -0
  52. package/dist/postgres-table.js +73 -0
  53. package/dist/postgres-views.d.ts +10 -0
  54. package/dist/postgres-views.js +119 -0
  55. package/dist/postgres-writer.d.ts +39 -0
  56. package/dist/postgres-writer.js +668 -0
  57. package/dist/view.d.ts +22 -0
  58. package/dist/view.js +81 -0
  59. package/package.json +47 -4
@@ -0,0 +1,100 @@
1
+ import { syncHistoryRelations } from '@eltjs/elt';
2
+ // When connector's stream last matched its upstream: the start of its last
3
+ // successful pass, or a later check that found it unchanged, each no later
4
+ // than when the source said the upstream itself was last current. NULL if it
5
+ // never succeeded. LEAST and GREATEST skip NULLs.
6
+ const currentAsOf = (connector, stream) => `(
7
+ SELECT CASE WHEN success.id IS NULL THEN NULL ELSE GREATEST(LEAST(success.started_at, success.upstream_current_as_of), k.checked_at) END
8
+ FROM (SELECT 1) one
9
+ LEFT JOIN LATERAL (
10
+ SELECT s.id, s.started_at, sc.upstream_current_as_of FROM _warehouse.extraction_coverage sc
11
+ JOIN _warehouse.sync_attempts s ON s.id = sc.attempt_id
12
+ WHERE s.connector = ${connector} AND sc.stream = ${stream} AND sc.status = 'succeeded'
13
+ ORDER BY s.completed_at DESC, s.id DESC LIMIT 1
14
+ ) success ON true
15
+ LEFT JOIN _warehouse.stream_checks k ON k.connector = ${connector} AND k.stream = ${stream}
16
+ )`;
17
+ const queries = {
18
+ sync_attempts: 'SELECT id AS attempt_id, connector, source, started_at, completed_at, status, error, failure_type FROM _warehouse.sync_attempts',
19
+ extraction_coverage: `SELECT c.attempt_id, a.connector, a.source, a.started_at, a.completed_at,
20
+ c.stream, c.target_schema, c.target_table,
21
+ EXISTS (SELECT FROM pg_class t JOIN pg_namespace n ON n.oid = t.relnamespace
22
+ WHERE n.nspname = c.target_schema AND t.relname = c.target_table AND t.relkind = 'r') AS target_exists,
23
+ c.sync_mode, c.destination_sync_mode, c.description, c.selection,
24
+ c.status, c.written_count, c.deleted_count, c.failures
25
+ FROM _warehouse.extraction_coverage c JOIN _warehouse.sync_attempts a ON a.id = c.attempt_id`,
26
+ sync_status: `SELECT DISTINCT ON (a.connector) a.connector,
27
+ a.id AS latest_attempt_id, a.started_at, a.completed_at, a.status, a.error, a.failure_type,
28
+ success.id AS last_successful_attempt_id, success.completed_at AS last_successful_sync_at,
29
+ (SELECT CASE WHEN count(*) = count(current_as_of) THEN min(current_as_of) END
30
+ FROM (SELECT ${currentAsOf('a.connector', 'd.stream')} AS current_as_of
31
+ FROM (SELECT DISTINCT dc.stream FROM _warehouse.extraction_coverage dc
32
+ JOIN _warehouse.sync_attempts da ON da.id = dc.attempt_id
33
+ WHERE da.connector = a.connector) d) streams) AS current_as_of
34
+ FROM _warehouse.sync_attempts a
35
+ LEFT JOIN LATERAL (
36
+ SELECT id, completed_at FROM _warehouse.sync_attempts
37
+ WHERE connector = a.connector AND status = 'succeeded'
38
+ ORDER BY completed_at DESC, id DESC LIMIT 1
39
+ ) success ON true
40
+ ORDER BY a.connector, a.id DESC`,
41
+ stream_status: `SELECT DISTINCT ON (a.connector, c.stream) a.connector, c.stream,
42
+ c.target_schema, c.target_table,
43
+ a.id AS latest_attempt_id, a.started_at, a.completed_at, c.status,
44
+ success.id AS last_successful_attempt_id, success.completed_at AS last_successful_sync_at,
45
+ ${currentAsOf('a.connector', 'c.stream')} AS current_as_of
46
+ FROM _warehouse.extraction_coverage c
47
+ JOIN _warehouse.sync_attempts a ON a.id = c.attempt_id
48
+ LEFT JOIN LATERAL (
49
+ SELECT s.id, s.completed_at FROM _warehouse.extraction_coverage sc
50
+ JOIN _warehouse.sync_attempts s ON s.id = sc.attempt_id
51
+ WHERE s.connector = a.connector AND sc.stream = c.stream AND sc.status = 'succeeded'
52
+ ORDER BY s.completed_at DESC, s.id DESC LIMIT 1
53
+ ) success ON true
54
+ ORDER BY a.connector, c.stream, a.id DESC`,
55
+ };
56
+ // How readers see the sync history: what each pass declared and loaded.
57
+ // The sync history keeps no row versions, so a reading as of a position
58
+ // fails each of its views: the scalar subquery is a one-time filter, which
59
+ // Postgres evaluates before it reads a row.
60
+ export const syncHistoryViews = Object.values(syncHistoryRelations).map((relation) => ({
61
+ ...relation,
62
+ query: `SELECT * FROM (${queries[relation.name]}) AS "history" WHERE (SELECT _warehouse._elt_unversioned())`,
63
+ }));
64
+ export const syncHistoryTables = [
65
+ 'CREATE SCHEMA IF NOT EXISTS _warehouse',
66
+ `CREATE TABLE IF NOT EXISTS _warehouse.sync_attempts (
67
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
68
+ connector text NOT NULL CHECK (btrim(connector) <> ''), source text NOT NULL,
69
+ started_at timestamptz NOT NULL,
70
+ completed_at timestamptz,
71
+ status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
72
+ error text,
73
+ failure_type text CHECK (failure_type IN ('config', 'system')),
74
+ CHECK ((status = 'running') = (completed_at IS NULL)),
75
+ CHECK ((status IN ('partial', 'failed')) = (failure_type IS NOT NULL))
76
+ )`,
77
+ `CREATE OR REPLACE FUNCTION _warehouse._elt_unversioned() RETURNS boolean LANGUAGE plpgsql STABLE AS $$
78
+ BEGIN
79
+ IF coalesce(current_setting('elt.position', true), '') <> '' THEN
80
+ RAISE EXCEPTION 'The sync history cannot be read as of a position: it keeps no row versions';
81
+ END IF;
82
+ RETURN true;
83
+ END $$`,
84
+ 'CREATE INDEX IF NOT EXISTS sync_attempts_connector ON _warehouse.sync_attempts (connector, id DESC)',
85
+ `CREATE TABLE IF NOT EXISTS _warehouse.extraction_coverage (
86
+ attempt_id bigint NOT NULL REFERENCES _warehouse.sync_attempts(id),
87
+ stream text NOT NULL, target_schema text NOT NULL, target_table text NOT NULL,
88
+ sync_mode text NOT NULL, destination_sync_mode text NOT NULL,
89
+ description text NOT NULL CHECK (btrim(description) <> ''), selection jsonb NOT NULL,
90
+ status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
91
+ written_count bigint CHECK (written_count >= 0), deleted_count bigint CHECK (deleted_count >= 0),
92
+ failures jsonb NOT NULL DEFAULT '[]', upstream_current_as_of timestamptz,
93
+ PRIMARY KEY (attempt_id, stream)
94
+ )`,
95
+ // Each stream's latest check by a watch that found it unchanged.
96
+ `CREATE TABLE IF NOT EXISTS _warehouse.stream_checks (
97
+ connector text NOT NULL, stream text NOT NULL, checked_at timestamptz NOT NULL,
98
+ PRIMARY KEY (connector, stream)
99
+ )`,
100
+ ];
@@ -0,0 +1,13 @@
1
+ import { type Connection, type DeclaredCopy, type RecordedPass, type StreamCheck, SyncHistory } from '@eltjs/elt';
2
+ import type { PostgresTable } from './postgres-table.ts';
3
+ export declare class PostgresSyncHistory extends SyncHistory<PostgresTable> {
4
+ #private;
5
+ constructor({ url, readers, }: {
6
+ url: string;
7
+ readers?: readonly string[];
8
+ });
9
+ validate(connection: Connection<PostgresTable>): void;
10
+ protected recordChecks(connection: Connection<PostgresTable>, streamChecks: readonly StreamCheck[]): Promise<void>;
11
+ begin(connection: Connection<PostgresTable>, copies: readonly DeclaredCopy<PostgresTable>[], at: Date): Promise<RecordedPass<PostgresTable>>;
12
+ install(): Promise<void>;
13
+ }
@@ -0,0 +1,152 @@
1
+ import postgres from 'postgres';
2
+ import { SyncHistory, copyStatus, errorMessage, passError, passFailureType, passStatus, readerCatalog, recordedFailure, } from '@eltjs/elt';
3
+ import { identifier } from './identifier.js';
4
+ import { PostgresDestination } from './postgres-destination.js';
5
+ import { grantReading, installNumbering } from './postgres-numbering.js';
6
+ import { lockSchema } from './postgres-session.js';
7
+ import { syncHistoryTables, syncHistoryViews, } from './postgres-sync-history-schema.js';
8
+ import { publishPostgresViews } from './postgres-views.js';
9
+ // Lists what readers of marts query, its views, from the comments every load
10
+ // and publication writes there. A table in marts, such as the raw table of a
11
+ // computed dataset, is not for readers, as in the catalog of elt-sqlite.
12
+ const catalog = {
13
+ ...readerCatalog,
14
+ query: `SELECT 'view' AS kind, c.relname::text AS name, NULL::text AS data_type,
15
+ obj_description(c.oid, 'pg_class') AS description
16
+ FROM pg_class c
17
+ WHERE c.relnamespace = 'marts'::regnamespace AND c.relkind = 'v'
18
+ UNION ALL
19
+ SELECT 'column', c.relname || '.' || a.attname,
20
+ format_type(a.atttypid, a.atttypmod), col_description(c.oid, a.attnum)
21
+ FROM pg_class c
22
+ JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped
23
+ WHERE c.relnamespace = 'marts'::regnamespace AND c.relkind = 'v'
24
+ ORDER BY 2`,
25
+ };
26
+ // Records every pass in the _warehouse schema, and shows readers that history
27
+ // and the catalog through views in marts. Who may read marts is the
28
+ // database's own grant. readers are the roles that read the database as of
29
+ // a position with their own session, such as an agent's read-only role:
30
+ // each install grants them what a reading needs, as dbt applies the grants
31
+ // a model declares after each build, so no host names the objects of the
32
+ // numbering.
33
+ export class PostgresSyncHistory extends SyncHistory {
34
+ #url;
35
+ #readers;
36
+ constructor({ url, readers = [], }) {
37
+ super();
38
+ this.#url = url;
39
+ this.#readers = Object.freeze(readers.map((role) => identifier(role, 'reader role')));
40
+ Object.freeze(this);
41
+ }
42
+ validate(connection) {
43
+ this.#schema(connection);
44
+ }
45
+ async recordChecks(connection, streamChecks) {
46
+ await this.#write(async (transaction) => {
47
+ for (const { stream, at } of streamChecks)
48
+ await transaction `
49
+ INSERT INTO _warehouse.stream_checks (connector, stream, checked_at)
50
+ VALUES (${connection.name}, ${stream}, ${at})
51
+ ON CONFLICT (connector, stream) DO UPDATE SET checked_at = excluded.checked_at`;
52
+ });
53
+ }
54
+ // Every selected stream is declared before reading, even if it produces no
55
+ // rows or the process dies: an unfinished attempt stays running, never success.
56
+ async begin(connection, copies, at) {
57
+ const schema = this.#schema(connection);
58
+ const id = await this.#write(async (transaction) => {
59
+ const [attempt] = await transaction `
60
+ INSERT INTO _warehouse.sync_attempts (connector, source, started_at)
61
+ VALUES (${connection.name}, ${connection.source.identity}, ${at}) RETURNING id`;
62
+ if (attempt === undefined)
63
+ throw new Error('Sync attempt was not recorded');
64
+ for (const { copy, coverage } of copies)
65
+ await transaction `
66
+ INSERT INTO _warehouse.extraction_coverage
67
+ (attempt_id, stream, target_schema, target_table, sync_mode,
68
+ destination_sync_mode, description, selection)
69
+ VALUES (${attempt.id}, ${copy.from.name}, ${schema},
70
+ ${copy.to.name}, ${copy.configuration.syncMode},
71
+ ${copy.configuration.destinationSyncMode}, ${coverage.description},
72
+ ${transaction.json(coverage.selection)})`;
73
+ return attempt.id;
74
+ });
75
+ return {
76
+ finish: (outcomes, upstream, ended) => this.#finish(id, outcomes, upstream, ended),
77
+ fail: (error, failureType, ended) => this.#write(async (transaction) => {
78
+ await transaction `
79
+ UPDATE _warehouse.extraction_coverage SET status = 'failed',
80
+ failures = ${transaction.json([recordedFailure({ partition: null, error, failureType })])}
81
+ WHERE attempt_id = ${id}`;
82
+ await transaction `
83
+ UPDATE _warehouse.sync_attempts SET status = 'failed',
84
+ completed_at = ${ended}, error = ${errorMessage(error)},
85
+ failure_type = ${failureType}
86
+ WHERE id = ${id}`;
87
+ }),
88
+ };
89
+ }
90
+ // Creates the history's tables, publishes its views and the catalog, and
91
+ // grants each reader the numbering; safe to repeat.
92
+ async install() {
93
+ await this.#connect(async (sql) => {
94
+ if (this.#readers.length > 0)
95
+ await installNumbering(sql);
96
+ await sql.begin(async (transaction) => {
97
+ // The numbering of the destinations makes _warehouse too, under the
98
+ // same lock.
99
+ await lockSchema(transaction, '_warehouse');
100
+ for (const statement of syncHistoryTables)
101
+ await transaction.unsafe(statement);
102
+ await publishPostgresViews(transaction, {
103
+ schema: 'marts',
104
+ views: [...syncHistoryViews, catalog],
105
+ });
106
+ for (const role of this.#readers)
107
+ await grantReading(transaction, role);
108
+ });
109
+ });
110
+ }
111
+ #schema({ name, destination }) {
112
+ if (!(destination instanceof PostgresDestination))
113
+ throw new TypeError(`Connection ${name}: Postgres sync history records Postgres destinations only`);
114
+ return destination.schema;
115
+ }
116
+ #finish(id, outcomes, upstream, ended) {
117
+ return this.#write(async (transaction) => {
118
+ for (const outcome of outcomes)
119
+ await transaction `
120
+ UPDATE _warehouse.extraction_coverage SET
121
+ status = ${copyStatus(outcome)},
122
+ written_count = ${outcome.count}, deleted_count = ${outcome.deleted},
123
+ failures = ${transaction.json(outcome.failures.map(recordedFailure))}
124
+ WHERE attempt_id = ${id} AND stream = ${outcome.copy.from.name}`;
125
+ for (const [stream, at] of upstream)
126
+ await transaction `
127
+ UPDATE _warehouse.extraction_coverage SET upstream_current_as_of = ${at}
128
+ WHERE attempt_id = ${id} AND stream = ${stream}`;
129
+ await transaction `
130
+ UPDATE _warehouse.sync_attempts SET completed_at = ${ended},
131
+ status = ${passStatus(outcomes)}, error = ${passError(outcomes)},
132
+ failure_type = ${passFailureType(outcomes)}
133
+ WHERE id = ${id}`;
134
+ });
135
+ }
136
+ #write(work) {
137
+ return this.#connect((sql) => sql.begin(work));
138
+ }
139
+ async #connect(work) {
140
+ const sql = postgres(this.#url, {
141
+ max: 1,
142
+ onnotice: () => { },
143
+ connection: { application_name: 'elt-sync-history' },
144
+ });
145
+ try {
146
+ return await work(sql);
147
+ }
148
+ finally {
149
+ await sql.end();
150
+ }
151
+ }
152
+ }
@@ -0,0 +1,15 @@
1
+ import { type Stream, Target } from '@eltjs/elt';
2
+ import { PostgresColumn } from './postgres-column.ts';
3
+ export type PostgresReaderView = {
4
+ readonly schema: string;
5
+ readonly name: string;
6
+ };
7
+ export declare class PostgresTable extends Target {
8
+ readonly name: string;
9
+ readonly columns: readonly PostgresColumn[];
10
+ readonly readerView: PostgresReaderView | undefined;
11
+ constructor(name: string, columns?: readonly PostgresColumn[], readerView?: PostgresReaderView);
12
+ withReaderView(schema: string, name: string): PostgresTable;
13
+ resolve(stream: Stream): PostgresTable;
14
+ get quotedName(): string;
15
+ }
@@ -0,0 +1,73 @@
1
+ import { Target } from '@eltjs/elt';
2
+ import { quote } from './identifier.js';
3
+ import { PostgresColumn } from './postgres-column.js';
4
+ import { PostgresColumns } from './postgres-columns.js';
5
+ import { identifiers } from './postgres-identifiers.js';
6
+ import { schemaName } from './postgres-session.js';
7
+ // A reusable target definition. Empty columns mean infer from the Copy's
8
+ // stream. Its own name, its view's and its columns' follow the destination's
9
+ // naming rule, so a field Postgres cannot hold as it is still loads.
10
+ export class PostgresTable extends Target {
11
+ name;
12
+ columns;
13
+ readerView;
14
+ constructor(name, columns, readerView) {
15
+ if (typeof name !== 'string' ||
16
+ (readerView !== undefined && typeof readerView.name !== 'string'))
17
+ throw new TypeError('A table and its view are named by strings');
18
+ if (readerView !== undefined)
19
+ schemaName(readerView.schema);
20
+ if (columns !== undefined &&
21
+ (!Array.isArray(columns) ||
22
+ columns.length === 0 ||
23
+ !columns.every((column) => column instanceof PostgresColumn)))
24
+ throw new TypeError('A table requires at least one Postgres column');
25
+ const fields = (columns ?? []).map((column) => column.field);
26
+ if (new Set(fields).size !== fields.length)
27
+ throw new TypeError('Two columns hold one field');
28
+ if ((columns ?? []).filter((column) => column.isPrimaryKey).length > 1)
29
+ throw new TypeError('Only one primary-key column is supported');
30
+ const fileReads = (columns ?? []).flatMap((column) => column.fileRead === undefined ? [] : [column.fileRead]);
31
+ for (const column of columns ?? []) {
32
+ if (column.fileRead === undefined)
33
+ continue;
34
+ if (column.fileRead.outputType === 'bytes'
35
+ ? column.kind !== 'blob'
36
+ : column.kind !== 'text')
37
+ throw new TypeError('Original files require a BLOB column; parsed text and stored references require a TEXT column');
38
+ }
39
+ super(fileReads);
40
+ this.name = identifiers.relation(name);
41
+ this.columns = Object.freeze(identifiers
42
+ .columns(columns ?? [])
43
+ .map(([column, stored]) => column.named(stored)));
44
+ this.readerView =
45
+ readerView &&
46
+ Object.freeze({
47
+ schema: readerView.schema,
48
+ name: identifiers.relation(readerView.name),
49
+ });
50
+ Object.freeze(this);
51
+ }
52
+ // Loads also keep a view of this table in another schema, created with the
53
+ // table and described by the same comments, which every column then needs.
54
+ withReaderView(schema, name) {
55
+ return new PostgresTable(this.name, this.columns.length === 0 ? undefined : this.columns, { schema, name });
56
+ }
57
+ resolve(stream) {
58
+ if (this.columns.length === 0)
59
+ return new PostgresTable(this.name, PostgresColumns.fromSchema(stream.jsonSchema), this.readerView);
60
+ const { properties } = stream.jsonSchema;
61
+ // Explicit columns of a stream that declares no properties stand alone.
62
+ if (properties === undefined)
63
+ return this;
64
+ for (const column of this.columns)
65
+ if (column.fileRead === undefined &&
66
+ !Object.hasOwn(properties, column.field))
67
+ throw new TypeError(`Stream ${stream.name} does not describe field ${column.field}`);
68
+ return this;
69
+ }
70
+ get quotedName() {
71
+ return quote(this.name);
72
+ }
73
+ }
@@ -0,0 +1,10 @@
1
+ import postgres from 'postgres';
2
+ import type { ReaderRelation } from '@eltjs/elt';
3
+ export type PostgresView = ReaderRelation & {
4
+ readonly query: string;
5
+ };
6
+ export declare function checkView(view: PostgresView): void;
7
+ export declare function publishPostgresViews(sql: postgres.TransactionSql, { schema, views }: {
8
+ schema: string;
9
+ views: readonly PostgresView[];
10
+ }): Promise<void>;
@@ -0,0 +1,119 @@
1
+ import postgres from 'postgres';
2
+ import { identifier, quote } from './identifier.js';
3
+ import { rebindDependents } from './postgres-dependents.js';
4
+ import { lockSchema, schemaName } from './postgres-session.js';
5
+ function text(value, what) {
6
+ if (typeof value !== 'string' ||
7
+ !value.trim() ||
8
+ value.includes('\0') ||
9
+ !value.isWellFormed())
10
+ throw new TypeError(`Invalid ${what}`);
11
+ }
12
+ // Refuses, with no I/O, a view that Postgres cannot hold as it is declared.
13
+ export function checkView(view) {
14
+ identifier(view.name, 'view name');
15
+ text(view.query, 'view query');
16
+ text(view.description, 'view description');
17
+ for (const [column, description] of Object.entries(view.columns)) {
18
+ identifier(column, 'column name');
19
+ text(description, 'column description');
20
+ }
21
+ }
22
+ // Publishes ordinary views and their descriptions in a caller-owned
23
+ // transaction, in schema, where the bare names of their queries are found
24
+ // too. Supply trusted SQL, and list dependencies before dependents. A view
25
+ // whose stored definition is the one its query makes stays as it is, with
26
+ // its OID, so a publication that changes nothing changes no view; the others
27
+ // are made again, and the views that read them read the new ones (ADR 0006).
28
+ export async function publishPostgresViews(sql, { schema, views }) {
29
+ schemaName(schema);
30
+ const names = new Set();
31
+ for (const view of views) {
32
+ checkView(view);
33
+ if (names.has(view.name))
34
+ throw new TypeError('Duplicate view names');
35
+ names.add(view.name);
36
+ }
37
+ if (views.length === 0)
38
+ return;
39
+ // A failed definition restores every replaced view, even if the caller
40
+ // catches the error and continues its transaction.
41
+ await sql.savepoint(async (transaction) => {
42
+ await lockSchema(transaction, schema);
43
+ await transaction.unsafe(`CREATE SCHEMA IF NOT EXISTS ${quote(schema)}`);
44
+ const [{ path } = { path: '' }] = await transaction `
45
+ SELECT current_setting('search_path') AS path`;
46
+ await transaction `SELECT set_config('search_path', ${quote(schema)}, true)`;
47
+ const changed = [];
48
+ for (const view of views)
49
+ if (!(await stored(transaction, schema, view)))
50
+ changed.push(view);
51
+ if (changed.length > 0)
52
+ await rebindDependents(transaction, changed.map(({ name }) => `${quote(schema)}.${quote(name)}`), async () => {
53
+ for (const view of changed.toReversed())
54
+ await transaction.unsafe(`DROP VIEW IF EXISTS ${quote(schema)}.${quote(view.name)}`);
55
+ for (const view of changed) {
56
+ const relation = `${quote(schema)}.${quote(view.name)}`;
57
+ // Extended protocol accepts one statement, so a query cannot
58
+ // append DDL.
59
+ await transaction `CREATE VIEW ${transaction.unsafe(relation)} AS ${transaction.unsafe(view.query)}`;
60
+ const columns = await transaction `
61
+ SELECT attname AS name FROM pg_attribute
62
+ WHERE attrelid = to_regclass(${relation})
63
+ AND attnum > 0 AND NOT attisdropped ORDER BY attnum`;
64
+ if (columns.length !== Object.keys(view.columns).length ||
65
+ columns.some(({ name }) => !Object.hasOwn(view.columns, name)))
66
+ throw new TypeError(`View ${relation} must describe exactly its output columns: ${columns.map(({ name }) => name).join(', ')}`);
67
+ }
68
+ });
69
+ for (const view of views)
70
+ await describe(transaction, schema, view);
71
+ await transaction `SELECT set_config('search_path', ${path}, true)`;
72
+ });
73
+ }
74
+ // Whether the view in schema has the definition that its query makes now. The
75
+ // query is made as a TEMP view, which the savepoint takes back, because
76
+ // Postgres keeps a view's definition as it normalizes it, not as its text. A
77
+ // query that Postgres refuses counts as changed, so the publication fails with
78
+ // its error.
79
+ async function stored(sql, schema, view) {
80
+ const relation = `${quote(schema)}.${quote(view.name)}`;
81
+ const [existing] = await sql `
82
+ SELECT CASE WHEN c.relkind = 'v' THEN pg_get_viewdef(c.oid) END AS definition
83
+ FROM pg_class c WHERE c.oid = to_regclass(${relation})`;
84
+ if (existing?.definition == null)
85
+ return false;
86
+ await sql.unsafe('SAVEPOINT "_elt_candidate"');
87
+ try {
88
+ await sql `CREATE TEMP VIEW "_elt_candidate" AS ${sql.unsafe(view.query)}`;
89
+ const [made] = await sql `
90
+ SELECT pg_get_viewdef('pg_temp."_elt_candidate"'::regclass) AS definition`;
91
+ return made?.definition === existing.definition;
92
+ }
93
+ catch (error) {
94
+ if (error instanceof postgres.PostgresError)
95
+ return false;
96
+ throw error;
97
+ }
98
+ finally {
99
+ await sql.unsafe('ROLLBACK TO SAVEPOINT "_elt_candidate"');
100
+ await sql.unsafe('RELEASE SAVEPOINT "_elt_candidate"');
101
+ }
102
+ }
103
+ // COMMENT cannot bind values; Postgres quotes names and text itself.
104
+ async function describe(sql, schema, view) {
105
+ const comments = await sql `
106
+ SELECT format('COMMENT ON VIEW %I.%I IS %L', ${schema}::text,
107
+ ${view.name}::text, ${view.description}::text) AS statement
108
+ WHERE obj_description(to_regclass(format('%I.%I', ${schema}::text, ${view.name}::text)), 'pg_class')
109
+ IS DISTINCT FROM ${view.description}::text
110
+ UNION ALL
111
+ SELECT format('COMMENT ON COLUMN %I.%I.%I IS %L', ${schema}::text,
112
+ ${view.name}::text, c.key, c.value)
113
+ FROM jsonb_each_text(${sql.json(view.columns)}::jsonb) AS c
114
+ JOIN pg_attribute a ON a.attrelid = to_regclass(format('%I.%I', ${schema}::text, ${view.name}::text))
115
+ AND a.attname = c.key
116
+ WHERE col_description(a.attrelid, a.attnum) IS DISTINCT FROM c.value`;
117
+ for (const { statement } of comments)
118
+ await sql.unsafe(statement);
119
+ }
@@ -0,0 +1,39 @@
1
+ import { type Clearing, type CopyConfiguration, type FieldValues, type KeyValue, type Stage, Writer } from '@eltjs/elt';
2
+ import type { EncodedValue } from './postgres-column.ts';
3
+ import { TargetHistory } from './postgres-history.ts';
4
+ import type { PostgresLoad, Transaction } from './postgres-load.ts';
5
+ import type { PostgresTable } from './postgres-table.ts';
6
+ export declare const seq = "\"_elt_seq\"";
7
+ export declare const op = "\"_elt_op\"";
8
+ export declare abstract class PostgresWriter extends Writer {
9
+ #private;
10
+ readonly configuration: CopyConfiguration;
11
+ protected readonly url: string;
12
+ readonly schema: string;
13
+ readonly table: PostgresTable;
14
+ constructor(configuration: CopyConfiguration, url: string, schema: string, table: PostgresTable, retention: Temporal.Duration);
15
+ protected abstract initialize(transaction: Transaction, replacing: boolean): Promise<void>;
16
+ protected inspect(_sql: Transaction): Promise<void>;
17
+ get qualifiedName(): string;
18
+ protected get createTableSQL(): string;
19
+ protected build(_sql: Transaction, _into: string): Promise<void>;
20
+ protected adopt(_sql: Transaction): Promise<void>;
21
+ protected get written(): string[];
22
+ protected get staged(): string;
23
+ protected encode(record: unknown): EncodedValue[];
24
+ protected deletionRow(_key: Readonly<Record<string, KeyValue>>): EncodedValue[];
25
+ protected get replaces(): boolean;
26
+ protected kept(_stage: string): string;
27
+ protected merge(sql: Transaction, stage: string, written: Written, into: string, _history: TargetHistory | null): Promise<void>;
28
+ protected dedupIndexes(transaction: Transaction): Promise<string[]>;
29
+ private values;
30
+ clear({ writer, clock, connection: connector }: Clearing, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
31
+ prepare(load: PostgresLoad, { writer, reloading }: {
32
+ writer: string;
33
+ reloading: boolean;
34
+ }): Promise<Stage>;
35
+ }
36
+ export type Written = {
37
+ readonly loadedAt: string;
38
+ readonly committedIn: number;
39
+ };