@eltjs/elt-postgresql 0.0.0-stage → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +668 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { syncHistoryRelations } from '@eltjs/elt';
|
|
2
|
+
// When connector's stream last matched its upstream: the start of its last
|
|
3
|
+
// successful pass, or a later check that found it unchanged, each no later
|
|
4
|
+
// than when the source said the upstream itself was last current. NULL if it
|
|
5
|
+
// never succeeded. LEAST and GREATEST skip NULLs.
|
|
6
|
+
const currentAsOf = (connector, stream) => `(
|
|
7
|
+
SELECT CASE WHEN success.id IS NULL THEN NULL ELSE GREATEST(LEAST(success.started_at, success.upstream_current_as_of), k.checked_at) END
|
|
8
|
+
FROM (SELECT 1) one
|
|
9
|
+
LEFT JOIN LATERAL (
|
|
10
|
+
SELECT s.id, s.started_at, sc.upstream_current_as_of FROM _warehouse.extraction_coverage sc
|
|
11
|
+
JOIN _warehouse.sync_attempts s ON s.id = sc.attempt_id
|
|
12
|
+
WHERE s.connector = ${connector} AND sc.stream = ${stream} AND sc.status = 'succeeded'
|
|
13
|
+
ORDER BY s.completed_at DESC, s.id DESC LIMIT 1
|
|
14
|
+
) success ON true
|
|
15
|
+
LEFT JOIN _warehouse.stream_checks k ON k.connector = ${connector} AND k.stream = ${stream}
|
|
16
|
+
)`;
|
|
17
|
+
const queries = {
|
|
18
|
+
sync_attempts: 'SELECT id AS attempt_id, connector, source, started_at, completed_at, status, error, failure_type FROM _warehouse.sync_attempts',
|
|
19
|
+
extraction_coverage: `SELECT c.attempt_id, a.connector, a.source, a.started_at, a.completed_at,
|
|
20
|
+
c.stream, c.target_schema, c.target_table,
|
|
21
|
+
EXISTS (SELECT FROM pg_class t JOIN pg_namespace n ON n.oid = t.relnamespace
|
|
22
|
+
WHERE n.nspname = c.target_schema AND t.relname = c.target_table AND t.relkind = 'r') AS target_exists,
|
|
23
|
+
c.sync_mode, c.destination_sync_mode, c.description, c.selection,
|
|
24
|
+
c.status, c.written_count, c.deleted_count, c.failures
|
|
25
|
+
FROM _warehouse.extraction_coverage c JOIN _warehouse.sync_attempts a ON a.id = c.attempt_id`,
|
|
26
|
+
sync_status: `SELECT DISTINCT ON (a.connector) a.connector,
|
|
27
|
+
a.id AS latest_attempt_id, a.started_at, a.completed_at, a.status, a.error, a.failure_type,
|
|
28
|
+
success.id AS last_successful_attempt_id, success.completed_at AS last_successful_sync_at,
|
|
29
|
+
(SELECT CASE WHEN count(*) = count(current_as_of) THEN min(current_as_of) END
|
|
30
|
+
FROM (SELECT ${currentAsOf('a.connector', 'd.stream')} AS current_as_of
|
|
31
|
+
FROM (SELECT DISTINCT dc.stream FROM _warehouse.extraction_coverage dc
|
|
32
|
+
JOIN _warehouse.sync_attempts da ON da.id = dc.attempt_id
|
|
33
|
+
WHERE da.connector = a.connector) d) streams) AS current_as_of
|
|
34
|
+
FROM _warehouse.sync_attempts a
|
|
35
|
+
LEFT JOIN LATERAL (
|
|
36
|
+
SELECT id, completed_at FROM _warehouse.sync_attempts
|
|
37
|
+
WHERE connector = a.connector AND status = 'succeeded'
|
|
38
|
+
ORDER BY completed_at DESC, id DESC LIMIT 1
|
|
39
|
+
) success ON true
|
|
40
|
+
ORDER BY a.connector, a.id DESC`,
|
|
41
|
+
stream_status: `SELECT DISTINCT ON (a.connector, c.stream) a.connector, c.stream,
|
|
42
|
+
c.target_schema, c.target_table,
|
|
43
|
+
a.id AS latest_attempt_id, a.started_at, a.completed_at, c.status,
|
|
44
|
+
success.id AS last_successful_attempt_id, success.completed_at AS last_successful_sync_at,
|
|
45
|
+
${currentAsOf('a.connector', 'c.stream')} AS current_as_of
|
|
46
|
+
FROM _warehouse.extraction_coverage c
|
|
47
|
+
JOIN _warehouse.sync_attempts a ON a.id = c.attempt_id
|
|
48
|
+
LEFT JOIN LATERAL (
|
|
49
|
+
SELECT s.id, s.completed_at FROM _warehouse.extraction_coverage sc
|
|
50
|
+
JOIN _warehouse.sync_attempts s ON s.id = sc.attempt_id
|
|
51
|
+
WHERE s.connector = a.connector AND sc.stream = c.stream AND sc.status = 'succeeded'
|
|
52
|
+
ORDER BY s.completed_at DESC, s.id DESC LIMIT 1
|
|
53
|
+
) success ON true
|
|
54
|
+
ORDER BY a.connector, c.stream, a.id DESC`,
|
|
55
|
+
};
|
|
56
|
+
// How readers see the sync history: what each pass declared and loaded.
|
|
57
|
+
// The sync history keeps no row versions, so a reading as of a position
|
|
58
|
+
// fails each of its views: the scalar subquery is a one-time filter, which
|
|
59
|
+
// Postgres evaluates before it reads a row.
|
|
60
|
+
export const syncHistoryViews = Object.values(syncHistoryRelations).map((relation) => ({
|
|
61
|
+
...relation,
|
|
62
|
+
query: `SELECT * FROM (${queries[relation.name]}) AS "history" WHERE (SELECT _warehouse._elt_unversioned())`,
|
|
63
|
+
}));
|
|
64
|
+
export const syncHistoryTables = [
|
|
65
|
+
'CREATE SCHEMA IF NOT EXISTS _warehouse',
|
|
66
|
+
`CREATE TABLE IF NOT EXISTS _warehouse.sync_attempts (
|
|
67
|
+
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
68
|
+
connector text NOT NULL CHECK (btrim(connector) <> ''), source text NOT NULL,
|
|
69
|
+
started_at timestamptz NOT NULL,
|
|
70
|
+
completed_at timestamptz,
|
|
71
|
+
status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
|
|
72
|
+
error text,
|
|
73
|
+
failure_type text CHECK (failure_type IN ('config', 'system')),
|
|
74
|
+
CHECK ((status = 'running') = (completed_at IS NULL)),
|
|
75
|
+
CHECK ((status IN ('partial', 'failed')) = (failure_type IS NOT NULL))
|
|
76
|
+
)`,
|
|
77
|
+
`CREATE OR REPLACE FUNCTION _warehouse._elt_unversioned() RETURNS boolean LANGUAGE plpgsql STABLE AS $$
|
|
78
|
+
BEGIN
|
|
79
|
+
IF coalesce(current_setting('elt.position', true), '') <> '' THEN
|
|
80
|
+
RAISE EXCEPTION 'The sync history cannot be read as of a position: it keeps no row versions';
|
|
81
|
+
END IF;
|
|
82
|
+
RETURN true;
|
|
83
|
+
END $$`,
|
|
84
|
+
'CREATE INDEX IF NOT EXISTS sync_attempts_connector ON _warehouse.sync_attempts (connector, id DESC)',
|
|
85
|
+
`CREATE TABLE IF NOT EXISTS _warehouse.extraction_coverage (
|
|
86
|
+
attempt_id bigint NOT NULL REFERENCES _warehouse.sync_attempts(id),
|
|
87
|
+
stream text NOT NULL, target_schema text NOT NULL, target_table text NOT NULL,
|
|
88
|
+
sync_mode text NOT NULL, destination_sync_mode text NOT NULL,
|
|
89
|
+
description text NOT NULL CHECK (btrim(description) <> ''), selection jsonb NOT NULL,
|
|
90
|
+
status text NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'partial', 'failed', 'cancelled')),
|
|
91
|
+
written_count bigint CHECK (written_count >= 0), deleted_count bigint CHECK (deleted_count >= 0),
|
|
92
|
+
failures jsonb NOT NULL DEFAULT '[]', upstream_current_as_of timestamptz,
|
|
93
|
+
PRIMARY KEY (attempt_id, stream)
|
|
94
|
+
)`,
|
|
95
|
+
// Each stream's latest check by a watch that found it unchanged.
|
|
96
|
+
`CREATE TABLE IF NOT EXISTS _warehouse.stream_checks (
|
|
97
|
+
connector text NOT NULL, stream text NOT NULL, checked_at timestamptz NOT NULL,
|
|
98
|
+
PRIMARY KEY (connector, stream)
|
|
99
|
+
)`,
|
|
100
|
+
];
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type Connection, type DeclaredCopy, type RecordedPass, type StreamCheck, SyncHistory } from '@eltjs/elt';
|
|
2
|
+
import type { PostgresTable } from './postgres-table.ts';
|
|
3
|
+
export declare class PostgresSyncHistory extends SyncHistory<PostgresTable> {
|
|
4
|
+
#private;
|
|
5
|
+
constructor({ url, readers, }: {
|
|
6
|
+
url: string;
|
|
7
|
+
readers?: readonly string[];
|
|
8
|
+
});
|
|
9
|
+
validate(connection: Connection<PostgresTable>): void;
|
|
10
|
+
protected recordChecks(connection: Connection<PostgresTable>, streamChecks: readonly StreamCheck[]): Promise<void>;
|
|
11
|
+
begin(connection: Connection<PostgresTable>, copies: readonly DeclaredCopy<PostgresTable>[], at: Date): Promise<RecordedPass<PostgresTable>>;
|
|
12
|
+
install(): Promise<void>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import postgres from 'postgres';
|
|
2
|
+
import { SyncHistory, copyStatus, errorMessage, passError, passFailureType, passStatus, readerCatalog, recordedFailure, } from '@eltjs/elt';
|
|
3
|
+
import { identifier } from './identifier.js';
|
|
4
|
+
import { PostgresDestination } from './postgres-destination.js';
|
|
5
|
+
import { grantReading, installNumbering } from './postgres-numbering.js';
|
|
6
|
+
import { lockSchema } from './postgres-session.js';
|
|
7
|
+
import { syncHistoryTables, syncHistoryViews, } from './postgres-sync-history-schema.js';
|
|
8
|
+
import { publishPostgresViews } from './postgres-views.js';
|
|
9
|
+
// Lists what readers of marts query, its views, from the comments every load
|
|
10
|
+
// and publication writes there. A table in marts, such as the raw table of a
|
|
11
|
+
// computed dataset, is not for readers, as in the catalog of elt-sqlite.
|
|
12
|
+
const catalog = {
|
|
13
|
+
...readerCatalog,
|
|
14
|
+
query: `SELECT 'view' AS kind, c.relname::text AS name, NULL::text AS data_type,
|
|
15
|
+
obj_description(c.oid, 'pg_class') AS description
|
|
16
|
+
FROM pg_class c
|
|
17
|
+
WHERE c.relnamespace = 'marts'::regnamespace AND c.relkind = 'v'
|
|
18
|
+
UNION ALL
|
|
19
|
+
SELECT 'column', c.relname || '.' || a.attname,
|
|
20
|
+
format_type(a.atttypid, a.atttypmod), col_description(c.oid, a.attnum)
|
|
21
|
+
FROM pg_class c
|
|
22
|
+
JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped
|
|
23
|
+
WHERE c.relnamespace = 'marts'::regnamespace AND c.relkind = 'v'
|
|
24
|
+
ORDER BY 2`,
|
|
25
|
+
};
|
|
26
|
+
// Records every pass in the _warehouse schema, and shows readers that history
|
|
27
|
+
// and the catalog through views in marts. Who may read marts is the
|
|
28
|
+
// database's own grant. readers are the roles that read the database as of
|
|
29
|
+
// a position with their own session, such as an agent's read-only role:
|
|
30
|
+
// each install grants them what a reading needs, as dbt applies the grants
|
|
31
|
+
// a model declares after each build, so no host names the objects of the
|
|
32
|
+
// numbering.
|
|
33
|
+
export class PostgresSyncHistory extends SyncHistory {
|
|
34
|
+
#url;
|
|
35
|
+
#readers;
|
|
36
|
+
constructor({ url, readers = [], }) {
|
|
37
|
+
super();
|
|
38
|
+
this.#url = url;
|
|
39
|
+
this.#readers = Object.freeze(readers.map((role) => identifier(role, 'reader role')));
|
|
40
|
+
Object.freeze(this);
|
|
41
|
+
}
|
|
42
|
+
validate(connection) {
|
|
43
|
+
this.#schema(connection);
|
|
44
|
+
}
|
|
45
|
+
async recordChecks(connection, streamChecks) {
|
|
46
|
+
await this.#write(async (transaction) => {
|
|
47
|
+
for (const { stream, at } of streamChecks)
|
|
48
|
+
await transaction `
|
|
49
|
+
INSERT INTO _warehouse.stream_checks (connector, stream, checked_at)
|
|
50
|
+
VALUES (${connection.name}, ${stream}, ${at})
|
|
51
|
+
ON CONFLICT (connector, stream) DO UPDATE SET checked_at = excluded.checked_at`;
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
// Every selected stream is declared before reading, even if it produces no
|
|
55
|
+
// rows or the process dies: an unfinished attempt stays running, never success.
|
|
56
|
+
async begin(connection, copies, at) {
|
|
57
|
+
const schema = this.#schema(connection);
|
|
58
|
+
const id = await this.#write(async (transaction) => {
|
|
59
|
+
const [attempt] = await transaction `
|
|
60
|
+
INSERT INTO _warehouse.sync_attempts (connector, source, started_at)
|
|
61
|
+
VALUES (${connection.name}, ${connection.source.identity}, ${at}) RETURNING id`;
|
|
62
|
+
if (attempt === undefined)
|
|
63
|
+
throw new Error('Sync attempt was not recorded');
|
|
64
|
+
for (const { copy, coverage } of copies)
|
|
65
|
+
await transaction `
|
|
66
|
+
INSERT INTO _warehouse.extraction_coverage
|
|
67
|
+
(attempt_id, stream, target_schema, target_table, sync_mode,
|
|
68
|
+
destination_sync_mode, description, selection)
|
|
69
|
+
VALUES (${attempt.id}, ${copy.from.name}, ${schema},
|
|
70
|
+
${copy.to.name}, ${copy.configuration.syncMode},
|
|
71
|
+
${copy.configuration.destinationSyncMode}, ${coverage.description},
|
|
72
|
+
${transaction.json(coverage.selection)})`;
|
|
73
|
+
return attempt.id;
|
|
74
|
+
});
|
|
75
|
+
return {
|
|
76
|
+
finish: (outcomes, upstream, ended) => this.#finish(id, outcomes, upstream, ended),
|
|
77
|
+
fail: (error, failureType, ended) => this.#write(async (transaction) => {
|
|
78
|
+
await transaction `
|
|
79
|
+
UPDATE _warehouse.extraction_coverage SET status = 'failed',
|
|
80
|
+
failures = ${transaction.json([recordedFailure({ partition: null, error, failureType })])}
|
|
81
|
+
WHERE attempt_id = ${id}`;
|
|
82
|
+
await transaction `
|
|
83
|
+
UPDATE _warehouse.sync_attempts SET status = 'failed',
|
|
84
|
+
completed_at = ${ended}, error = ${errorMessage(error)},
|
|
85
|
+
failure_type = ${failureType}
|
|
86
|
+
WHERE id = ${id}`;
|
|
87
|
+
}),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
// Creates the history's tables, publishes its views and the catalog, and
|
|
91
|
+
// grants each reader the numbering; safe to repeat.
|
|
92
|
+
async install() {
|
|
93
|
+
await this.#connect(async (sql) => {
|
|
94
|
+
if (this.#readers.length > 0)
|
|
95
|
+
await installNumbering(sql);
|
|
96
|
+
await sql.begin(async (transaction) => {
|
|
97
|
+
// The numbering of the destinations makes _warehouse too, under the
|
|
98
|
+
// same lock.
|
|
99
|
+
await lockSchema(transaction, '_warehouse');
|
|
100
|
+
for (const statement of syncHistoryTables)
|
|
101
|
+
await transaction.unsafe(statement);
|
|
102
|
+
await publishPostgresViews(transaction, {
|
|
103
|
+
schema: 'marts',
|
|
104
|
+
views: [...syncHistoryViews, catalog],
|
|
105
|
+
});
|
|
106
|
+
for (const role of this.#readers)
|
|
107
|
+
await grantReading(transaction, role);
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
#schema({ name, destination }) {
|
|
112
|
+
if (!(destination instanceof PostgresDestination))
|
|
113
|
+
throw new TypeError(`Connection ${name}: Postgres sync history records Postgres destinations only`);
|
|
114
|
+
return destination.schema;
|
|
115
|
+
}
|
|
116
|
+
#finish(id, outcomes, upstream, ended) {
|
|
117
|
+
return this.#write(async (transaction) => {
|
|
118
|
+
for (const outcome of outcomes)
|
|
119
|
+
await transaction `
|
|
120
|
+
UPDATE _warehouse.extraction_coverage SET
|
|
121
|
+
status = ${copyStatus(outcome)},
|
|
122
|
+
written_count = ${outcome.count}, deleted_count = ${outcome.deleted},
|
|
123
|
+
failures = ${transaction.json(outcome.failures.map(recordedFailure))}
|
|
124
|
+
WHERE attempt_id = ${id} AND stream = ${outcome.copy.from.name}`;
|
|
125
|
+
for (const [stream, at] of upstream)
|
|
126
|
+
await transaction `
|
|
127
|
+
UPDATE _warehouse.extraction_coverage SET upstream_current_as_of = ${at}
|
|
128
|
+
WHERE attempt_id = ${id} AND stream = ${stream}`;
|
|
129
|
+
await transaction `
|
|
130
|
+
UPDATE _warehouse.sync_attempts SET completed_at = ${ended},
|
|
131
|
+
status = ${passStatus(outcomes)}, error = ${passError(outcomes)},
|
|
132
|
+
failure_type = ${passFailureType(outcomes)}
|
|
133
|
+
WHERE id = ${id}`;
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
#write(work) {
|
|
137
|
+
return this.#connect((sql) => sql.begin(work));
|
|
138
|
+
}
|
|
139
|
+
async #connect(work) {
|
|
140
|
+
const sql = postgres(this.#url, {
|
|
141
|
+
max: 1,
|
|
142
|
+
onnotice: () => { },
|
|
143
|
+
connection: { application_name: 'elt-sync-history' },
|
|
144
|
+
});
|
|
145
|
+
try {
|
|
146
|
+
return await work(sql);
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
await sql.end();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type Stream, Target } from '@eltjs/elt';
|
|
2
|
+
import { PostgresColumn } from './postgres-column.ts';
|
|
3
|
+
export type PostgresReaderView = {
|
|
4
|
+
readonly schema: string;
|
|
5
|
+
readonly name: string;
|
|
6
|
+
};
|
|
7
|
+
export declare class PostgresTable extends Target {
|
|
8
|
+
readonly name: string;
|
|
9
|
+
readonly columns: readonly PostgresColumn[];
|
|
10
|
+
readonly readerView: PostgresReaderView | undefined;
|
|
11
|
+
constructor(name: string, columns?: readonly PostgresColumn[], readerView?: PostgresReaderView);
|
|
12
|
+
withReaderView(schema: string, name: string): PostgresTable;
|
|
13
|
+
resolve(stream: Stream): PostgresTable;
|
|
14
|
+
get quotedName(): string;
|
|
15
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { Target } from '@eltjs/elt';
|
|
2
|
+
import { quote } from './identifier.js';
|
|
3
|
+
import { PostgresColumn } from './postgres-column.js';
|
|
4
|
+
import { PostgresColumns } from './postgres-columns.js';
|
|
5
|
+
import { identifiers } from './postgres-identifiers.js';
|
|
6
|
+
import { schemaName } from './postgres-session.js';
|
|
7
|
+
// A reusable target definition. Empty columns mean infer from the Copy's
|
|
8
|
+
// stream. Its own name, its view's and its columns' follow the destination's
|
|
9
|
+
// naming rule, so a field Postgres cannot hold as it is still loads.
|
|
10
|
+
export class PostgresTable extends Target {
|
|
11
|
+
name;
|
|
12
|
+
columns;
|
|
13
|
+
readerView;
|
|
14
|
+
constructor(name, columns, readerView) {
|
|
15
|
+
if (typeof name !== 'string' ||
|
|
16
|
+
(readerView !== undefined && typeof readerView.name !== 'string'))
|
|
17
|
+
throw new TypeError('A table and its view are named by strings');
|
|
18
|
+
if (readerView !== undefined)
|
|
19
|
+
schemaName(readerView.schema);
|
|
20
|
+
if (columns !== undefined &&
|
|
21
|
+
(!Array.isArray(columns) ||
|
|
22
|
+
columns.length === 0 ||
|
|
23
|
+
!columns.every((column) => column instanceof PostgresColumn)))
|
|
24
|
+
throw new TypeError('A table requires at least one Postgres column');
|
|
25
|
+
const fields = (columns ?? []).map((column) => column.field);
|
|
26
|
+
if (new Set(fields).size !== fields.length)
|
|
27
|
+
throw new TypeError('Two columns hold one field');
|
|
28
|
+
if ((columns ?? []).filter((column) => column.isPrimaryKey).length > 1)
|
|
29
|
+
throw new TypeError('Only one primary-key column is supported');
|
|
30
|
+
const fileReads = (columns ?? []).flatMap((column) => column.fileRead === undefined ? [] : [column.fileRead]);
|
|
31
|
+
for (const column of columns ?? []) {
|
|
32
|
+
if (column.fileRead === undefined)
|
|
33
|
+
continue;
|
|
34
|
+
if (column.fileRead.outputType === 'bytes'
|
|
35
|
+
? column.kind !== 'blob'
|
|
36
|
+
: column.kind !== 'text')
|
|
37
|
+
throw new TypeError('Original files require a BLOB column; parsed text and stored references require a TEXT column');
|
|
38
|
+
}
|
|
39
|
+
super(fileReads);
|
|
40
|
+
this.name = identifiers.relation(name);
|
|
41
|
+
this.columns = Object.freeze(identifiers
|
|
42
|
+
.columns(columns ?? [])
|
|
43
|
+
.map(([column, stored]) => column.named(stored)));
|
|
44
|
+
this.readerView =
|
|
45
|
+
readerView &&
|
|
46
|
+
Object.freeze({
|
|
47
|
+
schema: readerView.schema,
|
|
48
|
+
name: identifiers.relation(readerView.name),
|
|
49
|
+
});
|
|
50
|
+
Object.freeze(this);
|
|
51
|
+
}
|
|
52
|
+
// Loads also keep a view of this table in another schema, created with the
|
|
53
|
+
// table and described by the same comments, which every column then needs.
|
|
54
|
+
withReaderView(schema, name) {
|
|
55
|
+
return new PostgresTable(this.name, this.columns.length === 0 ? undefined : this.columns, { schema, name });
|
|
56
|
+
}
|
|
57
|
+
resolve(stream) {
|
|
58
|
+
if (this.columns.length === 0)
|
|
59
|
+
return new PostgresTable(this.name, PostgresColumns.fromSchema(stream.jsonSchema), this.readerView);
|
|
60
|
+
const { properties } = stream.jsonSchema;
|
|
61
|
+
// Explicit columns of a stream that declares no properties stand alone.
|
|
62
|
+
if (properties === undefined)
|
|
63
|
+
return this;
|
|
64
|
+
for (const column of this.columns)
|
|
65
|
+
if (column.fileRead === undefined &&
|
|
66
|
+
!Object.hasOwn(properties, column.field))
|
|
67
|
+
throw new TypeError(`Stream ${stream.name} does not describe field ${column.field}`);
|
|
68
|
+
return this;
|
|
69
|
+
}
|
|
70
|
+
get quotedName() {
|
|
71
|
+
return quote(this.name);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import postgres from 'postgres';
|
|
2
|
+
import type { ReaderRelation } from '@eltjs/elt';
|
|
3
|
+
export type PostgresView = ReaderRelation & {
|
|
4
|
+
readonly query: string;
|
|
5
|
+
};
|
|
6
|
+
export declare function checkView(view: PostgresView): void;
|
|
7
|
+
export declare function publishPostgresViews(sql: postgres.TransactionSql, { schema, views }: {
|
|
8
|
+
schema: string;
|
|
9
|
+
views: readonly PostgresView[];
|
|
10
|
+
}): Promise<void>;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import postgres from 'postgres';
|
|
2
|
+
import { identifier, quote } from './identifier.js';
|
|
3
|
+
import { rebindDependents } from './postgres-dependents.js';
|
|
4
|
+
import { lockSchema, schemaName } from './postgres-session.js';
|
|
5
|
+
function text(value, what) {
|
|
6
|
+
if (typeof value !== 'string' ||
|
|
7
|
+
!value.trim() ||
|
|
8
|
+
value.includes('\0') ||
|
|
9
|
+
!value.isWellFormed())
|
|
10
|
+
throw new TypeError(`Invalid ${what}`);
|
|
11
|
+
}
|
|
12
|
+
// Refuses, with no I/O, a view that Postgres cannot hold as it is declared.
|
|
13
|
+
export function checkView(view) {
|
|
14
|
+
identifier(view.name, 'view name');
|
|
15
|
+
text(view.query, 'view query');
|
|
16
|
+
text(view.description, 'view description');
|
|
17
|
+
for (const [column, description] of Object.entries(view.columns)) {
|
|
18
|
+
identifier(column, 'column name');
|
|
19
|
+
text(description, 'column description');
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
// Publishes ordinary views and their descriptions in a caller-owned
|
|
23
|
+
// transaction, in schema, where the bare names of their queries are found
|
|
24
|
+
// too. Supply trusted SQL, and list dependencies before dependents. A view
|
|
25
|
+
// whose stored definition is the one its query makes stays as it is, with
|
|
26
|
+
// its OID, so a publication that changes nothing changes no view; the others
|
|
27
|
+
// are made again, and the views that read them read the new ones (ADR 0006).
|
|
28
|
+
export async function publishPostgresViews(sql, { schema, views }) {
|
|
29
|
+
schemaName(schema);
|
|
30
|
+
const names = new Set();
|
|
31
|
+
for (const view of views) {
|
|
32
|
+
checkView(view);
|
|
33
|
+
if (names.has(view.name))
|
|
34
|
+
throw new TypeError('Duplicate view names');
|
|
35
|
+
names.add(view.name);
|
|
36
|
+
}
|
|
37
|
+
if (views.length === 0)
|
|
38
|
+
return;
|
|
39
|
+
// A failed definition restores every replaced view, even if the caller
|
|
40
|
+
// catches the error and continues its transaction.
|
|
41
|
+
await sql.savepoint(async (transaction) => {
|
|
42
|
+
await lockSchema(transaction, schema);
|
|
43
|
+
await transaction.unsafe(`CREATE SCHEMA IF NOT EXISTS ${quote(schema)}`);
|
|
44
|
+
const [{ path } = { path: '' }] = await transaction `
|
|
45
|
+
SELECT current_setting('search_path') AS path`;
|
|
46
|
+
await transaction `SELECT set_config('search_path', ${quote(schema)}, true)`;
|
|
47
|
+
const changed = [];
|
|
48
|
+
for (const view of views)
|
|
49
|
+
if (!(await stored(transaction, schema, view)))
|
|
50
|
+
changed.push(view);
|
|
51
|
+
if (changed.length > 0)
|
|
52
|
+
await rebindDependents(transaction, changed.map(({ name }) => `${quote(schema)}.${quote(name)}`), async () => {
|
|
53
|
+
for (const view of changed.toReversed())
|
|
54
|
+
await transaction.unsafe(`DROP VIEW IF EXISTS ${quote(schema)}.${quote(view.name)}`);
|
|
55
|
+
for (const view of changed) {
|
|
56
|
+
const relation = `${quote(schema)}.${quote(view.name)}`;
|
|
57
|
+
// Extended protocol accepts one statement, so a query cannot
|
|
58
|
+
// append DDL.
|
|
59
|
+
await transaction `CREATE VIEW ${transaction.unsafe(relation)} AS ${transaction.unsafe(view.query)}`;
|
|
60
|
+
const columns = await transaction `
|
|
61
|
+
SELECT attname AS name FROM pg_attribute
|
|
62
|
+
WHERE attrelid = to_regclass(${relation})
|
|
63
|
+
AND attnum > 0 AND NOT attisdropped ORDER BY attnum`;
|
|
64
|
+
if (columns.length !== Object.keys(view.columns).length ||
|
|
65
|
+
columns.some(({ name }) => !Object.hasOwn(view.columns, name)))
|
|
66
|
+
throw new TypeError(`View ${relation} must describe exactly its output columns: ${columns.map(({ name }) => name).join(', ')}`);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
for (const view of views)
|
|
70
|
+
await describe(transaction, schema, view);
|
|
71
|
+
await transaction `SELECT set_config('search_path', ${path}, true)`;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
// Whether the view in schema has the definition that its query makes now. The
|
|
75
|
+
// query is made as a TEMP view, which the savepoint takes back, because
|
|
76
|
+
// Postgres keeps a view's definition as it normalizes it, not as its text. A
|
|
77
|
+
// query that Postgres refuses counts as changed, so the publication fails with
|
|
78
|
+
// its error.
|
|
79
|
+
async function stored(sql, schema, view) {
|
|
80
|
+
const relation = `${quote(schema)}.${quote(view.name)}`;
|
|
81
|
+
const [existing] = await sql `
|
|
82
|
+
SELECT CASE WHEN c.relkind = 'v' THEN pg_get_viewdef(c.oid) END AS definition
|
|
83
|
+
FROM pg_class c WHERE c.oid = to_regclass(${relation})`;
|
|
84
|
+
if (existing?.definition == null)
|
|
85
|
+
return false;
|
|
86
|
+
await sql.unsafe('SAVEPOINT "_elt_candidate"');
|
|
87
|
+
try {
|
|
88
|
+
await sql `CREATE TEMP VIEW "_elt_candidate" AS ${sql.unsafe(view.query)}`;
|
|
89
|
+
const [made] = await sql `
|
|
90
|
+
SELECT pg_get_viewdef('pg_temp."_elt_candidate"'::regclass) AS definition`;
|
|
91
|
+
return made?.definition === existing.definition;
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
if (error instanceof postgres.PostgresError)
|
|
95
|
+
return false;
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
finally {
|
|
99
|
+
await sql.unsafe('ROLLBACK TO SAVEPOINT "_elt_candidate"');
|
|
100
|
+
await sql.unsafe('RELEASE SAVEPOINT "_elt_candidate"');
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// COMMENT cannot bind values; Postgres quotes names and text itself.
|
|
104
|
+
async function describe(sql, schema, view) {
|
|
105
|
+
const comments = await sql `
|
|
106
|
+
SELECT format('COMMENT ON VIEW %I.%I IS %L', ${schema}::text,
|
|
107
|
+
${view.name}::text, ${view.description}::text) AS statement
|
|
108
|
+
WHERE obj_description(to_regclass(format('%I.%I', ${schema}::text, ${view.name}::text)), 'pg_class')
|
|
109
|
+
IS DISTINCT FROM ${view.description}::text
|
|
110
|
+
UNION ALL
|
|
111
|
+
SELECT format('COMMENT ON COLUMN %I.%I.%I IS %L', ${schema}::text,
|
|
112
|
+
${view.name}::text, c.key, c.value)
|
|
113
|
+
FROM jsonb_each_text(${sql.json(view.columns)}::jsonb) AS c
|
|
114
|
+
JOIN pg_attribute a ON a.attrelid = to_regclass(format('%I.%I', ${schema}::text, ${view.name}::text))
|
|
115
|
+
AND a.attname = c.key
|
|
116
|
+
WHERE col_description(a.attrelid, a.attnum) IS DISTINCT FROM c.value`;
|
|
117
|
+
for (const { statement } of comments)
|
|
118
|
+
await sql.unsafe(statement);
|
|
119
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { type Clearing, type CopyConfiguration, type FieldValues, type KeyValue, type Stage, Writer } from '@eltjs/elt';
|
|
2
|
+
import type { EncodedValue } from './postgres-column.ts';
|
|
3
|
+
import { TargetHistory } from './postgres-history.ts';
|
|
4
|
+
import type { PostgresLoad, Transaction } from './postgres-load.ts';
|
|
5
|
+
import type { PostgresTable } from './postgres-table.ts';
|
|
6
|
+
export declare const seq = "\"_elt_seq\"";
|
|
7
|
+
export declare const op = "\"_elt_op\"";
|
|
8
|
+
export declare abstract class PostgresWriter extends Writer {
|
|
9
|
+
#private;
|
|
10
|
+
readonly configuration: CopyConfiguration;
|
|
11
|
+
protected readonly url: string;
|
|
12
|
+
readonly schema: string;
|
|
13
|
+
readonly table: PostgresTable;
|
|
14
|
+
constructor(configuration: CopyConfiguration, url: string, schema: string, table: PostgresTable, retention: Temporal.Duration);
|
|
15
|
+
protected abstract initialize(transaction: Transaction, replacing: boolean): Promise<void>;
|
|
16
|
+
protected inspect(_sql: Transaction): Promise<void>;
|
|
17
|
+
get qualifiedName(): string;
|
|
18
|
+
protected get createTableSQL(): string;
|
|
19
|
+
protected build(_sql: Transaction, _into: string): Promise<void>;
|
|
20
|
+
protected adopt(_sql: Transaction): Promise<void>;
|
|
21
|
+
protected get written(): string[];
|
|
22
|
+
protected get staged(): string;
|
|
23
|
+
protected encode(record: unknown): EncodedValue[];
|
|
24
|
+
protected deletionRow(_key: Readonly<Record<string, KeyValue>>): EncodedValue[];
|
|
25
|
+
protected get replaces(): boolean;
|
|
26
|
+
protected kept(_stage: string): string;
|
|
27
|
+
protected merge(sql: Transaction, stage: string, written: Written, into: string, _history: TargetHistory | null): Promise<void>;
|
|
28
|
+
protected dedupIndexes(transaction: Transaction): Promise<string[]>;
|
|
29
|
+
private values;
|
|
30
|
+
clear({ writer, clock, connection: connector }: Clearing, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
|
|
31
|
+
prepare(load: PostgresLoad, { writer, reloading }: {
|
|
32
|
+
writer: string;
|
|
33
|
+
reloading: boolean;
|
|
34
|
+
}): Promise<Stage>;
|
|
35
|
+
}
|
|
36
|
+
export type Written = {
|
|
37
|
+
readonly loadedAt: string;
|
|
38
|
+
readonly committedIn: number;
|
|
39
|
+
};
|