@eltjs/elt-postgresql 0.0.0-stage → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +813 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
package/dist/view.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type Connection, DerivedDataset } from '@eltjs/elt';
|
|
2
|
+
import { PostgresDestination } from './postgres-destination.ts';
|
|
3
|
+
import type { PostgresTable } from './postgres-table.ts';
|
|
4
|
+
export declare class View extends DerivedDataset<PostgresTable> {
|
|
5
|
+
#private;
|
|
6
|
+
readonly name: string;
|
|
7
|
+
readonly into: PostgresDestination;
|
|
8
|
+
constructor({ into, name, description, columns, sql, }: {
|
|
9
|
+
into: PostgresDestination;
|
|
10
|
+
name: string;
|
|
11
|
+
description: string;
|
|
12
|
+
columns: Readonly<Record<string, string>>;
|
|
13
|
+
sql: string;
|
|
14
|
+
});
|
|
15
|
+
get qualifiedName(): string;
|
|
16
|
+
validate(connections: readonly Connection<PostgresTable>[]): void;
|
|
17
|
+
refresh({ connections, earlier, }: {
|
|
18
|
+
readonly connections: readonly Connection<PostgresTable>[];
|
|
19
|
+
readonly earlier: readonly DerivedDataset<PostgresTable>[];
|
|
20
|
+
}): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
export declare function made(connections: readonly Connection<PostgresTable>[], into: PostgresDestination): Set<string>;
|
package/dist/view.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import postgres from 'postgres';
|
|
2
|
+
import { DerivedDataset } from '@eltjs/elt';
|
|
3
|
+
import { PostgresDestination } from './postgres-destination.js';
|
|
4
|
+
import { PostgresSession } from './postgres-session.js';
|
|
5
|
+
import { checkView, publishPostgresViews, } from './postgres-views.js';
|
|
6
|
+
// A view over what the pipeline loads into one Postgres database: derived
|
|
7
|
+
// data that Postgres computes on each read, as a dbt model materialized as a
|
|
8
|
+
// view is. It lives in the schema of into, where its SQL's bare names resolve
|
|
9
|
+
// too. Its inputs are the relations its SQL reads, so it names none of them.
|
|
10
|
+
// It appears once its query compiles, so a reader never finds a view that
|
|
11
|
+
// fails; until a copy of the pipeline has made each relation it reads, and
|
|
12
|
+
// each derived dataset before it that it reads has appeared, it waits, as dbt
|
|
13
|
+
// skips a model whose upstream model was skipped. Any other error of its
|
|
14
|
+
// query is the view's own failure.
|
|
15
|
+
export class View extends DerivedDataset {
|
|
16
|
+
name;
|
|
17
|
+
// The database and schema the view lives in, where a computed dataset
|
|
18
|
+
// after it reads it.
|
|
19
|
+
into;
|
|
20
|
+
#view;
|
|
21
|
+
constructor({ into, name, description, columns, sql, }) {
|
|
22
|
+
super();
|
|
23
|
+
this.name = name;
|
|
24
|
+
this.into = into;
|
|
25
|
+
this.#view = { name, description, columns, query: sql };
|
|
26
|
+
checkView(this.#view);
|
|
27
|
+
Object.freeze(this);
|
|
28
|
+
}
|
|
29
|
+
get qualifiedName() {
|
|
30
|
+
return `${this.into.schema}.${this.name}`;
|
|
31
|
+
}
|
|
32
|
+
validate(connections) {
|
|
33
|
+
if (made(connections, this.into).has(this.qualifiedName))
|
|
34
|
+
throw new TypeError(`View ${this.qualifiedName} has the name of a table or reader view a copy makes`);
|
|
35
|
+
}
|
|
36
|
+
async refresh({ connections, earlier, }) {
|
|
37
|
+
await using session = new PostgresSession(this.into.url, 'elt-derived');
|
|
38
|
+
try {
|
|
39
|
+
await session.sql.begin((transaction) => publishPostgresViews(transaction, {
|
|
40
|
+
schema: this.into.schema,
|
|
41
|
+
views: [this.#view],
|
|
42
|
+
}));
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
const missing = missingRelation(error, this.into.schema);
|
|
46
|
+
if (missing !== undefined &&
|
|
47
|
+
(made(connections, this.into).has(missing) ||
|
|
48
|
+
earlier.some((dataset) => derivedRelation(dataset) === missing)))
|
|
49
|
+
return;
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
// The schema-qualified relation that Postgres found missing, with a bare name
|
|
55
|
+
// resolved in the schema the view's SQL reads bare names from.
|
|
56
|
+
function missingRelation(error, schema) {
|
|
57
|
+
if (!(error instanceof postgres.PostgresError) || error.code !== '42P01')
|
|
58
|
+
return undefined;
|
|
59
|
+
const name = /^relation "(.+)" does not exist$/.exec(error.message)?.[1];
|
|
60
|
+
if (name === undefined)
|
|
61
|
+
return undefined;
|
|
62
|
+
return name.includes('.') ? name : `${schema}.${name}`;
|
|
63
|
+
}
|
|
64
|
+
// The tables and reader views that the pipeline's copies make in the
|
|
65
|
+
// database of into, schema-qualified.
|
|
66
|
+
export function made(connections, into) {
|
|
67
|
+
return new Set(connections.flatMap(({ destination, steps }) => destination instanceof PostgresDestination && destination.url === into.url
|
|
68
|
+
? steps.flatMap(({ to }) => [
|
|
69
|
+
`${destination.schema}.${to.name}`,
|
|
70
|
+
...(to.readerView === undefined
|
|
71
|
+
? []
|
|
72
|
+
: [`${to.readerView.schema}.${to.readerView.name}`]),
|
|
73
|
+
])
|
|
74
|
+
: []));
|
|
75
|
+
}
|
|
76
|
+
// The relation that a derived dataset of this package makes for readers.
|
|
77
|
+
function derivedRelation(dataset) {
|
|
78
|
+
return 'qualifiedName' in dataset && typeof dataset.qualifiedName === 'string'
|
|
79
|
+
? dataset.qualifiedName
|
|
80
|
+
: undefined;
|
|
81
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,49 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@eltjs/elt-postgresql",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.3.15",
|
|
4
|
+
"description": "The Postgres destination of elt: each stream is a typed table in one schema, with a reader view in another, a history of each row for readings as of a position, and the views and computed datasets that a pipeline derives.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/ezzabuzaid/elt",
|
|
9
|
+
"directory": "packages/destinations/postgresql"
|
|
10
|
+
},
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/ezzabuzaid/elt/issues"
|
|
13
|
+
},
|
|
14
|
+
"homepage": "https://github.com/ezzabuzaid/elt/tree/main/packages/destinations/postgresql#readme",
|
|
15
|
+
"keywords": [
|
|
16
|
+
"elt",
|
|
17
|
+
"postgres",
|
|
18
|
+
"postgresql",
|
|
19
|
+
"destination",
|
|
20
|
+
"pipeline"
|
|
21
|
+
],
|
|
22
|
+
"type": "module",
|
|
23
|
+
"exports": {
|
|
24
|
+
"./package.json": "./package.json",
|
|
25
|
+
".": {
|
|
26
|
+
"types": "./dist/index.d.ts",
|
|
27
|
+
"default": "./dist/index.js"
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"files": [
|
|
31
|
+
"dist",
|
|
32
|
+
"!dist/spec/**",
|
|
33
|
+
"!**/*.tsbuildinfo",
|
|
34
|
+
"!**/*.test.*"
|
|
35
|
+
],
|
|
36
|
+
"publishConfig": {
|
|
37
|
+
"access": "public"
|
|
38
|
+
},
|
|
39
|
+
"dependencies": {
|
|
40
|
+
"@eltjs/elt": "^0.3.15",
|
|
41
|
+
"@zukhruf/async": "^0.8.27",
|
|
42
|
+
"postgres": "^3.4.9"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"@zukhruf/fs": "^0.8.27",
|
|
46
|
+
"@zukhruf/snapshot-differential": "^0.8.27",
|
|
47
|
+
"@zukhruf/testing": "^0.8.27"
|
|
48
|
+
}
|
|
49
|
+
}
|