@eltjs/elt-postgresql 0.0.0-stage → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +359 -2
- package/dist/computed-dataset.d.ts +24 -0
- package/dist/computed-dataset.js +346 -0
- package/dist/identifier.d.ts +2 -0
- package/dist/identifier.js +14 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +8 -0
- package/dist/postgres-append-writer.d.ts +5 -0
- package/dist/postgres-append-writer.js +11 -0
- package/dist/postgres-as-of.d.ts +14 -0
- package/dist/postgres-as-of.js +51 -0
- package/dist/postgres-checkpoint-store.d.ts +10 -0
- package/dist/postgres-checkpoint-store.js +83 -0
- package/dist/postgres-column.d.ts +53 -0
- package/dist/postgres-column.js +302 -0
- package/dist/postgres-columns.d.ts +13 -0
- package/dist/postgres-columns.js +80 -0
- package/dist/postgres-comments.d.ts +6 -0
- package/dist/postgres-comments.js +20 -0
- package/dist/postgres-deduplicating-writer.d.ts +25 -0
- package/dist/postgres-deduplicating-writer.js +135 -0
- package/dist/postgres-dependents.d.ts +2 -0
- package/dist/postgres-dependents.js +105 -0
- package/dist/postgres-destination.d.ts +31 -0
- package/dist/postgres-destination.js +87 -0
- package/dist/postgres-file-store.d.ts +23 -0
- package/dist/postgres-file-store.js +80 -0
- package/dist/postgres-fit.d.ts +9 -0
- package/dist/postgres-fit.js +25 -0
- package/dist/postgres-history.d.ts +17 -0
- package/dist/postgres-history.js +141 -0
- package/dist/postgres-identifiers.d.ts +8 -0
- package/dist/postgres-identifiers.js +26 -0
- package/dist/postgres-load.d.ts +20 -0
- package/dist/postgres-load.js +73 -0
- package/dist/postgres-numbering.d.ts +42 -0
- package/dist/postgres-numbering.js +311 -0
- package/dist/postgres-overwrite-writer.d.ts +4 -0
- package/dist/postgres-overwrite-writer.js +8 -0
- package/dist/postgres-privileges.d.ts +7 -0
- package/dist/postgres-privileges.js +50 -0
- package/dist/postgres-reader-view.d.ts +11 -0
- package/dist/postgres-reader-view.js +64 -0
- package/dist/postgres-session.d.ts +11 -0
- package/dist/postgres-session.js +61 -0
- package/dist/postgres-sync-history-schema.d.ts +31 -0
- package/dist/postgres-sync-history-schema.js +100 -0
- package/dist/postgres-sync-history.d.ts +13 -0
- package/dist/postgres-sync-history.js +152 -0
- package/dist/postgres-table.d.ts +15 -0
- package/dist/postgres-table.js +73 -0
- package/dist/postgres-views.d.ts +10 -0
- package/dist/postgres-views.js +119 -0
- package/dist/postgres-writer.d.ts +39 -0
- package/dist/postgres-writer.js +668 -0
- package/dist/view.d.ts +22 -0
- package/dist/view.js +81 -0
- package/package.json +47 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright 2026 ezzabuzaid
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -1,3 +1,360 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @eltjs/elt-postgresql
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The Postgres destination of elt. It loads each table of a connection into one schema of a Postgres database, with typed columns, and it does not block readers while it loads. It can keep a reader view of each table in a different schema, such as `marts`, and it publishes documented SQL views that the application defines. It also has a checkpoint store and a sync history that keep their data in the same database.
|
|
4
|
+
|
|
5
|
+
This README tells about the Postgres destination. The pipeline itself is in [elt](../../elt/README.md). The database of the repository, its roles and its `marts` schema are in [infra](../../../infra/README.md).
|
|
6
|
+
|
|
7
|
+
The words in these documents have one meaning each. See the glossary in [CONTEXT.md](./CONTEXT.md), and the glossary of [elt](../../elt/CONTEXT.md) for the words of all of elt.
|
|
8
|
+
|
|
9
|
+
## Load a stream into Postgres
|
|
10
|
+
|
|
11
|
+
```ts
|
|
12
|
+
import { Connection, Copy, Pipeline } from '@eltjs/elt';
|
|
13
|
+
import {
|
|
14
|
+
PostgresCheckpointStore,
|
|
15
|
+
PostgresDestination,
|
|
16
|
+
PostgresSyncHistory,
|
|
17
|
+
} from '@eltjs/elt-postgresql';
|
|
18
|
+
|
|
19
|
+
const url = 'postgres://warehouse:warehouse@127.0.0.1:55432/warehouse';
|
|
20
|
+
const warehouse = new PostgresDestination({ url, schema: 'apple_notes' });
|
|
21
|
+
|
|
22
|
+
await new Pipeline({
|
|
23
|
+
history: new PostgresSyncHistory({ url }),
|
|
24
|
+
connections: [
|
|
25
|
+
new Connection({
|
|
26
|
+
name: 'apple-notes',
|
|
27
|
+
source: notes,
|
|
28
|
+
destination: warehouse,
|
|
29
|
+
checkpoints: new PostgresCheckpointStore({ url, schema: 'apple_notes' }),
|
|
30
|
+
steps: [
|
|
31
|
+
new Copy(
|
|
32
|
+
notes.notes,
|
|
33
|
+
warehouse.table('raw_notes').withReaderView('marts', 'notes'),
|
|
34
|
+
{
|
|
35
|
+
id: 'apple-notes:notes',
|
|
36
|
+
syncMode: 'incremental',
|
|
37
|
+
destinationSyncMode: 'append_dedup',
|
|
38
|
+
},
|
|
39
|
+
),
|
|
40
|
+
],
|
|
41
|
+
}),
|
|
42
|
+
],
|
|
43
|
+
}).run();
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
- `PostgresDestination({ url, schema })` loads each table of a connection into one schema. The first load makes the schema.
|
|
47
|
+
- The URL holds credentials, so `location()`, which checkpoints keep, names only the schema and the table. `destination.url` gives the URL to the views and computed datasets of the destination; it is a getter, so JSON and `util.inspect` do not show it.
|
|
48
|
+
- The destination checks the declarations without a connection to the database.
|
|
49
|
+
- A schema name must have at most 63 bytes, must not hold NUL, and must not start with `pg_`.
|
|
50
|
+
|
|
51
|
+
## Columns
|
|
52
|
+
|
|
53
|
+
Inferred columns follow the stream schema. The [string formats](../../elt/README.md#string-formats) get their own types, so readers can do date arithmetic:
|
|
54
|
+
|
|
55
|
+
| JSON Schema | Postgres |
|
|
56
|
+
| ----------------------------------------------- | ------------------------------------------------------------------ |
|
|
57
|
+
| `string` | `TEXT` |
|
|
58
|
+
| `string` + `format: 'date'` | `DATE` |
|
|
59
|
+
| `string` + `format: 'date-time'`, precision ≤ 6 | `TIMESTAMPTZ` |
|
|
60
|
+
| `string` + `format: 'date-time-local'`, ≤ 6 | `TIMESTAMP` |
|
|
61
|
+
| `string` + `format: 'time-local'`, ≤ 6 | `TIME` |
|
|
62
|
+
| any of those three at precision 7–9 | `TEXT`, sorted with `COLLATE "C"` |
|
|
63
|
+
| `string` + `format: 'int64'` | `BIGINT` |
|
|
64
|
+
| `string` + `format: 'decimal'` | `NUMERIC(precision, scale)`, or `NUMERIC` when not declared |
|
|
65
|
+
| `string` + `contentEncoding: 'base64'` | `BYTEA` |
|
|
66
|
+
| `integer` | `BIGINT` |
|
|
67
|
+
| `number` | `DOUBLE PRECISION` |
|
|
68
|
+
| `boolean` | `BOOLEAN` |
|
|
69
|
+
| `array` of any of the above | a native array of that type, such as `BIGINT[]` or `TIMESTAMPTZ[]` |
|
|
70
|
+
| `array` of objects | `JSONB[]`, one `jsonb` object for each element |
|
|
71
|
+
|
|
72
|
+
- Postgres keeps time to the microsecond. A time with more fraction digits, such as the `datetime2(7)` of SQL Server, stays exact text. It sorts in time order, because each value of the column has the same width. Cast it (`::timestamp`) for date arithmetic, which rounds.
|
|
73
|
+
- A `TIMESTAMPTZ` keeps the instant. It does not keep the offset that the value had.
|
|
74
|
+
- Arrays keep the order of their elements, and load a JSON `null` as SQL `NULL`. Readers use `= ANY(...)`, `unnest()` and `cardinality()`. `marts.catalog` shows the type of the element.
|
|
75
|
+
- ISO dates count years astronomically, and Postgres does not. Thus the year `0000` loads as `0001 BC`, which is the same day. Each other year is written as it is given.
|
|
76
|
+
- Explicit columns are `columns.text`, `integer`, `real`, `boolean`, `date`, `timestamp` and `blob(field)`, with `.notNull()` and `.primaryKey()`.
|
|
77
|
+
- A plain `blob()` stores `BYTEA`. `blob().from(file)` keeps the original file in chunk tables, and `text().from(file).parse(parser)` stores the parsed text as `TEXT`. See [file bytes in the database](#file-bytes-in-the-database).
|
|
78
|
+
- Table, view and column names follow the [naming rule](../../elt/README.md#names). Postgres compares names with their case, keeps 63 bytes of a name, and reserves `loaded_at`, `committed_in`, `replaced_in` and the system column names `xmin`, `xmax`, `cmin`, `cmax`, `ctid` and `tableoid`. The naming rule turns a field `_elt_row_hash` into `elt_row_hash`, so no field takes that name.
|
|
79
|
+
|
|
80
|
+
## File bytes in the database
|
|
81
|
+
|
|
82
|
+
Postgres takes the same `text().from(file).parse(parser)` and `blob().from(file)` declarations as SQLite. See [stored files](../../elt/README.md#stored-files-and-document-parsing) in elt.
|
|
83
|
+
|
|
84
|
+
- Parsed content is `TEXT`.
|
|
85
|
+
- The column of an original file holds a UUID. Its bytes are in a table `_elt_files_<hash>` for each column, in the schema of the destination, as `(file UUID, n BIGINT, bytes BYTEA)` chunks of up to 4 MiB.
|
|
86
|
+
- The hash is the first 40 hex characters of the SHA-256 of `JSON.stringify([tableName, columnName])`.
|
|
87
|
+
- To read the original, join on the file UUID and order by `n`.
|
|
88
|
+
- An empty file has one empty chunk. A file that is not available gives a null value.
|
|
89
|
+
- The chunks of a load wait in a `TEMP` table that only the session sees, until the commit of their rows moves them. A discard empties that table.
|
|
90
|
+
- After each merge, the commit removes the chunks that no row of the table, or of its hidden table while a reload is open, refers to, for example the chunks of a record that deduplication refused.
|
|
91
|
+
- A clear of a copy also removes its files.
|
|
92
|
+
- A plain `blob()` column with no `.from(file)` stores an inline `BYTEA` value.
|
|
93
|
+
|
|
94
|
+
## Descriptions
|
|
95
|
+
|
|
96
|
+
Sources describe their data with the `description` of JSON Schema: at the root for the meaning of the source record, and on each property for the meaning of the field, its nulls, its units and its relations in the source. These descriptions belong to the source. They hold no names of destination tables. Notes describes its five streams and their fields in this way.
|
|
97
|
+
|
|
98
|
+
- Each Postgres copy publishes those descriptions as native table and column comments.
|
|
99
|
+
- Inferred columns and explicit columns keep the descriptions of their fields. A column that the copy leaves out gets no comment.
|
|
100
|
+
- The table comment tells the meaning of the source record apart from the modes of the copy: the extraction and load modes, an append history where source identities can repeat, and the keys and conflict rules of deduplication.
|
|
101
|
+
- Text from a file names its parser. A column of an original file tells of its UUID and of the chunk table that holds the bytes.
|
|
102
|
+
- `loaded_at` tells of the load that last wrote the row. It does not tell of a change in the source or of the health of a sync.
|
|
103
|
+
- The first commit of a stream writes the comments, and each later load replaces them. A property description that you remove clears its old column comment.
|
|
104
|
+
- Descriptions must be strings with no NUL and no malformed Unicode. The destination checks them before it opens storage. The native `format` of Postgres quotes the names and the text.
|
|
105
|
+
- Read comments with `obj_description` and `col_description`, or with a catalog view over them.
|
|
106
|
+
- A view that changes the meaning of a field must describe that new meaning itself.
|
|
107
|
+
- The status of a sync and the coverage of an extraction are data of the run, not comments of the schema.
|
|
108
|
+
|
|
109
|
+
## Locks and commits
|
|
110
|
+
|
|
111
|
+
A run holds one connection to the schema and takes no lock. Loads into one schema run side by side, and a [checkpoint store](#checkpoints) keeps two runs of one copy apart. See [ADR 0003](./docs/adr/0003-a-load-takes-no-schema-lock.md).
|
|
112
|
+
|
|
113
|
+
- Postgres refuses two sessions that make one schema or one table at the same moment. Thus the first commits of the tables of one schema take turns for the DDL that they share: the schema and `_elt_writers`. The schema is made in a transaction of its own, which begins only when the lock of the schema is held, because a transaction that waits for an advisory lock does not see the catalog changes that it waited for. Connections that load into one schema can start together.
|
|
114
|
+
- As in the Postgres destination of Airbyte, no transaction stays open between commits. Thus the read of the source holds no locks on the targets and keeps nothing from vacuum, however many tables a run loads. See [ADR 0002](./docs/adr/0002-no-transaction-stays-open-between-commits.md).
|
|
115
|
+
- The prepare of a stream only checks the stored table and makes its stage. Thus a run that fails before the first commit of a stream leaves nothing of that stream.
|
|
116
|
+
- Each stream stages its operations in a `TEMP` table that only its session sees, and commits them at its own checkpoints. Each commit is one short transaction.
|
|
117
|
+
|
|
118
|
+
Each commit:
|
|
119
|
+
|
|
120
|
+
- Operations get to the stage in batches of at most 1000 rows or 16 MiB. Each batch goes as one JSON parameter and is cast for each column, so large text and binary values travel a few at a time.
|
|
121
|
+
- A record that is too large to send alone, past the longest string of V8 or the largest message of Postgres (1 GB), fails its stream. The error names the table, its largest column and its key.
|
|
122
|
+
- Original files wait in a chunk table that only the session sees, until the commit moves them.
|
|
123
|
+
- The first commit of a stream makes its table, comments, index and reader view, and records its writer.
|
|
124
|
+
- A commit merges the stage of one stream into its table. The result is the same as when the operations apply one at a time: staged `DELETE`s remove their keys, only the records after the last `DELETE` of a key count, `replace` keeps the last record, and `cursor_newer` keeps the first record with the greatest cursor. Text cursors compare by bytes (`COLLATE "C"`).
|
|
125
|
+
- Each row has `loaded_at` (`TIMESTAMPTZ`), the time on the clock of the pipeline when the load that wrote its values opened. A deduplicating load or an overwrite that brings the same values again leaves the row, its `loaded_at` and its `committed_in` as they are.
|
|
126
|
+
- Each row has `_elt_row_hash` (`TEXT`), the SHA-256 of its values, which the load computes as it stages the row. A null value does not count, so a column that the stream adds makes no new version. A stored file counts as the SHA-256 of its bytes, not as its UUID.
|
|
127
|
+
- Each row has `committed_in` (`INTEGER`), the number of the commit that last wrote it. See [commit numbers and positions](#commit-numbers-and-positions).
|
|
128
|
+
- An overwrite replaces the rows of the table by difference, with `DELETE`, not `TRUNCATE`, at the commit of its stream: each stored row that no staged record matches by `_elt_row_hash` goes, each record that no row matches enters, and each row that matches stays as it is. Two identical rows match as a multiset, so a keyless overwrite that sends one of them again removes only the other. The exclusive lock of `TRUNCATE` would block readers until the commit ends. Until then, readers see the earlier load. A [reset](../../elt/README.md#resets) deletes its scope in the same way. See [ADR 0004](./docs/adr/0004-an-overwrite-deletes-rows-and-does-not-truncate.md).
|
|
129
|
+
- Deduplication upserts on a unique index that is named after the table and the key (`_elt_dedup_<hash>`). Postgres makes the index one time, and makes it again only when the key changes. A load that replaces the rows makes the index after it removes the rows that leave.
|
|
130
|
+
- The writer of each table is in `<schema>._elt_writers`, by the [target ownership](../../elt/README.md#target-ownership) rule.
|
|
131
|
+
|
|
132
|
+
Shape changes:
|
|
133
|
+
|
|
134
|
+
- When the columns, types or `NOT NULL`s of a stored table do not fit the stream, the load [evolves it](../../elt/README.md#shape-changes) in place before it reads, as the Postgres destination of Airbyte does. `ALTER TABLE` adds a nullable column, drops a column that the stream dropped (with its chunk table, for a file column), and converts a column with a new type `USING column::type`. Before the conversion, each stored value that is not null must cast back to itself (`column::type::old IS DISTINCT FROM column` finds none), so `1.5` in `double precision` fails the load to `bigint`, naming the column, and `2` converts. A conversion from `text` is the parse that the load applies to each new value, so the conversion itself refuses a value it cannot parse. It keeps `NOT NULL` and the primary key on the explicit key columns of the target, and moves the key when they change. The reader view is made again over the result.
|
|
135
|
+
- The change waits at most 3 seconds for readers to let go. Otherwise it fails the copy until the next run.
|
|
136
|
+
- A value that does not convert, or rows that repeat a new key, fail the change. The table stays as it was.
|
|
137
|
+
- The views that read the table, its history or its reader view, also views that elt did not make, read the table again after the change, with their owners, options, comments and privileges. A view that does not fit the change, such as one that reads a column that the change drops, fails the change, and the error names the view. See [ADR 0006](./docs/adr/0006-a-change-of-a-table-makes-the-views-over-it-again.md).
|
|
138
|
+
- A [reset](../../elt/README.md#resets) of the full stream, or an overwrite whose stored table does not fit, reloads into a hidden table `_elt_next_<hash>`. The first commit of the reload makes it. When the stream ends with no failure:
|
|
139
|
+
- If the stored table still has the stream's shape, the load changes it in place, by difference: each row that the hidden table has again stays, with its commit and its `loaded_at`; each other row goes into the history; and each new row enters. The table keeps its indexes, its triggers and the views that depend on it.
|
|
140
|
+
- If not, the load swaps the hidden table in, also with a wait of at most 3 seconds, and drops the chunk table of a file column that it does not have now. The history keeps every row of the old table. The swapped-in table and its reader view keep the privileges of the old ones, and the views over them read the new table, as for a change of shape.
|
|
141
|
+
|
|
142
|
+
## Commit numbers and positions
|
|
143
|
+
|
|
144
|
+
Each commit has a number. One numbering orders the commits of every schema of the database, so a reader such as eca reads the whole warehouse at one position, as a Kafka consumer reads up to the last stable offset.
|
|
145
|
+
|
|
146
|
+
- The numbering is in the schema `_warehouse`, beside the [sync history](#sync-history). The first load or clear in the database makes it. It takes the same lock as the sync history, because Postgres refuses two sessions that make one schema at the same moment. Thus first loads into many schemas can start together.
|
|
147
|
+
- `_warehouse._elt_commits` has one row for each commit: its `number`, its `load` and its `committed_at`. The sequence `_warehouse._elt_commit_numbers` gives the numbers. A commit takes its number at its start, so two commits that run at the same time can end in a different order than their numbers.
|
|
148
|
+
- `_warehouse._elt_loads` has one row for each load: its `connector` (the name of the connection) or its `dataset` (the name of a computed dataset), its `opened_at`, and its `closed_at`, which is null while the load is open. A load of a connection opens when a pass starts to load, and closes when the pass ends; a load of a dataset is one refresh. When a load opens, it closes each load of the same connector or dataset that is still open: the process of that pass or refresh ended first, for example because it was killed. A closed load cannot commit: a commit locks the row of its load (`FOR SHARE`), so a close at the same time waits for the commit, or makes it fail.
|
|
149
|
+
- A clear that empties a table is a load of one commit, in the same transaction. It does not close the open loads of its connector.
|
|
150
|
+
- A commit into a [hidden table](./CONTEXT.md) shows nothing to readers. When the stream ends, one more commit gives the rows to the table with its number, and readers see them from that commit on.
|
|
151
|
+
- A change of shape that casts a column or drops one with values is a commit of the load too. The history keeps each row that it changes, as it was, and the row hash of the row becomes null, so the next record of the row writes it again. A change of type that the history cannot follow by a [widening](./CONTEXT.md) is a commit of the load too, even when it changes no row.
|
|
152
|
+
- `_warehouse._elt_epoch` keeps the epoch, an id that names the numbering, and `min_valid`, the oldest commit that a reading can read as of. A new database, or a database whose `_warehouse` was dropped, starts a new epoch, with `min_valid` 0.
|
|
153
|
+
- `await destination.positions(after?)` gives the positions of the database after the position `after`, in ascending order, or all positions. A position is `<epoch>:<number>`. It names the last commit of a closed load, when no other load has commits before and after it, so a reader there sees no part of a pass in any schema. The list starts at `min_valid`. When `after` is of a different epoch, is after the newest settled commit, or is before `min_valid`, it gives `{ expired }` with the reason. A string that is not a position throws a `TypeError`. A database with no numbering has no positions.
|
|
154
|
+
|
|
155
|
+
How a reader knows which numbers are done:
|
|
156
|
+
|
|
157
|
+
- Commits run side by side, so a number can still be in flight while a higher one has committed. Each commit holds a transaction lock on its number (`pg_advisory_xact_lock`) until it ends. The newest settled commit is the highest number below the first number whose lock is still held. Numbers of commits that rolled back are gaps that the reader skips. This is the high-water mark of event stores, and no commit waits for another.
|
|
158
|
+
- Postgres releases the locks of a transaction after its commit is visible to new snapshots. Thus the reader finds the high-water mark first, and then reads `_elt_commits` and `_elt_loads` in a `REPEATABLE READ` snapshot. `pg_locks` is not part of a snapshot, so the other order could show a commit at or below the mark that the snapshot does not see.
|
|
159
|
+
- A commit takes its number and then its lock, in two steps. A reader that looked between them would pass the number. Thus each commit holds a shared lock (`pg_advisory_lock_shared`) across the two steps, and a reader holds the same lock alone while it reads the last number taken and the locks held. This is a readers–writer lock with the commits as the many readers.
|
|
160
|
+
|
|
161
|
+
The history of a table:
|
|
162
|
+
|
|
163
|
+
- `<schema>._elt_history_<hash>` keeps each row that a commit replaced or removed, with `replaced_in`, the number of that commit, as a SQL Server temporal table keeps its history beside the current table. The writes that keep rows are a deduplicating upsert that changes the values (behind the same cursor guard), a staged `DELETE`, a reset of a partition, an overwrite, a reload, and a change of shape. A write into a hidden table keeps nothing, because no reader saw its rows.
|
|
164
|
+
- `clear()` forgets the rows that it removes. Its commit C sets `min_valid` to C, if it is higher, with `expired_by` `clear`, and the history of the cleared table deletes each of its rows, with the stored files that only they refer to. A reading or `positions(after)` before C gives `{ expired }`, with the reason that a clear forgot the rows of its table. The history of another table keeps its rows with `replaced_in <= C` until its next commit or the retention deletes them; a reading skips them, as for a change of type.
|
|
165
|
+
- Each column of the history has the type of the table's column of the same name. It takes each new column of the table. A column that the table drops stays in the history with its last type.
|
|
166
|
+
- When a change of shape, or the swap of a reload, gives a column of the table another type, the history changes the type of its column in the same commit. This includes a column that the table dropped and a later change adds back with another type. Delta Lake, Iceberg, BigQuery and Snowflake change a type in place only when the change is a widening, and they read the past with its own schema, which one Postgres view cannot do. The destination follows the same rule:
|
|
167
|
+
- A widening maps each old value to one new value exactly, with no loss: any type to `text`, `bigint` to `numeric`, `date` to `timestamp without time zone`, and an array whose item widens. The history casts its column with the same cast (`USING column::type`), and each position stays readable.
|
|
168
|
+
- Any other change, such as `text` to `bigint`, or `double precision` to `bigint`, acts as a rewrite. `bigint` to `double precision` is not a widening, because above 2^53 it loses digits. The commit E of the change sets `min_valid` to E, if it is higher. The history of the table deletes each row with `replaced_in <= E`, which only positions before E read, together with the stored chunks that only those rows refer to, and then casts its column. A reading or `positions(after)` before E gives `{ expired }`, with the reason that a commit changed the type of a column. Each position from E on reads exactly. The history of other tables keeps its rows with `replaced_in <= E` until a retention deletes them.
|
|
169
|
+
- A file that a row of the history refers to stays while the row stays: its chunks, or its file in a file storage (`.store(files)`).
|
|
170
|
+
|
|
171
|
+
Retention:
|
|
172
|
+
|
|
173
|
+
- `new PostgresDestination({ url, schema, retention })` sets how long the history keeps a position readable, as a `Temporal.Duration`: 30 days unless set, as Delta Lake keeps its log. A negative one throws a `RangeError`.
|
|
174
|
+
- When a commit takes its number, let W be the newest position at or below the newest commit older than the window. The commit sets `min_valid` to W, if it is higher. W is a position, not a commit, so a pass that stays open longer than the window keeps the position before it readable.
|
|
175
|
+
- When the commit moves `min_valid`, it also deletes each row of `_warehouse._elt_commits` with a number less than W, and each closed load of `_warehouse._elt_loads` that has no commit left. W is a position, so each load with a commit before W ends at or before W. The row of W stays, because a reading as of W gives its `committed_at`. A `committed_in` of a row is only compared with numbers, so it needs no row in `_elt_commits`. An open load stays, also when it has no commit yet. The row of `_elt_epoch` stays locked until the commit ends, so a commit at the same time that read an older `min_valid` does not move it back.
|
|
176
|
+
- In the same commit, the writer deletes each row of its table's history with `replaced_in <= min_valid`, and then the chunks that no row of the table or of its history refers to. The history of another table deletes its rows at that table's next commit; until then a reading skips them, because no position at or after `min_valid` reads them.
|
|
177
|
+
- A reading or `positions(after)` before `min_valid` gives `{ expired }`, with the reason that names the retention, or the change of type, that set it last.
|
|
178
|
+
- One numbering serves every destination of the database, so the shortest retention among them decides for all readers.
|
|
179
|
+
|
|
180
|
+
A reading as of a position:
|
|
181
|
+
|
|
182
|
+
- Each [reader view](#reader-views) reads its table as of the position that its transaction sets in `elt.position`: the rows of the table with `committed_in <= P`, and the rows of the history with `committed_in <= P < replaced_in`. A session that sets nothing reads the table as it is now, and no row of the history. A view built on a reader view, such as a [view](#declare-a-view) in `marts` or a view of the host in another schema, reads as of the same position. A read makes no object.
|
|
183
|
+
- `await destination.reading(position?)` opens its own session and gives `{ sql, position, at }`, or `{ expired }`. Dispose it with `await using` to end the session. It takes the high-water mark, then begins `REPEATABLE READ READ ONLY`, so no commit after it opens shows. With no position, it reads the newest position. Below the newest commit that its snapshot sees, it sets `elt.position` for its transaction. `at` is when the commit of the position was made.
|
|
184
|
+
- A position of a different epoch, past the newest settled commit, or before `min_valid`, gives `{ expired }`. A number that is not a position, such as a commit inside a pass, and a string that is not a position, throw a `TypeError`.
|
|
185
|
+
- The [sync history](#sync-history) keeps no row versions. A reading below the newest commit fails each of its views in `marts`, with the message that it cannot be read as of a position. The tables in `_warehouse` read as they are now.
|
|
186
|
+
- A table with no reader view reads as it is now. Read through the reader view.
|
|
187
|
+
|
|
188
|
+
## Checkpoints
|
|
189
|
+
|
|
190
|
+
`PostgresCheckpointStore({ url, schema })` keeps the checkpoints in `<schema>._elt_checkpoints`, beside the data. See [ADR 0001](./docs/adr/0001-checkpoints-are-kept-beside-the-data.md).
|
|
191
|
+
|
|
192
|
+
- The columns are `id`, `binding`, `reloading` and `state`. `binding` and `state` are `JSON`, which keeps state that `JSONB` would refuse, such as `\u0000` and lone surrogates.
|
|
193
|
+
- `DROP SCHEMA … CASCADE` resets the data and the checkpoints together.
|
|
194
|
+
- The store takes a session advisory lock for each copy `id`, on its own connection, in sorted order. Different IDs run in parallel. A run that finds one ID in use releases the IDs that it took and fails with `Checkpoint <id> is in use by another run`.
|
|
195
|
+
- Each save autocommits, so no checkpoint transaction stays open while the load runs.
|
|
196
|
+
- The store makes the schema and the table in their own committed transaction, under the lock of the schema. Thus two runs that make them at the same time take turns.
|
|
197
|
+
|
|
198
|
+
## Reader views
|
|
199
|
+
|
|
200
|
+
`table.withReaderView(schema, name)` returns the same target. A load also shows it as the view `schema.name`, with the external schema of the stream, its columns and `loaded_at`, and with the comments of the table.
|
|
201
|
+
|
|
202
|
+
- `committed_in`, `_elt_row_hash` and `replaced_in` are internal: only the SQL of the destination reads them, for the history, the retention, the readings and the difference. The reader view does not show them, now or as of a position. Thus a column that the destination adds for itself changes nothing that a reader sees, as SQL Server hides the period columns of a temporal table that it marks `HIDDEN`. The raw table shows them; readers do not query it.
|
|
203
|
+
|
|
204
|
+
- The load makes the view in the same commit as its table, so the view exists when the table exists. It follows inserts, updates and deletes, and it is not made again for them.
|
|
205
|
+
- The stream needs a JSON Schema `description`, and each column needs one too. File columns and `loaded_at` bring their own, and so does `committed_in` of the raw table. A missing description fails the declaration of the copy before a connection reads.
|
|
206
|
+
- The schema must exist. In the [warehouse](../../../infra/README.md), `marts` is part of the contract of the reader, and default privileges make the view readable.
|
|
207
|
+
- A load never replaces a view that exists, because a replace of a view that readers can see would lock them out. It only writes the comments again, which does not block readers.
|
|
208
|
+
- A relation with the name of the view that is not a view of the table, or that has different columns, is refused before the load reads. The error names it.
|
|
209
|
+
- A table that evolves or reloads gets its view again, as [above](#locks-and-commits).
|
|
210
|
+
- The view reads its table and the [history](#commit-numbers-and-positions) of the table as of a position. Each column of the history has the type that the column of the table has now, so the view reads it with no cast.
|
|
211
|
+
|
|
212
|
+
## Declare a view
|
|
213
|
+
|
|
214
|
+
```ts
|
|
215
|
+
import { PostgresDestination, View } from '@eltjs/elt-postgresql';
|
|
216
|
+
|
|
217
|
+
const marts = new PostgresDestination({ url, schema: 'marts' });
|
|
218
|
+
|
|
219
|
+
const unread = new View({
|
|
220
|
+
into: marts,
|
|
221
|
+
name: 'unread_count',
|
|
222
|
+
description: 'How many messages each chat has that the user did not read.',
|
|
223
|
+
columns: {
|
|
224
|
+
chat: 'The chat.',
|
|
225
|
+
n: 'Number of messages that the user did not read.',
|
|
226
|
+
},
|
|
227
|
+
sql: 'SELECT chat, count(*) AS n FROM messages WHERE read = 0 GROUP BY chat',
|
|
228
|
+
});
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Give the view to the pipeline in `derived`. The view is `marts.unread_count`: it lives in the schema of `into`, and the bare names of its SQL, such as `messages`, are found in that schema too. Name a relation of another schema with its schema, such as `raw.raw_messages`.
|
|
232
|
+
|
|
233
|
+
- A view names no inputs. Its SQL reads them, as in [elt-sqlite](../sqlite/docs/adr/0001-a-view-names-no-inputs.md).
|
|
234
|
+
- The view appears when its query compiles. Thus a reader never finds a view that fails.
|
|
235
|
+
- While the SQL reads a raw table or a reader view that a copy of the pipeline has not made yet, the view waits. It appears at the refresh after that copy loads. Postgres tells which relation is missing (`42P01`), and the view waits only when the pipeline makes that relation.
|
|
236
|
+
- While the SQL reads a view or a computed dataset that comes before it in `derived` and that still waits, the view waits too, as dbt skips a model whose upstream model was skipped.
|
|
237
|
+
- Any other error of the query, for example a misspelled table, is a dataset failure. It fails the run with the error of Postgres.
|
|
238
|
+
- The columns must be the same as the columns that the query gives. `marts.catalog` lists them with their descriptions when the view is in `marts`.
|
|
239
|
+
- A refresh that changes nothing leaves the view as it was. The refresh makes the query in a savepoint and compares its definition with the stored one, so the view keeps its OID, its grants and its readers.
|
|
240
|
+
- A changed declaration replaces the view in one transaction. The views that read it, also views that elt did not make, read the new view, with their owners, comments and privileges ([ADR 0006](./docs/adr/0006-a-change-of-a-table-makes-the-views-over-it-again.md)). The view keeps its own privileges.
|
|
241
|
+
|
|
242
|
+
## Declare a computed dataset
|
|
243
|
+
|
|
244
|
+
```ts
|
|
245
|
+
import { DocumentText } from '@eltjs/elt';
|
|
246
|
+
import { ComputedDataset } from '@eltjs/elt-postgresql';
|
|
247
|
+
|
|
248
|
+
const text = new ComputedDataset({
|
|
249
|
+
into: marts,
|
|
250
|
+
name: 'document_text',
|
|
251
|
+
description: 'The text of each stored document.',
|
|
252
|
+
from: [documents],
|
|
253
|
+
compute: new DocumentText({
|
|
254
|
+
parser: () => parser,
|
|
255
|
+
files: files.directory,
|
|
256
|
+
column: 'documentRef',
|
|
257
|
+
}),
|
|
258
|
+
});
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
`documents` is the target of a copy, and `from` can name more than one. The rule reads each row that those copies loaded, through the reader view of a target that has one. `from` can also name a `View` that comes before the computed dataset in `derived`. See [Write a rule](../../elt/README.md#write-a-rule) in elt.
|
|
262
|
+
|
|
263
|
+
A computed dataset keeps one output row for each key, in the table `raw_<name>` in the schema of `into`, beside its history. Readers query the view `<name>` in the same schema. Its columns are the same as in [elt-sqlite](../sqlite/README.md#declare-a-computed-dataset): the key, `input_hash`, `rule`, `status` (`computed`, `empty` or `failed`), `error`, and the columns of the rule. A `text` column of a rule is `TEXT`, and a `blob` column is `BYTEA`.
|
|
264
|
+
|
|
265
|
+
At each refresh, the computed dataset compares the inputs with the output rows:
|
|
266
|
+
|
|
267
|
+
- It computes each key that is new, or whose input or rule identity changed. The refresh after the passes also computes each key whose status is `failed`, once for each pass.
|
|
268
|
+
- It deletes each output row whose key no input has any more.
|
|
269
|
+
- It keeps an output row when the rule stops handling its input.
|
|
270
|
+
- Each call of the rule commits as it settles, so readers see the output rows as they land. The calls run side by side; their commits take turns on the one session of the refresh.
|
|
271
|
+
- When the declaration of the raw table changes, the refresh makes the table again. The history keeps each row of the old table, and the views over the dataset read the new one.
|
|
272
|
+
|
|
273
|
+
A refresh is a load in `_warehouse._elt_loads`, under the name of its dataset, so a [reading](#commit-numbers-and-positions) shows the computed dataset as of a position:
|
|
274
|
+
|
|
275
|
+
- Each transaction of the refresh that changes a row takes one commit number. A transaction that changes no row takes none. No commit inside a refresh that is open is a position.
|
|
276
|
+
- The raw table has `committed_in` and `_elt_row_hash` after the columns of the rule. The view `<name>` does not show them, and it reads as of the position of its reading, as a reader view does. A computed dataset has no `loaded_at`.
|
|
277
|
+
- A rule column cannot have the name of the key, of an engine column, of `committed_in`, `_elt_row_hash` or `replaced_in`. The constructor refuses such a rule with a `TypeError` that names the column and the dataset.
|
|
278
|
+
- When the rule makes a row again with the same values, for example a `failed` row that fails again with the same error, the row stays as it is, with its commit.
|
|
279
|
+
- Before a commit changes or deletes an output row, it copies the row into the history of the raw table.
|
|
280
|
+
|
|
281
|
+
## When something fails
|
|
282
|
+
|
|
283
|
+
- **A rule failure** gives an output row with the status `failed` and the reason in `error`. The run continues, and the refresh after the next pass tries the input again.
|
|
284
|
+
- **A dataset failure**, for example a query that Postgres refuses, rejects the run, or it stops the watch, with a `DerivedDatasetError` that names the dataset.
|
|
285
|
+
- **A table that is gone**, for example one that a person dropped by hand, is no failure. A computed dataset that reads it waits and keeps its output rows. When the copy loads the table again, the dataset asks the rule only for inputs that changed.
|
|
286
|
+
- **A copy that failed** keeps its last committed rows. The derived datasets read those rows.
|
|
287
|
+
|
|
288
|
+
## Sync history
|
|
289
|
+
|
|
290
|
+
`PostgresSyncHistory({ url })` is the [sync history](../../elt/README.md#sync-history) of a pipeline that loads into Postgres. Use the same database for the history and the destinations of the connections.
|
|
291
|
+
|
|
292
|
+
- It records only connections with a Postgres destination. It refuses other connections when the pipeline validates.
|
|
293
|
+
- It records into `_warehouse.sync_attempts`, `_warehouse.extraction_coverage` and `_warehouse.stream_checks`. The last one holds the latest check of each stream by a watch that found it unchanged.
|
|
294
|
+
- `install()` makes those tables and publishes the `marts` views `sync_attempts`, `extraction_coverage`, `sync_status`, `stream_status` and `catalog`, in one transaction. The pipeline calls it before each run and watch, and it is safe to repeat. See [ADR 0005](./docs/adr/0005-the-history-install-publishes-the-catalog.md).
|
|
295
|
+
- `new PostgresSyncHistory({ url, readers: ['agent_reader'] })` names the roles that read the database as of a position with their own session: `new PostgresDestination({ url: readerUrl, schema })`, then `positions()` and `reading()`. Each install grants each reader what a reading needs, which is the numbering in `_warehouse` and nothing else of it, as dbt applies the grants that a model declares after each build. Thus no host names the objects of the numbering, and a reader keeps its grants when the numbering is made again. The install grants nothing else: who may read `marts` is a grant of the database, such as its default privileges. A role that does not exist fails the install.
|
|
296
|
+
- Before a pass reads, the history inserts one `sync_attempts` row, with the connection name as `connector` and the source identity as `source`. It inserts one `extraction_coverage` row for each selected stream, with the declared `coverage(stream)` of the source and the schema of the destination as `target_schema`. The outcomes close them.
|
|
297
|
+
- A run declares each stream that a connection selected. A watch pass reads only the streams whose change signals changed, so an attempt vouches only for its own `extraction_coverage` rows.
|
|
298
|
+
- A connection that is not valid or that stopped, a failed change signal and a failed upkeep each record a failed attempt with no outcomes.
|
|
299
|
+
- `_warehouse` tables are made only when they are missing.
|
|
300
|
+
|
|
301
|
+
The relations in `marts`:
|
|
302
|
+
|
|
303
|
+
| Relation | Contents |
|
|
304
|
+
| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
305
|
+
| `catalog` | Each view and column in `marts`, with its description. The start point of a reader. A table is not for readers, also when it is in `marts`, such as the raw table of a computed dataset. |
|
|
306
|
+
| `sync_status` | For each connection: the latest attempt, the end of the last successful pass, and `current_as_of` (the oldest of its streams). |
|
|
307
|
+
| `stream_status` | For each connection and stream: the latest attempt that declared the stream with the status of its copy, and the last attempt in which that copy succeeded. A watch pass reads only changed streams, so judge a stream here, by `current_as_of`. |
|
|
308
|
+
| `sync_attempts` | The kept history of attempts, with start and end times, status and errors. |
|
|
309
|
+
| `extraction_coverage` | For each attempt and stream: the configured selection, the explanation of the scope, the target, the outcome of the copy, the committed counts and the failed partitions. |
|
|
310
|
+
|
|
311
|
+
What the relations tell:
|
|
312
|
+
|
|
313
|
+
- Each attempt keeps its own declarations.
|
|
314
|
+
- `sync_status` gives the latest attempt apart from the most recent attempt in which all copies succeeded. `stream_status` gives the same two facts for each stream, because a watch pass declares only the streams that changed.
|
|
315
|
+
- A pass with no change moves the success forward, also with zero writes.
|
|
316
|
+
- A failed or partial pass keeps the earlier success, and its scope stays available by the attempt ID.
|
|
317
|
+
- The status of each copy and the failed partitions let a reader find the streams that succeeded in a partial attempt.
|
|
318
|
+
- The counts are the accepted operations that the pass committed, also deduplication no-ops and deletions of keys that were not there. They are not counts of changed rows or totals.
|
|
319
|
+
- A first failure has no time of success.
|
|
320
|
+
- Coverage is the [configured scope](../../elt/README.md#coverage). Judge freshness by `current_as_of` ([the four clocks](../../elt/README.md#the-four-clocks)).
|
|
321
|
+
|
|
322
|
+
## Decisions
|
|
323
|
+
|
|
324
|
+
- [ADR 0001](./docs/adr/0001-checkpoints-are-kept-beside-the-data.md): checkpoints are kept beside the data.
|
|
325
|
+
- [ADR 0002](./docs/adr/0002-no-transaction-stays-open-between-commits.md): no transaction stays open between commits.
|
|
326
|
+
- [ADR 0003](./docs/adr/0003-a-load-takes-no-schema-lock.md): a load takes no schema lock.
|
|
327
|
+
- [ADR 0004](./docs/adr/0004-an-overwrite-deletes-rows-and-does-not-truncate.md): an overwrite deletes rows and does not truncate.
|
|
328
|
+
- [ADR 0005](./docs/adr/0005-the-history-install-publishes-the-catalog.md): the history install publishes the catalog.
|
|
329
|
+
- [ADR 0006](./docs/adr/0006-a-change-of-a-table-makes-the-views-over-it-again.md): a change of a table makes the views over it again.
|
|
330
|
+
- [ADR 0007](./docs/adr/0007-readers-are-named-on-the-sync-history.md): readers are named on the sync history.
|
|
331
|
+
- [ADR 0008](./docs/adr/0008-derived-datasets-follow-elt-sqlite.md): derived datasets follow elt-sqlite.
|
|
332
|
+
|
|
333
|
+
## Development
|
|
334
|
+
|
|
335
|
+
Run the commands from the workspace root. The tests start their own Postgres in Docker through `Postgres` of `@zukhruf/testing/postgres`.
|
|
336
|
+
|
|
337
|
+
```sh
|
|
338
|
+
npx nx run elt-postgresql:test # builds, then runs the tests
|
|
339
|
+
npx nx run elt-postgresql:typecheck # formats, lints, then type checks
|
|
340
|
+
npx nx run elt-postgresql:build # compiles src/ to dist/
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
```
|
|
344
|
+
src/
|
|
345
|
+
postgres-destination.ts PostgresDestination: tables and the load
|
|
346
|
+
postgres-writer.ts the stage, the commit, evolve and reload of a table
|
|
347
|
+
postgres-column.ts the type of each column
|
|
348
|
+
postgres-reader-view.ts the reader view of a table, as of a position
|
|
349
|
+
postgres-history.ts the history of a table, and the row hash
|
|
350
|
+
postgres-numbering.ts commit numbers, loads and positions
|
|
351
|
+
postgres-as-of.ts a reading of the database as of a position
|
|
352
|
+
view.ts View
|
|
353
|
+
computed-dataset.ts ComputedDataset
|
|
354
|
+
postgres-views.ts the publication of views, for View and the sync history
|
|
355
|
+
postgres-dependents.ts the views over a table, made again over its change
|
|
356
|
+
postgres-privileges.ts the privileges of a relation, kept across its change
|
|
357
|
+
postgres-checkpoint-store.ts PostgresCheckpointStore
|
|
358
|
+
postgres-sync-history.ts PostgresSyncHistory and the catalog
|
|
359
|
+
postgres-sync-history-schema.ts the tables and views of the history
|
|
360
|
+
```
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type Connection, DerivedDataset, type PipelineClock, type RefreshTrigger, type Rule } from '@eltjs/elt';
|
|
2
|
+
import { PostgresDestination } from './postgres-destination.ts';
|
|
3
|
+
import type { PostgresTable } from './postgres-table.ts';
|
|
4
|
+
import { View } from './view.ts';
|
|
5
|
+
export declare class ComputedDataset<Input, Output extends string | Uint8Array> extends DerivedDataset<PostgresTable> {
|
|
6
|
+
#private;
|
|
7
|
+
readonly name: string;
|
|
8
|
+
readonly into: PostgresDestination;
|
|
9
|
+
constructor({ into, name, description, from, compute, }: {
|
|
10
|
+
into: PostgresDestination;
|
|
11
|
+
name: string;
|
|
12
|
+
description: string;
|
|
13
|
+
from: readonly (PostgresTable | View)[];
|
|
14
|
+
compute: Rule<Input, Output>;
|
|
15
|
+
});
|
|
16
|
+
get qualifiedName(): string;
|
|
17
|
+
validate(connections: readonly Connection<PostgresTable>[]): void;
|
|
18
|
+
refresh({ connections, clock, signal, trigger, }: {
|
|
19
|
+
readonly connections: readonly Connection<PostgresTable>[];
|
|
20
|
+
readonly clock: PipelineClock;
|
|
21
|
+
readonly signal?: AbortSignal;
|
|
22
|
+
readonly trigger: RefreshTrigger;
|
|
23
|
+
}): Promise<void>;
|
|
24
|
+
}
|