@eltjs/elt-postgresql 0.3.14 → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/postgres-writer.js +155 -10
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -160,7 +160,7 @@ How a reader knows which numbers are done:
|
|
|
160
160
|
|
|
161
161
|
The history of a table:
|
|
162
162
|
|
|
163
|
-
- `<schema>._elt_history_<hash>` keeps each row that a commit replaced or removed, with `replaced_in`, the number of that commit, as a SQL Server temporal table keeps its history beside the current table. The writes that keep rows are a deduplicating upsert that changes the values (behind the same cursor guard), a staged `DELETE`,
|
|
163
|
+
- `<schema>._elt_history_<hash>` keeps each row that a commit replaced or removed, with `replaced_in`, the number of that commit, as a SQL Server temporal table keeps its history beside the current table. The writes that keep rows are a deduplicating upsert that changes the values (behind the same cursor guard), a staged `DELETE`, the end of the reload of a reset partition, an overwrite, a reload, and a change of shape. `<schema>._elt_reloads` names the partitions of each open reload of a reset partition. A write into a hidden table keeps nothing, because no reader saw its rows.
|
|
164
164
|
- `clear()` forgets the rows that it removes. Its commit C sets `min_valid` to C, if it is higher, with `expired_by` `clear`, and the history of the cleared table deletes each of its rows, with the stored files that only they refer to. A reading or `positions(after)` before C gives `{ expired }`, with the reason that a clear forgot the rows of its table. The history of another table keeps its rows with `replaced_in <= C` until its next commit or the retention deletes them; a reading skips them, as for a change of type.
|
|
165
165
|
- Each column of the history has the type of the table's column of the same name. It takes each new column of the table. A column that the table drops stays in the history with its last type.
|
|
166
166
|
- When a change of shape, or the swap of a reload, gives a column of the table another type, the history changes the type of its column in the same commit. This includes a column that the table dropped and a later change adds back with another type. Delta Lake, Iceberg, BigQuery and Snowflake change a type in place only when the change is a widening, and they read the past with its own schema, which one Postgres view cannot do. The destination follows the same rule:
|
package/dist/postgres-writer.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { hash } from 'node:crypto';
|
|
2
2
|
import { isDeepStrictEqual } from 'node:util';
|
|
3
|
-
import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, reloadMode, } from '@eltjs/elt';
|
|
3
|
+
import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, readPartitions, reloadMode, } from '@eltjs/elt';
|
|
4
4
|
import { quote } from './identifier.js';
|
|
5
5
|
import { TargetComments } from './postgres-comments.js';
|
|
6
6
|
import { rebindDependents } from './postgres-dependents.js';
|
|
@@ -9,7 +9,7 @@ import { storedColumns, storedFit } from './postgres-fit.js';
|
|
|
9
9
|
import { TargetHistory, rowHash, version } from './postgres-history.js';
|
|
10
10
|
import { commitOnce, expireBefore, installNumbering, } from './postgres-numbering.js';
|
|
11
11
|
import { TargetReaderView } from './postgres-reader-view.js';
|
|
12
|
-
import { PostgresSession, lockSchema, makeSchema } from './postgres-session.js';
|
|
12
|
+
import { PostgresSession, lockSchema, makeSchema, } from './postgres-session.js';
|
|
13
13
|
const batchSize = 1000;
|
|
14
14
|
// A batch also flushes once its JSON reaches this size, so rows of large text
|
|
15
15
|
// or binary values are sent a few at a time rather than a thousand at once.
|
|
@@ -224,6 +224,7 @@ export class PostgresWriter extends Writer {
|
|
|
224
224
|
}
|
|
225
225
|
// Clearing a copy also abandons the reload it left open.
|
|
226
226
|
await transaction.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
|
|
227
|
+
await this.#reloaded(transaction);
|
|
227
228
|
for (const column of this.table.columns.filter((column) => column.storesFile)) {
|
|
228
229
|
const store = new PostgresFileStore(this.schema, this.table, column);
|
|
229
230
|
const [chunks] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [store.qualifiedName]);
|
|
@@ -326,13 +327,28 @@ export class PostgresWriter extends Writer {
|
|
|
326
327
|
// keeps its indexes, its triggers and what depends on it. Each row of the
|
|
327
328
|
// hidden target that no row matches enters with this commit, and the
|
|
328
329
|
// loaded_at of the load that wrote it.
|
|
329
|
-
async #reloadInPlace(sql, committedIn) {
|
|
330
|
-
const target = { table: this.qualifiedName, id: 'ctid' };
|
|
330
|
+
async #reloadInPlace(sql, committedIn, target = { table: this.qualifiedName, id: 'ctid' }) {
|
|
331
331
|
const reloaded = { table: this.#hidden, id: 'ctid' };
|
|
332
332
|
await this.#history.remove(sql, committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, reloaded)})`);
|
|
333
333
|
await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.table.columns.map((column) => column.quotedName).join(', ')}, "loaded_at", $1::integer, ${version} FROM ${this.#hidden} WHERE ctid IN (${this.#unmatched(reloaded, target)}) ORDER BY ctid`, [committedIn]);
|
|
334
334
|
await sql.unsafe(`DROP TABLE ${this.#hidden}`);
|
|
335
335
|
}
|
|
336
|
+
// Gives a completed reload of some partitions their rows of the hidden
|
|
337
|
+
// target, in place and by difference: the rows of the target in those
|
|
338
|
+
// partitions are replaced as #reloadInPlace replaces a whole target, and
|
|
339
|
+
// the rows of every other partition stay as they are.
|
|
340
|
+
async #reloadPartitionsInPlace(sql, partitions, committedIn) {
|
|
341
|
+
const scoped = quote(`_elt_reloaded_${this.#hash}`);
|
|
342
|
+
const [within, values] = this.#within(partitions);
|
|
343
|
+
await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${scoped}`);
|
|
344
|
+
await sql.unsafe(`CREATE TEMP TABLE ${scoped} AS SELECT ctid AS "row" FROM ${this.qualifiedName} WHERE ${within}`, values);
|
|
345
|
+
await this.#reloadInPlace(sql, committedIn, {
|
|
346
|
+
table: this.qualifiedName,
|
|
347
|
+
id: 'ctid',
|
|
348
|
+
rows: `ctid IN (SELECT "row" FROM pg_temp.${scoped})`,
|
|
349
|
+
});
|
|
350
|
+
await sql.unsafe(`DROP TABLE pg_temp.${scoped}`);
|
|
351
|
+
}
|
|
336
352
|
// Refuses a cast of column from its stored type from that would change a
|
|
337
353
|
// stored value, such as double precision 1.5 to bigint 2, before the ALTER,
|
|
338
354
|
// as a STRICT SQLite table refuses it: each value that is not null must
|
|
@@ -437,6 +453,23 @@ export class PostgresWriter extends Writer {
|
|
|
437
453
|
#scope(partition) {
|
|
438
454
|
if (partition === null)
|
|
439
455
|
return ['', []];
|
|
456
|
+
const [condition, values] = this.#within([partition]);
|
|
457
|
+
return [` WHERE ${condition}`, values];
|
|
458
|
+
}
|
|
459
|
+
// The rows of any of the partitions, as a condition over the target's or
|
|
460
|
+
// the stage's columns, with one JSON parameter for each partition: NULL for
|
|
461
|
+
// a row that does not name its partition, such as a staged DELETE whose
|
|
462
|
+
// key leaves the partition out.
|
|
463
|
+
#within(partitions) {
|
|
464
|
+
const each = partitions.map((partition, index) => this.#partition(partition, `$${index + 1}::text::json`));
|
|
465
|
+
return [
|
|
466
|
+
each.length === 0
|
|
467
|
+
? 'FALSE'
|
|
468
|
+
: each.map(([condition]) => `(${condition})`).join(' OR '),
|
|
469
|
+
each.map(([, value]) => value),
|
|
470
|
+
];
|
|
471
|
+
}
|
|
472
|
+
#partition(partition, parameter) {
|
|
440
473
|
const columns = Object.keys(partition).map((field) => {
|
|
441
474
|
const column = this.table.columns.find((column) => column.field === field);
|
|
442
475
|
if (column === undefined)
|
|
@@ -444,10 +477,45 @@ export class PostgresWriter extends Writer {
|
|
|
444
477
|
return column;
|
|
445
478
|
});
|
|
446
479
|
return [
|
|
447
|
-
|
|
448
|
-
|
|
480
|
+
columns
|
|
481
|
+
.map((column, index) => `${column.quotedName} = ${column.valueFrom(parameter, index)}`)
|
|
482
|
+
.join(' AND '),
|
|
483
|
+
JSON.stringify(columns.map((column) => column.encode(partition))),
|
|
449
484
|
];
|
|
450
485
|
}
|
|
486
|
+
get #reloads() {
|
|
487
|
+
return `${quote(this.schema)}."_elt_reloads"`;
|
|
488
|
+
}
|
|
489
|
+
// The partitions that an open reload of the target reloads, in the
|
|
490
|
+
// writer's own table beside it, written in the commit that opens or
|
|
491
|
+
// continues the reload, so a run that resumes the reload continues it. A
|
|
492
|
+
// reload of the whole stream has no row.
|
|
493
|
+
async #reloading(sql) {
|
|
494
|
+
const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
|
|
495
|
+
if (made?.exists !== true)
|
|
496
|
+
return wholeStream;
|
|
497
|
+
const [row] = await sql.unsafe(`SELECT "partitions" FROM ${this.#reloads} WHERE "target" = $1`, [this.table.name]);
|
|
498
|
+
return row === undefined
|
|
499
|
+
? wholeStream
|
|
500
|
+
: new SomePartitions(readPartitions(this.stream, JSON.parse(row.partitions)));
|
|
501
|
+
}
|
|
502
|
+
async #reload(sql, reloads) {
|
|
503
|
+
if (!(reloads instanceof SomePartitions)) {
|
|
504
|
+
await this.#reloaded(sql);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
// The first reloads of the schema's tables make the table together.
|
|
508
|
+
await lockSchema(sql, this.schema);
|
|
509
|
+
await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#reloads} ("target" TEXT PRIMARY KEY, "partitions" TEXT NOT NULL)`);
|
|
510
|
+
await sql.unsafe(`INSERT INTO ${this.#reloads} ("target", "partitions") VALUES ($1, $2) ON CONFLICT ("target") DO UPDATE SET "partitions" = excluded."partitions"`, [this.table.name, JSON.stringify(reloads.partitions)]);
|
|
511
|
+
}
|
|
512
|
+
async #reloaded(sql) {
|
|
513
|
+
const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
|
|
514
|
+
if (made?.exists === true)
|
|
515
|
+
await sql.unsafe(`DELETE FROM ${this.#reloads} WHERE "target" = $1`, [
|
|
516
|
+
this.table.name,
|
|
517
|
+
]);
|
|
518
|
+
}
|
|
451
519
|
// One stream's load. Operations wait in a session-private TEMP stage, so no
|
|
452
520
|
// other session sees them and a crash leaves nothing behind. prepare only
|
|
453
521
|
// refuses, evolves a stored table the stream no longer fits unless the load
|
|
@@ -461,13 +529,21 @@ export class PostgresWriter extends Writer {
|
|
|
461
529
|
async prepare(load, { writer, reloading }) {
|
|
462
530
|
const { sql } = load;
|
|
463
531
|
const stage = quote(`_elt_stage_${this.#hash}`);
|
|
532
|
+
// The rows of a commit that belong to the partitions a reload reloads.
|
|
533
|
+
const reloadStage = quote(`_elt_reload_stage_${this.#hash}`);
|
|
534
|
+
const staging = (table) => `CREATE TEMP TABLE ${table} (${seq} BIGINT PRIMARY KEY, ${op} TEXT NOT NULL, ${this.table.columns.map((column) => `${column.quotedName} ${column.storageType}`).join(', ')}, ${version} TEXT)`;
|
|
535
|
+
// What an open reload reloads; the whole stream until a RESET of a
|
|
536
|
+
// partition opens one.
|
|
537
|
+
let reloads = wholeStream;
|
|
464
538
|
const stores = this.table.columns
|
|
465
539
|
.filter((column) => column.storesFile)
|
|
466
540
|
.map((column) => new PostgresFileStore(this.schema, this.table, column));
|
|
467
541
|
let mode = await load.transaction(async (sql) => {
|
|
468
542
|
await this.#refuse(sql, writer);
|
|
469
543
|
await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${stage}`);
|
|
470
|
-
await sql.unsafe(
|
|
544
|
+
await sql.unsafe(staging(stage));
|
|
545
|
+
await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${reloadStage}`);
|
|
546
|
+
await sql.unsafe(staging(reloadStage));
|
|
471
547
|
for (const store of stores)
|
|
472
548
|
await store.stage(sql);
|
|
473
549
|
const target = await storedFit(sql, this.table, this.qualifiedName, stage);
|
|
@@ -479,8 +555,12 @@ export class PostgresWriter extends Writer {
|
|
|
479
555
|
hidden,
|
|
480
556
|
});
|
|
481
557
|
// A hidden target no reload continues is a leftover readers never saw.
|
|
482
|
-
if (decided
|
|
558
|
+
if (decided === 'continue')
|
|
559
|
+
reloads = await this.#reloading(sql);
|
|
560
|
+
else {
|
|
483
561
|
await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
|
|
562
|
+
await this.#reloaded(sql);
|
|
563
|
+
}
|
|
484
564
|
await this.#view?.refuse(sql, target === 'stale');
|
|
485
565
|
if (decided === 'evolve') {
|
|
486
566
|
await this.#evolve(sql, stage, load);
|
|
@@ -491,6 +571,11 @@ export class PostgresWriter extends Writer {
|
|
|
491
571
|
return decided;
|
|
492
572
|
});
|
|
493
573
|
const fresh = mode === 'create' || mode === 'reload';
|
|
574
|
+
// The reload as the last commit left it, which a discard goes back to.
|
|
575
|
+
let committed = {
|
|
576
|
+
mode,
|
|
577
|
+
reloads,
|
|
578
|
+
};
|
|
494
579
|
const open = () => mode === 'reload' || mode === 'continue';
|
|
495
580
|
const into = () => (open() ? this.#hidden : this.qualifiedName);
|
|
496
581
|
const tables = () => [
|
|
@@ -565,11 +650,23 @@ export class PostgresWriter extends Writer {
|
|
|
565
650
|
// The whole stream starts over in a new hidden target.
|
|
566
651
|
await drop();
|
|
567
652
|
mode = 'reload';
|
|
653
|
+
reloads = wholeStream;
|
|
568
654
|
return;
|
|
569
655
|
}
|
|
570
656
|
const [rows, values] = this.#scope(partition);
|
|
571
657
|
await sql.unsafe(`DELETE FROM ${stage}${rows}`, values);
|
|
572
658
|
resets.push(partition);
|
|
659
|
+
// The partition starts over in a hidden target, and readers see its
|
|
660
|
+
// rows as they were until the stream ends. An overwrite replaces
|
|
661
|
+
// the whole target anyway.
|
|
662
|
+
if (this.replaces)
|
|
663
|
+
return;
|
|
664
|
+
if (mode === 'load') {
|
|
665
|
+
mode = 'reload';
|
|
666
|
+
reloads = new SomePartitions([partition]);
|
|
667
|
+
}
|
|
668
|
+
else if (open() && reloads instanceof SomePartitions)
|
|
669
|
+
reloads = reloads.with(partition);
|
|
573
670
|
return;
|
|
574
671
|
}
|
|
575
672
|
// Each file is read here, before the source advances past it. The
|
|
@@ -609,6 +706,8 @@ export class PostgresWriter extends Writer {
|
|
|
609
706
|
await sql.unsafe(`CREATE TABLE ${this.#tableSQL(this.#hidden)}`);
|
|
610
707
|
await this.build(sql, this.#hidden);
|
|
611
708
|
}
|
|
709
|
+
if (open())
|
|
710
|
+
await this.#reload(sql, reloads);
|
|
612
711
|
const replacing = await claiming.claim(sql, committedIn);
|
|
613
712
|
// The rows of a hidden target were never shown, so only the target
|
|
614
713
|
// keeps the rows a commit replaces.
|
|
@@ -625,6 +724,18 @@ export class PostgresWriter extends Writer {
|
|
|
625
724
|
const written = { loadedAt: load.loadedAt, committedIn };
|
|
626
725
|
if (replacing)
|
|
627
726
|
await this.#replace(sql, stage, written);
|
|
727
|
+
else if (open() && reloads instanceof SomePartitions) {
|
|
728
|
+
// The rows of the reloaded partitions go to the hidden target,
|
|
729
|
+
// the others to the target. A DELETE whose key does not name
|
|
730
|
+
// its partition goes to both, so the reload cannot bring back a
|
|
731
|
+
// row that the source deleted.
|
|
732
|
+
const [within, values] = this.#within(reloads.partitions);
|
|
733
|
+
await sql.unsafe(`INSERT INTO ${reloadStage} SELECT * FROM ${stage} WHERE (${within}) IS TRUE OR (${op} = 'D' AND (${within}) IS NULL)`, values);
|
|
734
|
+
await sql.unsafe(`DELETE FROM ${stage} WHERE (${within}) IS TRUE`, values);
|
|
735
|
+
await this.merge(sql, stage, written, this.qualifiedName, this.#history);
|
|
736
|
+
await this.merge(sql, reloadStage, written, this.#hidden, null);
|
|
737
|
+
await sql.unsafe(`TRUNCATE ${reloadStage}`);
|
|
738
|
+
}
|
|
628
739
|
else
|
|
629
740
|
await this.merge(sql, stage, written, into(), history);
|
|
630
741
|
await sql.unsafe(`TRUNCATE ${stage}`);
|
|
@@ -636,6 +747,7 @@ export class PostgresWriter extends Writer {
|
|
|
636
747
|
// Its first commit made the hidden target, so the reload continues.
|
|
637
748
|
if (mode === 'reload')
|
|
638
749
|
mode = 'continue';
|
|
750
|
+
committed = { mode, reloads };
|
|
639
751
|
resets = [];
|
|
640
752
|
},
|
|
641
753
|
complete: async () => {
|
|
@@ -643,10 +755,17 @@ export class PostgresWriter extends Writer {
|
|
|
643
755
|
return;
|
|
644
756
|
await load.commit(async (sql, committedIn) => {
|
|
645
757
|
const target = await storedFit(sql, this.table, this.qualifiedName, stage);
|
|
646
|
-
|
|
758
|
+
// A reload of partitions keeps the target's shape: prepare evolves
|
|
759
|
+
// a stored target before it loads.
|
|
760
|
+
if (reloads instanceof SomePartitions && target === 'fits')
|
|
761
|
+
await this.#reloadPartitionsInPlace(sql, reloads.partitions, committedIn);
|
|
762
|
+
else if (reloads instanceof SomePartitions)
|
|
763
|
+
throw new TypeError(`${this.qualifiedName} changed its shape while some of its partitions reloaded; clear the copy to load it again`);
|
|
764
|
+
else if (target === 'fits')
|
|
647
765
|
await this.#reloadInPlace(sql, committedIn);
|
|
648
766
|
else
|
|
649
767
|
await this.#swap(sql, committedIn);
|
|
768
|
+
await this.#reloaded(sql);
|
|
650
769
|
await this.#history.expire(sql);
|
|
651
770
|
for (const store of stores)
|
|
652
771
|
await store.prune(sql, [
|
|
@@ -655,14 +774,40 @@ export class PostgresWriter extends Writer {
|
|
|
655
774
|
]);
|
|
656
775
|
});
|
|
657
776
|
mode = 'load';
|
|
777
|
+
reloads = wholeStream;
|
|
778
|
+
committed = { mode, reloads };
|
|
779
|
+
},
|
|
780
|
+
// A RESET that no commit took is undone with the operations after it:
|
|
781
|
+
// the source's checkpoint does not know it.
|
|
782
|
+
discard: async () => {
|
|
783
|
+
await drop();
|
|
784
|
+
({ mode, reloads } = committed);
|
|
658
785
|
},
|
|
659
|
-
discard: drop,
|
|
660
786
|
[Symbol.asyncDispose]: async () => {
|
|
661
787
|
await drop();
|
|
662
788
|
await sql.unsafe(`DROP TABLE pg_temp.${stage}`);
|
|
789
|
+
await sql.unsafe(`DROP TABLE pg_temp.${reloadStage}`);
|
|
663
790
|
for (const store of stores)
|
|
664
791
|
await store.unstage(sql);
|
|
665
792
|
},
|
|
666
793
|
};
|
|
667
794
|
}
|
|
668
795
|
}
|
|
796
|
+
// What an open reload reloads: the whole stream, or some partitions of it,
|
|
797
|
+
// while the others load into the target as before, as Delta Lake's
|
|
798
|
+
// replaceWhere replaces only the rows that its condition selects.
|
|
799
|
+
class WholeStream {
|
|
800
|
+
}
|
|
801
|
+
class SomePartitions {
|
|
802
|
+
partitions;
|
|
803
|
+
constructor(partitions) {
|
|
804
|
+
this.partitions = Object.freeze([...partitions]);
|
|
805
|
+
}
|
|
806
|
+
// These partitions and partition, which a later RESET reloads too.
|
|
807
|
+
with(partition) {
|
|
808
|
+
return this.partitions.some((other) => isDeepStrictEqual(other, partition))
|
|
809
|
+
? this
|
|
810
|
+
: new SomePartitions([...this.partitions, partition]);
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
const wholeStream = new WholeStream();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@eltjs/elt-postgresql",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.15",
|
|
4
4
|
"description": "The Postgres destination of elt: each stream is a typed table in one schema, with a reader view in another, a history of each row for readings as of a position, and the views and computed datasets that a pipeline derives.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"access": "public"
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@eltjs/elt": "^0.3.
|
|
40
|
+
"@eltjs/elt": "^0.3.15",
|
|
41
41
|
"@zukhruf/async": "^0.8.27",
|
|
42
42
|
"postgres": "^3.4.9"
|
|
43
43
|
},
|