@eltjs/elt-postgresql 0.3.14 → 0.3.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -160,7 +160,7 @@ How a reader knows which numbers are done:
160
160
 
161
161
  The history of a table:
162
162
 
163
- - `<schema>._elt_history_<hash>` keeps each row that a commit replaced or removed, with `replaced_in`, the number of that commit, as a SQL Server temporal table keeps its history beside the current table. The writes that keep rows are a deduplicating upsert that changes the values (behind the same cursor guard), a staged `DELETE`, a reset of a partition, an overwrite, a reload, and a change of shape. A write into a hidden table keeps nothing, because no reader saw its rows.
163
+ - `<schema>._elt_history_<hash>` keeps each row that a commit replaced or removed, with `replaced_in`, the number of that commit, as a SQL Server temporal table keeps its history beside the current table. The writes that keep rows are a deduplicating upsert that changes the values (behind the same cursor guard), a staged `DELETE`, the end of the reload of a reset partition, an overwrite, a reload, and a change of shape. `<schema>._elt_reloads` names the partitions of each open reload of a reset partition. A write into a hidden table keeps nothing, because no reader saw its rows.
164
164
  - `clear()` forgets the rows that it removes. Its commit C sets `min_valid` to C, if it is higher, with `expired_by` `clear`, and the history of the cleared table deletes each of its rows, with the stored files that only they refer to. A reading or `positions(after)` before C gives `{ expired }`, with the reason that a clear forgot the rows of its table. The history of another table keeps its rows with `replaced_in <= C` until its next commit or the retention deletes them; a reading skips them, as for a change of type.
165
165
  - Each column of the history has the type of the table's column of the same name. It takes each new column of the table. A column that the table drops stays in the history with its last type.
166
166
  - When a change of shape, or the swap of a reload, gives a column of the table another type, the history changes the type of its column in the same commit. This includes a column that the table dropped and a later change adds back with another type. Delta Lake, Iceberg, BigQuery and Snowflake change a type in place only when the change is a widening, and they read the past with its own schema, which one Postgres view cannot do. The destination follows the same rule:
@@ -1,6 +1,6 @@
1
1
  import { hash } from 'node:crypto';
2
2
  import { isDeepStrictEqual } from 'node:util';
3
- import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, reloadMode, } from '@eltjs/elt';
3
+ import { FileContent, TargetOwnedError, Writer, describeTarget, errorMessage, readPartitions, reloadMode, } from '@eltjs/elt';
4
4
  import { quote } from './identifier.js';
5
5
  import { TargetComments } from './postgres-comments.js';
6
6
  import { rebindDependents } from './postgres-dependents.js';
@@ -9,7 +9,7 @@ import { storedColumns, storedFit } from './postgres-fit.js';
9
9
  import { TargetHistory, rowHash, version } from './postgres-history.js';
10
10
  import { commitOnce, expireBefore, installNumbering, } from './postgres-numbering.js';
11
11
  import { TargetReaderView } from './postgres-reader-view.js';
12
- import { PostgresSession, lockSchema, makeSchema } from './postgres-session.js';
12
+ import { PostgresSession, lockSchema, makeSchema, } from './postgres-session.js';
13
13
  const batchSize = 1000;
14
14
  // A batch also flushes once its JSON reaches this size, so rows of large text
15
15
  // or binary values are sent a few at a time rather than a thousand at once.
@@ -224,6 +224,7 @@ export class PostgresWriter extends Writer {
224
224
  }
225
225
  // Clearing a copy also abandons the reload it left open.
226
226
  await transaction.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
227
+ await this.#reloaded(transaction);
227
228
  for (const column of this.table.columns.filter((column) => column.storesFile)) {
228
229
  const store = new PostgresFileStore(this.schema, this.table, column);
229
230
  const [chunks] = await transaction.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [store.qualifiedName]);
@@ -326,13 +327,28 @@ export class PostgresWriter extends Writer {
326
327
  // keeps its indexes, its triggers and what depends on it. Each row of the
327
328
  // hidden target that no row matches enters with this commit, and the
328
329
  // loaded_at of the load that wrote it.
329
- async #reloadInPlace(sql, committedIn) {
330
- const target = { table: this.qualifiedName, id: 'ctid' };
330
+ async #reloadInPlace(sql, committedIn, target = { table: this.qualifiedName, id: 'ctid' }) {
331
331
  const reloaded = { table: this.#hidden, id: 'ctid' };
332
332
  await this.#history.remove(sql, committedIn, this.qualifiedName, ` WHERE "_elt_old".ctid IN (${this.#unmatched(target, reloaded)})`);
333
333
  await sql.unsafe(`INSERT INTO ${this.qualifiedName} (${this.written.join(', ')}) SELECT ${this.table.columns.map((column) => column.quotedName).join(', ')}, "loaded_at", $1::integer, ${version} FROM ${this.#hidden} WHERE ctid IN (${this.#unmatched(reloaded, target)}) ORDER BY ctid`, [committedIn]);
334
334
  await sql.unsafe(`DROP TABLE ${this.#hidden}`);
335
335
  }
336
+ // Gives a completed reload of some partitions their rows of the hidden
337
+ // target, in place and by difference: the rows of the target in those
338
+ // partitions are replaced as #reloadInPlace replaces a whole target, and
339
+ // the rows of every other partition stay as they are.
340
+ async #reloadPartitionsInPlace(sql, partitions, committedIn) {
341
+ const scoped = quote(`_elt_reloaded_${this.#hash}`);
342
+ const [within, values] = this.#within(partitions);
343
+ await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${scoped}`);
344
+ await sql.unsafe(`CREATE TEMP TABLE ${scoped} AS SELECT ctid AS "row" FROM ${this.qualifiedName} WHERE ${within}`, values);
345
+ await this.#reloadInPlace(sql, committedIn, {
346
+ table: this.qualifiedName,
347
+ id: 'ctid',
348
+ rows: `ctid IN (SELECT "row" FROM pg_temp.${scoped})`,
349
+ });
350
+ await sql.unsafe(`DROP TABLE pg_temp.${scoped}`);
351
+ }
336
352
  // Refuses a cast of column from its stored type from that would change a
337
353
  // stored value, such as double precision 1.5 to bigint 2, before the ALTER,
338
354
  // as a STRICT SQLite table refuses it: each value that is not null must
@@ -437,6 +453,23 @@ export class PostgresWriter extends Writer {
437
453
  #scope(partition) {
438
454
  if (partition === null)
439
455
  return ['', []];
456
+ const [condition, values] = this.#within([partition]);
457
+ return [` WHERE ${condition}`, values];
458
+ }
459
+ // The rows of any of the partitions, as a condition over the target's or
460
+ // the stage's columns, with one JSON parameter for each partition: NULL for
461
+ // a row that does not name its partition, such as a staged DELETE whose
462
+ // key leaves the partition out.
463
+ #within(partitions) {
464
+ const each = partitions.map((partition, index) => this.#partition(partition, `$${index + 1}::text::json`));
465
+ return [
466
+ each.length === 0
467
+ ? 'FALSE'
468
+ : each.map(([condition]) => `(${condition})`).join(' OR '),
469
+ each.map(([, value]) => value),
470
+ ];
471
+ }
472
+ #partition(partition, parameter) {
440
473
  const columns = Object.keys(partition).map((field) => {
441
474
  const column = this.table.columns.find((column) => column.field === field);
442
475
  if (column === undefined)
@@ -444,10 +477,45 @@ export class PostgresWriter extends Writer {
444
477
  return column;
445
478
  });
446
479
  return [
447
- ` WHERE ${columns.map((column, index) => `${column.quotedName} = ${column.valueFrom('$1::text::json', index)}`).join(' AND ')}`,
448
- [JSON.stringify(columns.map((column) => column.encode(partition)))],
480
+ columns
481
+ .map((column, index) => `${column.quotedName} = ${column.valueFrom(parameter, index)}`)
482
+ .join(' AND '),
483
+ JSON.stringify(columns.map((column) => column.encode(partition))),
449
484
  ];
450
485
  }
486
+ get #reloads() {
487
+ return `${quote(this.schema)}."_elt_reloads"`;
488
+ }
489
+ // The partitions that an open reload of the target reloads, in the
490
+ // writer's own table beside it, written in the commit that opens or
491
+ // continues the reload, so a run that resumes the reload continues it. A
492
+ // reload of the whole stream has no row.
493
+ async #reloading(sql) {
494
+ const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
495
+ if (made?.exists !== true)
496
+ return wholeStream;
497
+ const [row] = await sql.unsafe(`SELECT "partitions" FROM ${this.#reloads} WHERE "target" = $1`, [this.table.name]);
498
+ return row === undefined
499
+ ? wholeStream
500
+ : new SomePartitions(readPartitions(this.stream, JSON.parse(row.partitions)));
501
+ }
502
+ async #reload(sql, reloads) {
503
+ if (!(reloads instanceof SomePartitions)) {
504
+ await this.#reloaded(sql);
505
+ return;
506
+ }
507
+ // The first reloads of the schema's tables make the table together.
508
+ await lockSchema(sql, this.schema);
509
+ await sql.unsafe(`CREATE TABLE IF NOT EXISTS ${this.#reloads} ("target" TEXT PRIMARY KEY, "partitions" TEXT NOT NULL)`);
510
+ await sql.unsafe(`INSERT INTO ${this.#reloads} ("target", "partitions") VALUES ($1, $2) ON CONFLICT ("target") DO UPDATE SET "partitions" = excluded."partitions"`, [this.table.name, JSON.stringify(reloads.partitions)]);
511
+ }
512
+ async #reloaded(sql) {
513
+ const [made] = await sql.unsafe('SELECT to_regclass($1) IS NOT NULL AS "exists"', [this.#reloads]);
514
+ if (made?.exists === true)
515
+ await sql.unsafe(`DELETE FROM ${this.#reloads} WHERE "target" = $1`, [
516
+ this.table.name,
517
+ ]);
518
+ }
451
519
  // One stream's load. Operations wait in a session-private TEMP stage, so no
452
520
  // other session sees them and a crash leaves nothing behind. prepare only
453
521
  // refuses, evolves a stored table the stream no longer fits unless the load
@@ -461,13 +529,21 @@ export class PostgresWriter extends Writer {
461
529
  async prepare(load, { writer, reloading }) {
462
530
  const { sql } = load;
463
531
  const stage = quote(`_elt_stage_${this.#hash}`);
532
+ // The rows of a commit that belong to the partitions a reload reloads.
533
+ const reloadStage = quote(`_elt_reload_stage_${this.#hash}`);
534
+ const staging = (table) => `CREATE TEMP TABLE ${table} (${seq} BIGINT PRIMARY KEY, ${op} TEXT NOT NULL, ${this.table.columns.map((column) => `${column.quotedName} ${column.storageType}`).join(', ')}, ${version} TEXT)`;
535
+ // What an open reload reloads; the whole stream until a RESET of a
536
+ // partition opens one.
537
+ let reloads = wholeStream;
464
538
  const stores = this.table.columns
465
539
  .filter((column) => column.storesFile)
466
540
  .map((column) => new PostgresFileStore(this.schema, this.table, column));
467
541
  let mode = await load.transaction(async (sql) => {
468
542
  await this.#refuse(sql, writer);
469
543
  await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${stage}`);
470
- await sql.unsafe(`CREATE TEMP TABLE ${stage} (${seq} BIGINT PRIMARY KEY, ${op} TEXT NOT NULL, ${this.table.columns.map((column) => `${column.quotedName} ${column.storageType}`).join(', ')}, ${version} TEXT)`);
544
+ await sql.unsafe(staging(stage));
545
+ await sql.unsafe(`DROP TABLE IF EXISTS pg_temp.${reloadStage}`);
546
+ await sql.unsafe(staging(reloadStage));
471
547
  for (const store of stores)
472
548
  await store.stage(sql);
473
549
  const target = await storedFit(sql, this.table, this.qualifiedName, stage);
@@ -479,8 +555,12 @@ export class PostgresWriter extends Writer {
479
555
  hidden,
480
556
  });
481
557
  // A hidden target no reload continues is a leftover readers never saw.
482
- if (decided !== 'continue')
558
+ if (decided === 'continue')
559
+ reloads = await this.#reloading(sql);
560
+ else {
483
561
  await sql.unsafe(`DROP TABLE IF EXISTS ${this.#hidden}`);
562
+ await this.#reloaded(sql);
563
+ }
484
564
  await this.#view?.refuse(sql, target === 'stale');
485
565
  if (decided === 'evolve') {
486
566
  await this.#evolve(sql, stage, load);
@@ -491,6 +571,11 @@ export class PostgresWriter extends Writer {
491
571
  return decided;
492
572
  });
493
573
  const fresh = mode === 'create' || mode === 'reload';
574
+ // The reload as the last commit left it, which a discard goes back to.
575
+ let committed = {
576
+ mode,
577
+ reloads,
578
+ };
494
579
  const open = () => mode === 'reload' || mode === 'continue';
495
580
  const into = () => (open() ? this.#hidden : this.qualifiedName);
496
581
  const tables = () => [
@@ -565,11 +650,23 @@ export class PostgresWriter extends Writer {
565
650
  // The whole stream starts over in a new hidden target.
566
651
  await drop();
567
652
  mode = 'reload';
653
+ reloads = wholeStream;
568
654
  return;
569
655
  }
570
656
  const [rows, values] = this.#scope(partition);
571
657
  await sql.unsafe(`DELETE FROM ${stage}${rows}`, values);
572
658
  resets.push(partition);
659
+ // The partition starts over in a hidden target, and readers see its
660
+ // rows as they were until the stream ends. An overwrite replaces
661
+ // the whole target anyway.
662
+ if (this.replaces)
663
+ return;
664
+ if (mode === 'load') {
665
+ mode = 'reload';
666
+ reloads = new SomePartitions([partition]);
667
+ }
668
+ else if (open() && reloads instanceof SomePartitions)
669
+ reloads = reloads.with(partition);
573
670
  return;
574
671
  }
575
672
  // Each file is read here, before the source advances past it. The
@@ -609,6 +706,8 @@ export class PostgresWriter extends Writer {
609
706
  await sql.unsafe(`CREATE TABLE ${this.#tableSQL(this.#hidden)}`);
610
707
  await this.build(sql, this.#hidden);
611
708
  }
709
+ if (open())
710
+ await this.#reload(sql, reloads);
612
711
  const replacing = await claiming.claim(sql, committedIn);
613
712
  // The rows of a hidden target were never shown, so only the target
614
713
  // keeps the rows a commit replaces.
@@ -625,6 +724,18 @@ export class PostgresWriter extends Writer {
625
724
  const written = { loadedAt: load.loadedAt, committedIn };
626
725
  if (replacing)
627
726
  await this.#replace(sql, stage, written);
727
+ else if (open() && reloads instanceof SomePartitions) {
728
+ // The rows of the reloaded partitions go to the hidden target,
729
+ // the others to the target. A DELETE whose key does not name
730
+ // its partition goes to both, so the reload cannot bring back a
731
+ // row that the source deleted.
732
+ const [within, values] = this.#within(reloads.partitions);
733
+ await sql.unsafe(`INSERT INTO ${reloadStage} SELECT * FROM ${stage} WHERE (${within}) IS TRUE OR (${op} = 'D' AND (${within}) IS NULL)`, values);
734
+ await sql.unsafe(`DELETE FROM ${stage} WHERE (${within}) IS TRUE`, values);
735
+ await this.merge(sql, stage, written, this.qualifiedName, this.#history);
736
+ await this.merge(sql, reloadStage, written, this.#hidden, null);
737
+ await sql.unsafe(`TRUNCATE ${reloadStage}`);
738
+ }
628
739
  else
629
740
  await this.merge(sql, stage, written, into(), history);
630
741
  await sql.unsafe(`TRUNCATE ${stage}`);
@@ -636,6 +747,7 @@ export class PostgresWriter extends Writer {
636
747
  // Its first commit made the hidden target, so the reload continues.
637
748
  if (mode === 'reload')
638
749
  mode = 'continue';
750
+ committed = { mode, reloads };
639
751
  resets = [];
640
752
  },
641
753
  complete: async () => {
@@ -643,10 +755,17 @@ export class PostgresWriter extends Writer {
643
755
  return;
644
756
  await load.commit(async (sql, committedIn) => {
645
757
  const target = await storedFit(sql, this.table, this.qualifiedName, stage);
646
- if (target === 'fits')
758
+ // A reload of partitions keeps the target's shape: prepare evolves
759
+ // a stored target before it loads.
760
+ if (reloads instanceof SomePartitions && target === 'fits')
761
+ await this.#reloadPartitionsInPlace(sql, reloads.partitions, committedIn);
762
+ else if (reloads instanceof SomePartitions)
763
+ throw new TypeError(`${this.qualifiedName} changed its shape while some of its partitions reloaded; clear the copy to load it again`);
764
+ else if (target === 'fits')
647
765
  await this.#reloadInPlace(sql, committedIn);
648
766
  else
649
767
  await this.#swap(sql, committedIn);
768
+ await this.#reloaded(sql);
650
769
  await this.#history.expire(sql);
651
770
  for (const store of stores)
652
771
  await store.prune(sql, [
@@ -655,14 +774,40 @@ export class PostgresWriter extends Writer {
655
774
  ]);
656
775
  });
657
776
  mode = 'load';
777
+ reloads = wholeStream;
778
+ committed = { mode, reloads };
779
+ },
780
+ // A RESET that no commit took is undone with the operations after it:
781
+ // the source's checkpoint does not know it.
782
+ discard: async () => {
783
+ await drop();
784
+ ({ mode, reloads } = committed);
658
785
  },
659
- discard: drop,
660
786
  [Symbol.asyncDispose]: async () => {
661
787
  await drop();
662
788
  await sql.unsafe(`DROP TABLE pg_temp.${stage}`);
789
+ await sql.unsafe(`DROP TABLE pg_temp.${reloadStage}`);
663
790
  for (const store of stores)
664
791
  await store.unstage(sql);
665
792
  },
666
793
  };
667
794
  }
668
795
  }
796
+ // What an open reload reloads: the whole stream, or some partitions of it,
797
+ // while the others load into the target as before, as Delta Lake's
798
+ // replaceWhere replaces only the rows that its condition selects.
799
+ class WholeStream {
800
+ }
801
+ class SomePartitions {
802
+ partitions;
803
+ constructor(partitions) {
804
+ this.partitions = Object.freeze([...partitions]);
805
+ }
806
+ // These partitions and partition, which a later RESET reloads too.
807
+ with(partition) {
808
+ return this.partitions.some((other) => isDeepStrictEqual(other, partition))
809
+ ? this
810
+ : new SomePartitions([...this.partitions, partition]);
811
+ }
812
+ }
813
+ const wholeStream = new WholeStream();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@eltjs/elt-postgresql",
3
- "version": "0.3.14",
3
+ "version": "0.3.15",
4
4
  "description": "The Postgres destination of elt: each stream is a typed table in one schema, with a reader view in another, a history of each row for readings as of a position, and the views and computed datasets that a pipeline derives.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -37,7 +37,7 @@
37
37
  "access": "public"
38
38
  },
39
39
  "dependencies": {
40
- "@eltjs/elt": "^0.3.14",
40
+ "@eltjs/elt": "^0.3.15",
41
41
  "@zukhruf/async": "^0.8.27",
42
42
  "postgres": "^3.4.9"
43
43
  },