pgsync 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 531d7975b4942abc5670b8c4ad43717da1326eacacc3b95f5d5797236cb82bb3
-  data.tar.gz: e32dfb82469fa2ebeeea844f0a09e87b8068bd7a201111b90d374eef779381af
+  metadata.gz: 0d3ff88829338544bfe3434a16a00640e6496b7b89b1ef5dd43e66f844ea51ce
+  data.tar.gz: d5457e8c2596fdda651c9739712fe0934af14028295ea9cd6addfc70438ced16
 SHA512:
-  metadata.gz: 4a33a680baaf706ad25fed35eaee3a8c3b191c7a5f6600b4ada42e55143a12226b0e1be189cf4c29d4866982e4f8c7b4915c4517375913083f3210f5fb131598
-  data.tar.gz: a3825c52c5e96c869ea06ed0ec969e1835aec5614f8bd1eb9255995972bdb3e526dd2996802e4d467b1497cfe371509b8a973cc8e3b61988d55f7fb0824d5293
+  metadata.gz: 6f9549d0d85cb502b5b88a3178fe617bf94f032d72787ab6dcf8f6097dc256ca2e969e2c4a434ebf33e4de17a4f11c590c57ef8cae420b20fe4f39bfed795baa
+  data.tar.gz: b10ac04af7dc26c3067c2ff73f163c69d73dfb5c1f559585adcf774312ac78ac07dcd790e1706cc571a90f0e4573060cedf01fc6ebd8b1732fbaa16b486213dd

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.3 (2020-06-09)
+
+- Added `--defer-constraints-v2` option
+- Ensure consistent source snapshot with `--disable-integrity`
+
 ## 0.6.2 (2020-06-09)
 
 - Added support for `--disable-integrity` on Amazon RDS

data/README.md

@@ -198,20 +198,20 @@ Rules starting with `unique_` require the table to have a single column primary
 
 Foreign keys can make it difficult to sync data. Three options are:
 
-1. Manually specify the order of tables
-2. Use deferrable constraints
-3. Disable foreign key triggers, which can silently break referential integrity
+1. Defer constraints (recommended)
+2. Manually specify the order of tables
+3. Disable foreign key triggers, which can silently break referential integrity (not recommended)
 
-When manually specifying the order, use `--jobs 1` so tables are synced one-at-a-time.
+To defer constraints, use:
 
 ```sh
-pgsync table1,table2,table3 --jobs 1
+pgsync --defer-constraints-v2
 ```
 
-If your tables have [deferrable constraints](https://begriffs.com/posts/2017-08-27-deferrable-sql-constraints.html), use:
+To manually specify the order of tables, use `--jobs 1` so tables are synced one-at-a-time.
 
 ```sh
-pgsync --defer-constraints
+pgsync table1,table2,table3 --jobs 1
 ```
 
 To disable foreign key triggers and potentially break referential integrity, use:

data/lib/pgsync/client.rb

@@ -67,7 +67,8 @@ Options:}
       o.integer "--batch-size", "batch size", default: 10000, help: false
       o.float "--sleep", "sleep", default: 0, help: false
       o.boolean "--fail-fast", "stop on the first failed table", default: false
-      o.boolean "--defer-constraints", "defer constraints", default: false
+      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
+      o.boolean "--defer-constraints-v2", "defer constraints", default: false
       o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
       o.boolean "--disable-integrity", "disable foreign key triggers", default: false
       # private, for testing

data/lib/pgsync/table_sync.rb

@@ -141,7 +141,7 @@ module PgSync
       options = {start: start, finish: finish}
 
       jobs = opts[:jobs]
-      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints]
+      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
         warning "--jobs ignored" if jobs
         jobs = 0
       end
@@ -172,8 +172,23 @@ module PgSync
     end
 
     def maybe_defer_constraints
-      if opts[:defer_constraints]
+      if opts[:disable_integrity] || opts[:disable_integrity_v2]
+        # create a transaction on the source
+        # to ensure we get a consistent snapshot
+        source.transaction do
+          yield
+        end
+      elsif opts[:defer_constraints] || opts[:defer_constraints_v2]
         destination.transaction do
+          if opts[:defer_constraints_v2]
+            table_constraints = non_deferrable_constraints(destination)
+            table_constraints.each do |table, constraints|
+              constraints.each do |constraint|
+                destination.execute("ALTER TABLE #{quote_ident_full(table)} ALTER CONSTRAINT #{quote_ident(constraint)} DEFERRABLE")
+              end
+            end
+          end
+
           destination.execute("SET CONSTRAINTS ALL DEFERRED")
 
           # create a transaction on the source
@@ -181,6 +196,20 @@ module PgSync
           source.transaction do
             yield
           end
+
+          # set them back
+          # there are 3 modes: DEFERRABLE INITIALLY DEFERRED, DEFERRABLE INITIALLY IMMEDIATE, and NOT DEFERRABLE
+          # we only update NOT DEFERRABLE
+          # https://www.postgresql.org/docs/current/sql-set-constraints.html
+          if opts[:defer_constraints_v2]
+            destination.execute("SET CONSTRAINTS ALL IMMEDIATE")
+
+            table_constraints.each do |table, constraints|
+              constraints.each do |constraint|
+                destination.execute("ALTER TABLE #{quote_ident_full(table)} ALTER CONSTRAINT #{quote_ident(constraint)} NOT DEFERRABLE")
+              end
+            end
+          end
         end
       else
         yield

data/lib/pgsync/task.rb

@@ -156,7 +156,7 @@ module PgSync
         destination.execute("INSERT INTO #{quoted_table} (SELECT * FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
       else
         # use delete instead of truncate for foreign keys
-        if opts[:defer_constraints]
+        if opts[:defer_constraints] || opts[:defer_constraints_v2]
           destination.execute("DELETE FROM #{quoted_table}")
         else
           destination.truncate(table)

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.6.2"
+  VERSION = "0.6.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Andrew Kane