pgsync 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa1085a96bdf17dcf1d5e240276faf9d26ee31a4c454e6bb042d9b6e919c6191
-  data.tar.gz: c1e3d4be165bf9365d8faedee4cf7177908c1f4565342467691842d376f96d98
+  metadata.gz: 32d1e60a133659dde98d3564d21fca4f1866e8f7b58518ef71f6dcaf59cf8c82
+  data.tar.gz: 650b50aee614ed737d2e99278cbb8910f7980a041dcf58d18288f6bc53826635
 SHA512:
-  metadata.gz: 68c7e7820c9e57e0f08f4f46030b9593b274bb1b731926d7c861d59424767c8dc650cf67a5849db463600392d36059fcfa451fe9c2e408213e35463c71f3cc6a
-  data.tar.gz: 66080f837d39782139fd59ca60edf7bc18cfc5f37e5f8b76ff8d5b2bfced3fbd2f542cf08775e78d06872eda0ed887cf64993a2b955260eee29b3ed8892e3686
+  metadata.gz: 8bd3e4373558c8f8519f0c118f51b55ac27a3a82fb1ab9a38e85bfee858e08a80fb2c37cab490878d2dda06121c13f45d81b3df6c87b8ede0e0060a50b94fee6
+  data.tar.gz: 922ecb8f57615025e5b92a93f82010bb667fbc968db12ebddeb49507ce90d6dd4c3c16ea3cd5777817bb321a8fefa54f6483b6ffae872f897e7ada64d68831b0

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.5.5 (2020-05-13)
+
+- Added `--jobs` option
+- Added experimental `--defer-constraints` option
+- Added experimental `--disable-user-triggers` option
+- Added experimental `--disable-integrity` option
+- Improved error message for older libpq
+
 ## 0.5.4 (2020-05-09)
 
 - Fixed output for `--in-batches`

data/README.md

@@ -65,7 +65,7 @@ Or truncate them
 pgsync products "where store_id = 1" --truncate
 ```
 
-### Exclude Tables
+## Exclude Tables
 
 ```sh
 pgsync --exclude users
@@ -79,7 +79,7 @@ exclude:
   - table2
 ```
 
-For Rails, you probably want to exclude schema migrations and ActiveRecord metadata.
+For Rails, you probably want to exclude schema migrations and Active Record metadata.
 
 ```yml
 exclude:
@@ -87,7 +87,7 @@ exclude:
   - ar_internal_metadata
 ```
 
-### Groups
+## Groups
 
 Define groups in `.pgsync.yml`:
 
@@ -123,7 +123,7 @@ And run:
 pgsync product:123
 ```
 
-### Schema
+## Schema
 
 **Note:** pgsync is designed to sync data. You should use a schema migration tool to manage schema changes. The methods in this section are provided for convenience but not recommended.
 
@@ -149,10 +149,6 @@ pgsync --schema-only
 
 pgsync does not try to sync Postgres extensions.
 
-## Data Protection
-
-Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
-
 ## Sensitive Information
 
 Prevent sensitive information like email addresses from leaving the remote server.
@@ -190,29 +186,49 @@ Options for replacement are:
 
 Rules starting with `unique_` require the table to have a primary key. `unique_phone` requires a numeric primary key.
 
-## Multiple Databases
+## Foreign Keys
 
-To use with multiple databases, run:
+Foreign keys can make it difficult to sync data. Three options are:
+
+1. Manually specify the order of tables
+2. Use deferrable constraints
+3. Disable triggers, which can silently break referential integrity
+
+When manually specifying the order, use `--jobs 1` so tables are synced one-at-a-time.
 
 ```sh
-pgsync --init db2
+pgsync table1,table2,table3 --jobs 1
 ```
 
-This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands with:
+If your tables have [deferrable constraints](https://begriffs.com/posts/2017-08-27-deferrable-sql-constraints.html), use:
 
 ```sh
-pgsync --db db2
+pgsync --defer-constraints
 ```
 
-## Safety
+**Note:** This feature is currently experimental.
 
-To keep you from accidentally overwriting production, the destination is limited to `localhost` or `127.0.0.1` by default.
+To disable triggers and potentially break referential integrity, use:
 
-To use another host, add `to_safe: true` to your `.pgsync.yml`.
+```sh
+pgsync --disable-integrity
+```
+
+**Note:** This feature is currently experimental.
+
+## Triggers
+
+Disable user triggers with:
+
+```sh
+pgsync --disable-user-triggers
+```
+
+**Note:** This feature is currently experimental.
 
-## Large Tables
+## Append-Only Tables
 
-For extremely large tables, sync in batches.
+For extremely large, append-only tables, sync in batches.
 
 ```sh
 pgsync large_table --in-batches
@@ -220,15 +236,31 @@ pgsync large_table --in-batches
 
 The script will resume where it left off when run again, making it great for backfills.
 
-## Foreign Keys
+## Connection Security
+
+Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
+
+## Safety
+
+To keep you from accidentally overwriting production, the destination is limited to `localhost` or `127.0.0.1` by default.
+
+To use another host, add `to_safe: true` to your `.pgsync.yml`.
 
-By default, tables are copied in parallel. If you use foreign keys, this can cause violations. You can specify tables to be copied serially with:
+## Multiple Databases
+
+To use with multiple databases, run:
 
 ```sh
-pgsync group1 --debug
+pgsync --init db2
 ```
 
-## Reference
+This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands with:
+
+```sh
+pgsync --db db2
+```
+
+## Other Commands
 
 Help
 
@@ -242,6 +274,12 @@ Version
 pgsync --version
 ```
 
+List tables
+
+```sh
+pgsync --list
+```
+
 ## Scripts
 
 Use groups when possible to take advantage of parallelism.

data/lib/pgsync/client.rb

@@ -43,6 +43,7 @@ Options:}
         o.string "-d", "--db", "database"
         o.string "-t", "--tables", "tables to sync"
         o.string "-g", "--groups", "groups to sync"
+        o.integer "-j", "--jobs", "number of tables to sync at a time"
         o.string "--schemas", "schemas to sync"
         o.string "--from", "source"
         o.string "--to", "destination"
@@ -68,6 +69,9 @@ Options:}
         o.integer "--batch-size", "batch size", default: 10000, help: false
         o.float "--sleep", "sleep", default: 0, help: false
         o.boolean "--fail-fast", "stop on the first failed table", default: false
+        o.boolean "--defer-constraints", "defer constraints", default: false
+        o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+        o.boolean "--disable-integrity", "disable foreign key triggers", default: false
         # o.array "--var", "pass a variable"
         o.boolean "-v", "--version", "print the version"
         o.boolean "-h", "--help", "prints help"

data/lib/pgsync/data_source.rb

@@ -2,9 +2,8 @@ module PgSync
   class DataSource
     attr_reader :url
 
-    def initialize(source, timeout: 3)
+    def initialize(source)
       @url = resolve_url(source)
-      @timeout = timeout
     end
 
     def exists?
@@ -85,10 +84,25 @@ module PgSync
       row && row["attname"]
     end
 
+    def triggers(table)
+      query = <<-SQL
+        SELECT
+          tgname AS name,
+          tgisinternal AS internal,
+          tgenabled != 'D' AS enabled,
+          tgconstraint != 0 AS integrity
+        FROM
+          pg_trigger
+        WHERE
+          pg_trigger.tgrelid = $1::regclass
+      SQL
+      execute(query, [quote_ident_full(table)])
+    end
+
     def conn
       @conn ||= begin
         begin
-          ENV["PGCONNECT_TIMEOUT"] ||= @timeout.to_s
+          ENV["PGCONNECT_TIMEOUT"] ||= "3"
           if @url =~ /\Apostgres(ql)?:\/\//
             config = @url
           else
@@ -108,6 +122,10 @@ module PgSync
       end
     end
 
+    def reconnect
+      @conn.reset
+    end
+
     def dump_command(tables)
       tables = tables ? tables.keys.map { |t| "-t #{Shellwords.escape(quote_ident_full(t))}" }.join(" ") : ""
       "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables} -d #{@url}"
@@ -132,6 +150,21 @@ module PgSync
       @search_path ||= execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
     end
 
+    def execute(query, params = [])
+      conn.exec_params(query, params).to_a
+    end
+
+    def transaction
+      if conn.transaction_status == 0
+        # not currently in transaction
+        conn.transaction do
+          yield
+        end
+      else
+        yield
+      end
+    end
+
     private
 
     def pg_restore_version
@@ -145,17 +178,18 @@ module PgSync
     end
 
     def conninfo
-      @conninfo ||= conn.conninfo_hash
+      @conninfo ||= begin
+        unless conn.respond_to?(:conninfo_hash)
+          raise Error, "libpq is too old. Upgrade it and run `gem install pg`"
+        end
+        conn.conninfo_hash
+      end
     end
 
     def quote_ident_full(ident)
       ident.split(".", 2).map { |v| quote_ident(v) }.join(".")
     end
 
-    def execute(query, params = [])
-      conn.exec_params(query, params).to_a
-    end
-
     def quote_ident(value)
       PG::Connection.quote_ident(value)
     end

data/lib/pgsync/sync.rb

@@ -12,7 +12,7 @@ module PgSync
         opts[opt] ||= config[opt.to_s]
       end
 
-      # TODO remove deprecations
+      # TODO remove deprecations in 0.6.0
       map_deprecations(args, opts)
 
       # start
@@ -22,35 +22,17 @@ module PgSync
         raise Error, "Usage:\n    pgsync [options]"
       end
 
-      source = DataSource.new(opts[:from])
       raise Error, "No source" unless source.exists?
-
-      destination = DataSource.new(opts[:to])
       raise Error, "No destination" unless destination.exists?
 
-      begin
-        # start connections
-        source.host
-        destination.host
-
-        unless opts[:to_safe] || destination.local?
-          raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
-        end
-
-        print_description("From", source)
-        print_description("To", destination)
-      ensure
-        source.close
-        destination.close
+      unless opts[:to_safe] || destination.local?
+        raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
       end
 
-      tables = nil
-      begin
-        tables = TableList.new(args, opts, source, config).tables
-      ensure
-        source.close
-      end
+      print_description("From", source)
+      print_description("To", destination)
 
+      tables = TableList.new(args, opts, source, config).tables
       confirm_tables_exist(source, tables, "source")
 
       if opts[:list]
@@ -78,8 +60,8 @@ module PgSync
         unless opts[:schema_only]
           confirm_tables_exist(destination, tables, "destination")
 
-          in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts|
-            TableSync.new.sync(config, table, opts.merge(table_opts), source.url, destination.url)
+          in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts, source, destination|
+            TableSync.new(source: source, destination: destination).sync(config, table, opts.merge(table_opts))
           end
         end
 
@@ -93,8 +75,6 @@ module PgSync
           raise Error, "Table does not exist in #{description}: #{table}"
         end
       end
-    ensure
-      data_source.close
     end
 
     def map_deprecations(args, opts)
@@ -197,20 +177,56 @@ module PgSync
       end
 
       options = {start: start, finish: finish}
-      if @options[:debug] || @options[:in_batches]
-        options[:in_processes] = 0
+
+      jobs = @options[:jobs]
+      if @options[:debug] || @options[:in_batches] || @options[:defer_constraints]
+        warn "--jobs ignored" if jobs
+        jobs = 0
+      end
+
+      if windows?
+        options[:in_threads] = jobs || 4
       else
-        options[:in_threads] = 4 if windows?
+        options[:in_processes] = jobs if jobs
       end
 
-      # could try to use `raise Parallel::Kill` to fail faster with --fail-fast
-      # see `fast_faster` branch
-      # however, need to make sure connections are cleaned up properly
-      Parallel.each(tables, **options, &block)
+      maybe_defer_constraints do
+        # could try to use `raise Parallel::Kill` to fail faster with --fail-fast
+        # see `fast_faster` branch
+        # however, need to make sure connections are cleaned up properly
+        Parallel.each(tables, **options) do |table, table_opts|
+          # must reconnect for new thread or process
+          # TODO only reconnect first time
+          unless options[:in_processes] == 0
+            source.reconnect
+            destination.reconnect
+          end
+
+          # TODO warn if there are non-deferrable constraints on the table
+
+          yield table, table_opts, source, destination
+        end
+      end
 
       fail_sync(failed_tables) if failed_tables.any?
     end
 
+    def maybe_defer_constraints
+      if @options[:defer_constraints]
+        destination.transaction do
+          destination.execute("SET CONSTRAINTS ALL DEFERRED")
+
+          # create a transaction on the source
+          # to ensure we get a consistent snapshot
+          source.transaction do
+            yield
+          end
+        end
+      else
+        yield
+      end
+    end
+
     def fail_sync(failed_tables)
       raise Error, "Sync failed for #{failed_tables.size} table#{failed_tables.size == 1 ? nil : "s"}: #{failed_tables.join(", ")}"
     end
@@ -241,5 +257,24 @@ module PgSync
     def windows?
       Gem.win_platform?
     end
+
+    def source
+      @source ||= data_source(@options[:from])
+    end
+
+    def destination
+      @destination ||= data_source(@options[:to])
+    end
+
+    def data_source(url)
+      ds = DataSource.new(url)
+      ObjectSpace.define_finalizer(self, self.class.finalize(ds))
+      ds
+    end
+
+    def self.finalize(ds)
+      # must use proc instead of stabby lambda
+      proc { ds.close }
+    end
   end
 end

data/lib/pgsync/table_sync.rb

@@ -2,14 +2,21 @@ module PgSync
   class TableSync
     include Utils
 
-    def sync(config, table, opts, source_url, destination_url)
-      start_time = Time.now
+    attr_reader :source, :destination
+
+    def initialize(source:, destination:)
+      @source = source
+      @destination = destination
+    end
 
-      source = DataSource.new(source_url, timeout: 0)
-      destination = DataSource.new(destination_url, timeout: 0)
+    def sync(config, table, opts)
+      maybe_disable_triggers(table, opts) do
+        sync_data(config, table, opts)
+      end
+    end
 
-      from_connection = source.conn
-      to_connection = destination.conn
+    def sync_data(config, table, opts)
+      start_time = Time.now
 
       from_fields = source.columns(table)
       to_fields = destination.columns(table)
@@ -79,13 +86,7 @@ module PgSync
           batch_sql_clause = " #{sql_clause.length > 0 ? "#{sql_clause} AND" : "WHERE"} #{where}"
 
           batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{batch_sql_clause}) TO STDOUT"
-          to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
-            from_connection.copy_data batch_copy_to_command do
-              while (row = from_connection.get_copy_data)
-                to_connection.put_copy_data(row)
-              end
-            end
-          end
+          copy(batch_copy_to_command, dest_table: table, dest_fields: fields)
 
           starting_id += batch_size
           i += 1
@@ -99,38 +100,31 @@ module PgSync
 
         # create a temp table
         temp_table = "pgsync_#{rand(1_000_000_000)}"
-        to_connection.exec("CREATE TEMPORARY TABLE #{quote_ident_full(temp_table)} AS SELECT * FROM #{quote_ident_full(table)} WITH NO DATA")
+        destination.execute("CREATE TEMPORARY TABLE #{quote_ident_full(temp_table)} AS TABLE #{quote_ident_full(table)} WITH NO DATA")
 
         # load data
-        to_connection.copy_data "COPY #{quote_ident_full(temp_table)} (#{fields}) FROM STDIN" do
-          from_connection.copy_data copy_to_command do
-            while (row = from_connection.get_copy_data)
-              to_connection.put_copy_data(row)
-            end
-          end
-        end
+        copy(copy_to_command, dest_table: temp_table, dest_fields: fields)
 
         if opts[:preserve]
           # insert into
-          to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident_full(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident_full(table)} WHERE #{quote_ident_full(table)}.#{quote_ident(primary_key)} = #{quote_ident_full(temp_table)}.#{quote_ident(primary_key)}))")
+          destination.execute("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident_full(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident_full(table)} WHERE #{quote_ident_full(table)}.#{quote_ident(primary_key)} = #{quote_ident_full(temp_table)}.#{quote_ident(primary_key)}))")
         else
-          to_connection.transaction do
-            to_connection.exec("DELETE FROM #{quote_ident_full(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident_full(temp_table)})")
-            to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident(temp_table)})")
+          destination.transaction do
+            destination.execute("DELETE FROM #{quote_ident_full(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident_full(temp_table)})")
+            destination.execute("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident(temp_table)})")
           end
         end
       else
-        destination.truncate(table)
-        to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
-          from_connection.copy_data copy_to_command do
-            while (row = from_connection.get_copy_data)
-              to_connection.put_copy_data(row)
-            end
-          end
+        # use delete instead of truncate for foreign keys
+        if opts[:defer_constraints]
+          destination.execute("DELETE FROM #{quote_ident_full(table)}")
+        else
+          destination.truncate(table)
         end
+        copy(copy_to_command, dest_table: table, dest_fields: fields)
       end
       seq_values.each do |seq, value|
-        to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
+        destination.execute("SELECT setval(#{escape(seq)}, #{escape(value)})")
       end
 
       message = nil
@@ -153,13 +147,21 @@ module PgSync
         end
 
       {status: "error", message: message}
-    ensure
-      source.close if source
-      destination.close if destination
     end
 
     private
 
+    def copy(source_command, dest_table:, dest_fields:)
+      destination_command = "COPY #{quote_ident_full(dest_table)} (#{dest_fields}) FROM STDIN"
+      destination.conn.copy_data(destination_command) do
+        source.conn.copy_data(source_command) do
+          while (row = source.conn.get_copy_data)
+            destination.conn.put_copy_data(row)
+          end
+        end
+      end
+    end
+
     # TODO better performance
     def rule_match?(table, column, rule)
       regex = Regexp.new('\A' + Regexp.escape(rule).gsub('\*','[^\.]*') + '\z')
@@ -231,5 +233,43 @@ module PgSync
     def quote_string(s)
       s.gsub(/\\/, '\&\&').gsub(/'/, "''")
     end
+
+    def maybe_disable_triggers(table, opts)
+      if opts[:disable_integrity] || opts[:disable_user_triggers]
+        destination.transaction do
+          triggers = destination.triggers(table)
+          triggers.select! { |t| t["enabled"] == "t" }
+          internal_triggers, user_triggers = triggers.partition { |t| t["internal"] == "t" }
+          integrity_triggers = internal_triggers.select { |t| t["integrity"] == "t" }
+          restore_triggers = []
+
+          if opts[:disable_integrity]
+            integrity_triggers.each do |trigger|
+              destination.execute("ALTER TABLE #{quote_ident_full(table)} DISABLE TRIGGER #{quote_ident(trigger["name"])}")
+            end
+            restore_triggers.concat(integrity_triggers)
+          end
+
+          if opts[:disable_user_triggers]
+            # important!
+            # rely on Postgres to disable user triggers
+            # we don't want to accidentally disable non-user triggers if logic above is off
+            destination.execute("ALTER TABLE #{quote_ident_full(table)} DISABLE TRIGGER USER")
+            restore_triggers.concat(user_triggers)
+          end
+
+          result = yield
+
+          # restore triggers that were previously enabled
+          restore_triggers.each do |trigger|
+            destination.execute("ALTER TABLE #{quote_ident_full(table)} ENABLE TRIGGER #{quote_ident(trigger["name"])}")
+          end
+
+          result
+        end
+      else
+        yield
+      end
+    end
   end
 end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.5.4"
+  VERSION = "0.5.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-10 00:00:00.000000000 Z
+date: 2020-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel