pgsync 0.6.0 → 0.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6c20cf99ebc0961d8699228883f6c2e9c95c9798cfb6f101cb78803df6b1b43
-  data.tar.gz: 7c719442c07e6db1d4704199aa2b95d32e1ba528797b2d21b15be85f2de71d79
+  metadata.gz: e3ef4779b0d0962cfb0a23bb8e1ec4e6a03923e1324a77280499d3c023aa7673
+  data.tar.gz: 59aa2f0868e3a1a920547326a49b86460ab2d8349eb0be68989899ca3bf7ada2
 SHA512:
-  metadata.gz: 1e3d735e6e3002e2fcb74de574f63f534472821aca0154c13d9d7ce6e9fa94426239cef875cd33724fec5a6cf2567b5656a80a9d92f89d9678ee6bee2c2ec70c
-  data.tar.gz: 3ab5d416f4c2364644c015d106631898a3615cc671179f8b1193cda5b1e894512fb39906ec1c27570ca9fbd186c9eb19af3afb88eba093bb21e8bc471c28f18b
+  metadata.gz: 9c6761c05533930479d39fcc318bd1c2601afa06ba50379c94bdaa69e3751fa3229559aa31806778cf17bdf79af0510bdbcb5d0c33dd6e4fdb1e164cda9ea49f
+  data.tar.gz: f8509b918137d97f80044ecac181a9499c128891d33467d91f2f74e6e49248f2ebbacee8bf6233c959c923d583ef8e14cd76b25d19b6b58c0d4166a7cc81af56

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+## 0.6.5 (2020-07-10)
+
+- Improved help
+
+## 0.6.4 (2020-06-10)
+
+- Log SQL with `--debug` option
+- Improved sequence queries
+
+## 0.6.3 (2020-06-09)
+
+- Added `--defer-constraints-v2` option
+- Ensure consistent source snapshot with `--disable-integrity`
+
+## 0.6.2 (2020-06-09)
+
+- Added support for `--disable-integrity` on Amazon RDS
+- Fixed error when excluded table not found in source
+
+## 0.6.1 (2020-06-07)
+
+- Added Django and Laravel integrations
+
 ## 0.6.0 (2020-06-07)
 
 - Added messages for different column types and non-deferrable constraints

data/README.md

@@ -35,7 +35,7 @@ This creates `.pgsync.yml` for you to customize. We recommend checking this into
 
 First, make sure your schema is set up in both databases. We recommend using a schema migration tool for this, but pgsync also provides a few [convenience methods](#schema). Once that’s done, you’re ready to sync data.
 
-Sync all tables
+Sync tables
 
 ```sh
 pgsync
@@ -198,20 +198,20 @@ Rules starting with `unique_` require the table to have a single column primary
 
 Foreign keys can make it difficult to sync data. Three options are:
 
-1. Manually specify the order of tables
-2. Use deferrable constraints
-3. Disable foreign key triggers, which can silently break referential integrity
+1. Defer constraints (recommended)
+2. Manually specify the order of tables
+3. Disable foreign key triggers, which can silently break referential integrity (not recommended)
 
-When manually specifying the order, use `--jobs 1` so tables are synced one-at-a-time.
+To defer constraints, use:
 
 ```sh
-pgsync table1,table2,table3 --jobs 1
+pgsync --defer-constraints-v2
 ```
 
-If your tables have [deferrable constraints](https://begriffs.com/posts/2017-08-27-deferrable-sql-constraints.html), use:
+To manually specify the order of tables, use `--jobs 1` so tables are synced one-at-a-time.
 
 ```sh
-pgsync --defer-constraints
+pgsync table1,table2,table3 --jobs 1
 ```
 
 To disable foreign key triggers and potentially break referential integrity, use:
@@ -220,6 +220,8 @@ To disable foreign key triggers and potentially break referential integrity, use
 pgsync --disable-integrity
 ```
 
+This requires superuser privileges on the `to` database. If syncing to (not from) Amazon RDS, use the `rds_superuser` role. If syncing to (not from) Heroku, there doesn’t appear to be a way to disable integrity.
+
 ## Triggers
 
 Disable user triggers with:
@@ -262,6 +264,57 @@ This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands w
 pgsync --db db2
 ```
 
+## Integrations
+
+- [Django](#django)
+- [Heroku](#heroku)
+- [Laravel](#laravel)
+- [Rails](#rails)
+
+### Django
+
+If you run `pgsync --init` in a Django project, migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - django_migrations
+```
+
+### Heroku
+
+If you run `pgsync --init` in a Heroku project, the `from` database will be set in `.pgsync.yml`.
+
+```yml
+from: $(heroku config:get DATABASE_URL)?sslmode=require
+```
+
+### Laravel
+
+If you run `pgsync --init` in a Laravel project, migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - migrations
+```
+
+### Rails
+
+If you run `pgsync --init` in a Rails project, Active Record metadata and schema migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - ar_internal_metadata
+  - schema_migrations
+```
+
+## Debugging
+
+To view the SQL that’s run, use:
+
+```sh
+pgsync --debug
+```
+
 ## Other Commands
 
 Help
@@ -337,6 +390,10 @@ Also check out:
 
 Inspired by [heroku-pg-transfer](https://github.com/ddollar/heroku-pg-transfer).
 
+## History
+
+View the [changelog](https://github.com/ankane/pgsync/blob/master/CHANGELOG.md)
+
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:

data/lib/pgsync.rb

@@ -18,6 +18,7 @@ require "pgsync/client"
 require "pgsync/data_source"
 require "pgsync/init"
 require "pgsync/schema_sync"
+require "pgsync/sequence"
 require "pgsync/sync"
 require "pgsync/table"
 require "pgsync/table_sync"

data/lib/pgsync/client.rb

@@ -39,39 +39,74 @@ module PgSync
     def slop_options
       o = Slop::Options.new
       o.banner = %{Usage:
-  pgsync [options]
+    pgsync [tables,groups] [sql] [options]}
 
-Options:}
-      o.string "-d", "--db", "database"
-      o.string "-t", "--tables", "tables to sync"
-      o.string "-g", "--groups", "groups to sync"
-      o.integer "-j", "--jobs", "number of tables to sync at a time"
+      # not shown
+      o.string "-t", "--tables", "tables to sync", help: false
+      o.string "-g", "--groups", "groups to sync", help: false
+
+      o.separator ""
+      o.separator "Table options:"
+      o.string "--exclude", "tables to exclude"
       o.string "--schemas", "schemas to sync"
-      o.string "--from", "source"
-      o.string "--to", "destination"
-      o.string "--exclude", "exclude tables"
-      o.string "--config", "config file"
-      o.boolean "--to-safe", "accept danger", default: false
-      o.boolean "--debug", "debug", default: false
-      o.boolean "--list", "list", default: false
-      o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
+      o.boolean "--all-schemas", "sync all schemas", default: false
+
+      o.separator ""
+      o.separator "Row options:"
+      o.boolean "--overwrite", "overwrite existing rows", default: false
       o.boolean "--preserve", "preserve existing rows", default: false
       o.boolean "--truncate", "truncate existing rows", default: false
-      o.boolean "--schema-first", "schema first", default: false
-      o.boolean "--schema-only", "schema only", default: false
-      o.boolean "--all-schemas", "all schemas", default: false
-      o.boolean "--no-rules", "do not apply data rules", default: false
-      o.boolean "--no-sequences", "do not sync sequences", default: false
-      o.boolean "--init", "init", default: false
-      o.boolean "--in-batches", "in batches", default: false, help: false
-      o.integer "--batch-size", "batch size", default: 10000, help: false
-      o.float "--sleep", "sleep", default: 0, help: false
-      o.boolean "--fail-fast", "stop on the first failed table", default: false
-      o.boolean "--defer-constraints", "defer constraints", default: false
-      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+
+      o.separator ""
+      o.separator "Foreign key options:"
+      o.boolean "--defer-constraints-v2", "defer constraints", default: false
       o.boolean "--disable-integrity", "disable foreign key triggers", default: false
-      o.boolean "-v", "--version", "print the version"
-      o.boolean "-h", "--help", "prints help"
+      o.integer "-j", "--jobs", "number of tables to sync at a time"
+
+      # replaced by v2
+      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
+      # private, for testing
+      o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
+
+      o.separator ""
+      o.separator "Schema options:"
+      o.boolean "--schema-first", "sync schema first", default: false
+      o.boolean "--schema-only", "sync schema only", default: false
+
+      o.separator ""
+      o.separator "Config options:"
+      # technically, defaults to searching path for .pgsync.yml, but this is simpler
+      o.string "--config", "config file (defaults to .pgsync.yml)"
+      o.string "-d", "--db", "database-specific config file"
+
+      o.separator ""
+      o.separator "Connection options:"
+      o.string "--from", "source database URL"
+      o.string "--to", "destination database URL"
+      o.boolean "--to-safe", "confirms destination is safe (when not localhost)", default: false
+
+      o.separator ""
+      o.separator "Other options:"
+      o.boolean "--debug", "show SQL statements", default: false
+      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+      o.boolean "--fail-fast", "stop on the first failed table", default: false
+      o.boolean "--no-rules", "don't apply data rules", default: false
+      o.boolean "--no-sequences", "don't sync sequences", default: false
+
+      # not shown in help
+      # o.separator ""
+      # o.separator "Append-only table options:"
+      o.boolean "--in-batches", "sync in batches", default: false, help: false
+      o.integer "--batch-size", "batch size", default: 10000, help: false
+      o.float "--sleep", "time to sleep between batches", default: 0, help: false
+
+      o.separator ""
+      o.separator "Other commands:"
+      o.boolean "--init", "create config file", default: false
+      o.boolean "--list", "list tables", default: false
+      o.boolean "-h", "--help", "print help"
+      o.boolean "-v", "--version", "print version"
+
       o
     end
   end

data/lib/pgsync/data_source.rb

@@ -4,8 +4,10 @@ module PgSync
 
     attr_reader :url
 
-    def initialize(url)
+    def initialize(url, name:, debug:)
       @url = url
+      @name = name
+      @debug = debug
     end
 
     def exists?
@@ -50,10 +52,6 @@ module PgSync
       table_set.include?(table)
     end
 
-    def sequences(table, columns)
-      execute("SELECT #{columns.map { |f| "pg_get_serial_sequence(#{escape("#{quote_ident_full(table)}")}, #{escape(f)}) AS #{quote_ident(f)}" }.join(", ")}").first.values.compact
-    end
-
     def max_id(table, primary_key, sql_clause = nil)
       execute("SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}").first["max"].to_i
     end
@@ -62,39 +60,14 @@ module PgSync
       execute("SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}").first["min"].to_i
     end
 
-    # this value comes from pg_get_serial_sequence which is already quoted
     def last_value(seq)
-      execute("SELECT last_value FROM #{seq}").first["last_value"]
+      execute("SELECT last_value FROM #{quote_ident_full(seq)}").first["last_value"]
     end
 
     def truncate(table)
       execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
     end
 
-    # https://stackoverflow.com/a/20537829
-    # TODO can simplify with array_position in Postgres 9.5+
-    def primary_key(table)
-      query = <<~SQL
-        SELECT
-          pg_attribute.attname,
-          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
-          pg_attribute.attnum,
-          pg_index.indkey
-        FROM
-          pg_index, pg_class, pg_attribute, pg_namespace
-        WHERE
-          nspname = $1 AND
-          relname = $2 AND
-          indrelid = pg_class.oid AND
-          pg_class.relnamespace = pg_namespace.oid AND
-          pg_attribute.attrelid = pg_class.oid AND
-          pg_attribute.attnum = any(pg_index.indkey) AND
-          indisprimary
-      SQL
-      rows = execute(query, [table.schema, table.name])
-      rows.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["attname"] }
-    end
-
     def triggers(table)
       query = <<~SQL
         SELECT
@@ -148,20 +121,34 @@ module PgSync
     end
 
     def execute(query, params = [])
+      log_sql query, params
       conn.exec_params(query, params).to_a
     end
 
     def transaction
       if conn.transaction_status == 0
         # not currently in transaction
-        conn.transaction do
-          yield
-        end
+        log_sql "BEGIN"
+        result =
+          conn.transaction do
+            yield
+          end
+        log_sql "COMMIT"
+        result
       else
         yield
       end
     end
 
+    # TODO log time for each statement
+    def log_sql(query, params = {})
+      if @debug
+        message = "#{colorize("[#{@name}]", :cyan)} #{query.gsub(/\s+/, " ").strip}"
+        message = "#{message} #{params.inspect}" if params.any?
+        log message
+      end
+    end
+
     private
 
     def concurrent_id

data/lib/pgsync/init.rb

@@ -30,8 +30,19 @@ module PgSync
           if rails?
             <<~EOS
               exclude:
-                - schema_migrations
                 - ar_internal_metadata
+                - schema_migrations
+            EOS
+          elsif django?
+            # TODO exclude other tables?
+            <<~EOS
+              exclude:
+                - django_migrations
+            EOS
+          elsif laravel?
+            <<~EOS
+              exclude:
+                - migrations
             EOS
           else
             <<~EOS
@@ -50,12 +61,30 @@ module PgSync
       end
     end
 
+    def django?
+      file_exists?("manage.py", /django/i)
+    end
+
     def heroku?
       `git remote -v 2>&1`.include?("git.heroku.com") rescue false
     end
 
+    def laravel?
+      file_exists?("artisan")
+    end
+
     def rails?
-      File.exist?("bin/rails")
+      file_exists?("bin/rails")
+    end
+
+    def file_exists?(path, contents = nil)
+      if contents
+        File.read(path).match(contents)
+      else
+        File.exist?(path)
+      end
+    rescue
+      false
     end
   end
 end

data/lib/pgsync/sequence.rb

@@ -0,0 +1,29 @@
+# minimal class to keep schema and sequence name separate
+module PgSync
+  class Sequence
+    attr_reader :schema, :name, :column
+
+    def initialize(schema, name, column:)
+      @schema = schema
+      @name = name
+      @column = column
+    end
+
+    def full_name
+      "#{schema}.#{name}"
+    end
+
+    def eql?(other)
+      other.schema == schema && other.name == name
+    end
+
+    # override hash when overriding eql?
+    def hash
+      [schema, name].hash
+    end
+
+    def to_s
+      full_name
+    end
+  end
+end

data/lib/pgsync/sync.rb

@@ -34,13 +34,13 @@ module PgSync
         raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
       end
 
+      print_description("From", source)
+      print_description("To", destination)
+
       if (opts[:preserve] || opts[:overwrite]) && destination.server_version_num < 90500
         raise Error, "Postgres 9.5+ is required for --preserve and --overwrite"
       end
 
-      print_description("From", source)
-      print_description("To", destination)
-
       resolver = TaskResolver.new(args: args, opts: opts, source: source, destination: destination, config: config, first_schema: first_schema)
       tasks =
         resolver.tasks.map do |task|
@@ -126,19 +126,20 @@ module PgSync
     end
 
     def source
-      @source ||= data_source(@options[:from])
+      @source ||= data_source(@options[:from], "from")
     end
 
     def destination
-      @destination ||= data_source(@options[:to])
+      @destination ||= data_source(@options[:to], "to")
     end
 
-    def data_source(url)
-      ds = DataSource.new(url)
+    def data_source(url, name)
+      ds = DataSource.new(url, name: name, debug: @options[:debug])
       ObjectSpace.define_finalizer(self, self.class.finalize(ds))
       ds
     end
 
+    # ideally aliases would work, but haven't found a nice way to do this
     def resolve_source(source)
       if source
         source = source.dup

data/lib/pgsync/table_sync.rb

@@ -17,6 +17,10 @@ module PgSync
 
       add_columns
 
+      add_primary_keys
+
+      add_sequences unless opts[:no_sequences]
+
       show_notes
 
       # don't sync tables with no shared fields
@@ -24,8 +28,6 @@ module PgSync
       run_tasks(tasks.reject { |task| task.shared_fields.empty? })
     end
 
-    # TODO only query specific tables
-    # TODO add sequences, primary keys, etc
     def add_columns
       source_columns = columns(source)
       destination_columns = columns(destination)
@@ -36,6 +38,79 @@ module PgSync
       end
     end
 
+    def add_primary_keys
+      destination_primary_keys = primary_keys(destination)
+
+      tasks.each do |task|
+        task.to_primary_key = destination_primary_keys[task.table] || []
+      end
+    end
+
+    def add_sequences
+      source_sequences = sequences(source)
+      destination_sequences = sequences(destination)
+
+      tasks.each do |task|
+        shared_columns = Set.new(task.shared_fields)
+
+        task.from_sequences = (source_sequences[task.table] || []).select { |s| shared_columns.include?(s.column) }
+        task.to_sequences = (destination_sequences[task.table] || []).select { |s| shared_columns.include?(s.column) }
+      end
+    end
+
+    def sequences(data_source)
+      query = <<~SQL
+        SELECT
+          nt.nspname as schema,
+          t.relname as table,
+          a.attname as column,
+          n.nspname as sequence_schema,
+          s.relname as sequence
+        FROM
+          pg_class s
+        INNER JOIN
+          pg_depend d ON d.objid = s.oid
+        INNER JOIN
+          pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
+        INNER JOIN
+          pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
+        INNER JOIN
+          pg_namespace n ON n.oid = s.relnamespace
+        INNER JOIN
+          pg_namespace nt ON nt.oid = t.relnamespace
+        WHERE
+          s.relkind = 'S'
+      SQL
+      data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|
+        [k, v.map { |r| Sequence.new(r["sequence_schema"], r["sequence"], column: r["column"]) }]
+      end.to_h
+    end
+
+    def primary_keys(data_source)
+      # https://stackoverflow.com/a/20537829
+      # TODO can simplify with array_position in Postgres 9.5+
+      query = <<~SQL
+        SELECT
+          nspname AS schema,
+          relname AS table,
+          pg_attribute.attname AS column,
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
+          pg_attribute.attnum,
+          pg_index.indkey
+        FROM
+          pg_index, pg_class, pg_attribute, pg_namespace
+        WHERE
+          indrelid = pg_class.oid AND
+          pg_class.relnamespace = pg_namespace.oid AND
+          pg_attribute.attrelid = pg_class.oid AND
+          pg_attribute.attnum = any(pg_index.indkey) AND
+          indisprimary
+      SQL
+      data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|
+        [k, v.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["column"] }]
+      end.to_h
+    end
+
     def show_notes
       # for tables
       resolver.notes.each do |note|
@@ -93,28 +168,33 @@ module PgSync
     def run_tasks(tasks, &block)
       notices = []
       failed_tables = []
-
-      spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
-      task_spinners = {}
       started_at = {}
 
+      show_spinners = output.tty? && !opts[:in_batches] && !opts[:debug]
+      if show_spinners
+        spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
+        task_spinners = {}
+      end
+
       start = lambda do |task, i|
         message = ":spinner #{display_item(task)}"
-        spinner = spinners.register(message)
-        if opts[:in_batches]
-          # log instead of spin for non-tty
-          log message.sub(":spinner", "⠋")
-        else
+
+        if show_spinners
+          spinner = spinners.register(message)
           spinner.auto_spin
+          task_spinners[task] = spinner
+        elsif opts[:in_batches]
+          log message.sub(":spinner", "⠋")
         end
-        task_spinners[task] = spinner
+
         started_at[task] = Time.now
       end
 
       finish = lambda do |task, i, result|
-        spinner = task_spinners[task]
         time = (Time.now - started_at[task]).round(1)
 
+        success = result[:status] == "success"
+
         message =
           if result[:message]
             "(#{result[:message].lines.first.to_s.strip})"
@@ -124,24 +204,31 @@ module PgSync
 
         notices.concat(result[:notices])
 
-        if result[:status] == "success"
-          spinner.success(message)
+        if show_spinners
+          spinner = task_spinners[task]
+          if success
+            spinner.success(message)
+          else
+            spinner.error(message)
+          end
         else
-          spinner.error(message)
-          failed_tables << task_name(task)
-          fail_sync(failed_tables) if opts[:fail_fast]
+          status = success ? "✔" : "✖"
+          log [status, display_item(task), message].join(" ")
         end
 
-        unless spinner.send(:tty?)
-          status = result[:status] == "success" ? "✔" : "✖"
-          log [status, display_item(task), message].join(" ")
+        unless success
+          failed_tables << task_name(task)
+          fail_sync(failed_tables) if opts[:fail_fast]
         end
       end
 
       options = {start: start, finish: finish}
 
       jobs = opts[:jobs]
-      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints]
+
+      # disable multiple jobs for defer constraints and disable integrity
+      # so we can use a transaction to ensure a consistent snapshot
+      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
         warning "--jobs ignored" if jobs
         jobs = 0
       end
@@ -171,9 +258,25 @@ module PgSync
       fail_sync(failed_tables) if failed_tables.any?
     end
 
+    # TODO add option to open transaction on source when manually specifying order of tables
     def maybe_defer_constraints
-      if opts[:defer_constraints]
+      if opts[:disable_integrity] || opts[:disable_integrity_v2]
+        # create a transaction on the source
+        # to ensure we get a consistent snapshot
+        source.transaction do
+          yield
+        end
+      elsif opts[:defer_constraints] || opts[:defer_constraints_v2]
         destination.transaction do
+          if opts[:defer_constraints_v2]
+            table_constraints = non_deferrable_constraints(destination)
+            table_constraints.each do |table, constraints|
+              constraints.each do |constraint|
+                destination.execute("ALTER TABLE #{quote_ident_full(table)} ALTER CONSTRAINT #{quote_ident(constraint)} DEFERRABLE")
+              end
+            end
+          end
+
           destination.execute("SET CONSTRAINTS ALL DEFERRED")
 
           # create a transaction on the source
@@ -181,6 +284,20 @@ module PgSync
           source.transaction do
             yield
           end
+
+          # set them back
+          # there are 3 modes: DEFERRABLE INITIALLY DEFERRED, DEFERRABLE INITIALLY IMMEDIATE, and NOT DEFERRABLE
+          # we only update NOT DEFERRABLE
+          # https://www.postgresql.org/docs/current/sql-set-constraints.html
+          if opts[:defer_constraints_v2]
+            destination.execute("SET CONSTRAINTS ALL IMMEDIATE")
+
+            table_constraints.each do |table, constraints|
+              constraints.each do |constraint|
+                destination.execute("ALTER TABLE #{quote_ident_full(table)} ALTER CONSTRAINT #{quote_ident(constraint)} NOT DEFERRABLE")
+              end
+            end
+          end
         end
       else
         yield

data/lib/pgsync/task.rb

@@ -3,7 +3,7 @@ module PgSync
     include Utils
 
     attr_reader :source, :destination, :config, :table, :opts
-    attr_accessor :from_columns, :to_columns
+    attr_accessor :from_columns, :to_columns, :from_sequences, :to_sequences, :to_primary_key
 
     def initialize(source:, destination:, config:, table:, opts:)
       @source = source
@@ -11,6 +11,8 @@ module PgSync
       @config = config
       @table = table
       @opts = opts
+      @from_sequences = []
+      @to_sequences = []
     end
 
     def quoted_table
@@ -39,14 +41,6 @@ module PgSync
       @shared_fields ||= to_fields & from_fields
     end
 
-    def from_sequences
-      @from_sequences ||= opts[:no_sequences] ? [] : source.sequences(table, shared_fields)
-    end
-
-    def to_sequences
-      @to_sequences ||= opts[:no_sequences] ? [] : destination.sequences(table, shared_fields)
-    end
-
     def shared_sequences
       @shared_sequences ||= to_sequences & from_sequences
     end
@@ -88,15 +82,10 @@ module PgSync
       sql_clause << " #{opts[:sql]}" if opts[:sql]
 
       bad_fields = opts[:no_rules] ? [] : config["data_rules"]
-      primary_key = destination.primary_key(table)
+      primary_key = to_primary_key
       copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, _| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, primary_key)} AS #{quote_ident(f)}" : "#{quoted_table}.#{quote_ident(f)}" }.join(", ")
       fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
 
-      seq_values = {}
-      shared_sequences.each do |seq|
-        seq_values[seq] = source.last_value(seq)
-      end
-
       copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quoted_table}#{sql_clause}) TO STDOUT"
       if opts[:in_batches]
         raise Error, "No primary key" if primary_key.empty?
@@ -156,15 +145,18 @@ module PgSync
         destination.execute("INSERT INTO #{quoted_table} (SELECT * FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
       else
         # use delete instead of truncate for foreign keys
-        if opts[:defer_constraints]
+        if opts[:defer_constraints] || opts[:defer_constraints_v2]
           destination.execute("DELETE FROM #{quoted_table}")
         else
           destination.truncate(table)
         end
         copy(copy_to_command, dest_table: table, dest_fields: fields)
       end
-      seq_values.each do |seq, value|
-        destination.execute("SELECT setval(#{escape(seq)}, #{escape(value)})")
+
+      # update sequences
+      shared_sequences.each do |seq|
+        value = source.last_value(seq)
+        destination.execute("SELECT setval(#{escape(quote_ident_full(seq))}, #{escape(value)})")
       end
 
       {status: "success"}
@@ -214,6 +206,10 @@ module PgSync
 
     def copy(source_command, dest_table:, dest_fields:)
       destination_command = "COPY #{quote_ident_full(dest_table)} (#{dest_fields}) FROM STDIN"
+
+      source.log_sql(source_command)
+      destination.log_sql(destination_command)
+
       destination.conn.copy_data(destination_command) do
         source.conn.copy_data(source_command) do
           while (row = source.conn.get_copy_data)
@@ -275,7 +271,7 @@ module PgSync
     end
 
     def maybe_disable_triggers
-      if opts[:disable_integrity] || opts[:disable_user_triggers]
+      if opts[:disable_integrity] || opts[:disable_integrity_v2] || opts[:disable_user_triggers]
         destination.transaction do
           triggers = destination.triggers(table)
           triggers.select! { |t| t["enabled"] == "t" }
@@ -283,7 +279,17 @@ module PgSync
           integrity_triggers = internal_triggers.select { |t| t["integrity"] == "t" }
           restore_triggers = []
 
-          if opts[:disable_integrity]
+          # both --disable-integrity options require superuser privileges
+          # however, only v2 works on Amazon RDS, which added specific support for it
+          # https://aws.amazon.com/about-aws/whats-new/2014/11/10/amazon-rds-postgresql-read-replicas/
+          #
+          # session_replication_role disables more than foreign keys (like triggers and rules)
+          # this is probably fine, but keep the current default for now
+          if opts[:disable_integrity_v2] || (opts[:disable_integrity] && rds?)
+            # SET LOCAL lasts until the end of the transaction
+            # https://www.postgresql.org/docs/current/sql-set.html
+            destination.execute("SET LOCAL session_replication_role = replica")
+          elsif opts[:disable_integrity]
             integrity_triggers.each do |trigger|
               destination.execute("ALTER TABLE #{quoted_table} DISABLE TRIGGER #{quote_ident(trigger["name"])}")
             end
@@ -311,5 +317,9 @@ module PgSync
         yield
       end
     end
+
+    def rds?
+      destination.execute("SELECT name, setting FROM pg_settings WHERE name LIKE 'rds.%'").any?
+    end
   end
 end

data/lib/pgsync/task_resolver.rb

@@ -148,7 +148,7 @@ module PgSync
           regex = Regexp.new('\A' + Regexp.escape(value).gsub('\*','[^\.]*') + '\z')
           tables.reject! { |t| regex.match(t.full_name) || regex.match(t.name) }
         else
-          tables -= [fully_resolve(to_table(value))]
+          tables -= [fully_resolve(to_table(value), error: false)].compact
         end
       end
 
@@ -181,9 +181,11 @@ module PgSync
     end
 
     # for tables without a schema, find the table in the search path
-    def fully_resolve(table)
+    def fully_resolve(table, error: true)
       return table if table.schema
-      no_schema_tables[table.name] || (raise Error, "Table not found in source: #{table.name}")
+      resolved_table = no_schema_tables[table.name]
+      raise Error, "Table not found in source: #{table.name}" if !resolved_table && error
+      resolved_table
     end
 
     # parse command line arguments and YAML

data/lib/pgsync/utils.rb

@@ -3,7 +3,8 @@ module PgSync
     COLOR_CODES = {
       red: 31,
       green: 32,
-      yellow: 33
+      yellow: 33,
+      cyan: 36
     }
 
     def log(message = nil)
@@ -59,7 +60,7 @@ module PgSync
     end
 
     def quote_ident_full(ident)
-      if ident.is_a?(Table)
+      if ident.is_a?(Table) || ident.is_a?(Sequence)
         [quote_ident(ident.schema), quote_ident(ident.name)].join(".")
       else # temp table names are strings
         quote_ident(ident)

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.6.0"
+  VERSION = "0.6.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-07 00:00:00.000000000 Z
+date: 2020-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.1
+        version: 4.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.1
+        version: 4.8.2
 - !ruby/object:Gem::Dependency
   name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
@@ -125,6 +125,7 @@ files:
 - lib/pgsync/data_source.rb
 - lib/pgsync/init.rb
 - lib/pgsync/schema_sync.rb
+- lib/pgsync/sequence.rb
 - lib/pgsync/sync.rb
 - lib/pgsync/table.rb
 - lib/pgsync/table_sync.rb