pgsync 0.6.2 → 0.6.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 531d7975b4942abc5670b8c4ad43717da1326eacacc3b95f5d5797236cb82bb3
-  data.tar.gz: e32dfb82469fa2ebeeea844f0a09e87b8068bd7a201111b90d374eef779381af
+  metadata.gz: 4af2ef537f2130fb3b60ce705c17dc0ea2292a6d12f074ac14dafdc18e5419e8
+  data.tar.gz: 441442b39578dd09b4cbe22a9514f3eda59b7876a8d0c1541eda39631ac9a757
 SHA512:
-  metadata.gz: 4a33a680baaf706ad25fed35eaee3a8c3b191c7a5f6600b4ada42e55143a12226b0e1be189cf4c29d4866982e4f8c7b4915c4517375913083f3210f5fb131598
-  data.tar.gz: a3825c52c5e96c869ea06ed0ec969e1835aec5614f8bd1eb9255995972bdb3e526dd2996802e4d467b1497cfe371509b8a973cc8e3b61988d55f7fb0824d5293
+  metadata.gz: '0357909b28157593f8fcc21678efa53c767454c41a2f858b577da9e802aa29d49ffd3f8f835554098d670d82b018e27d7299a091e41022fd1b98a7eb1529232a'
+  data.tar.gz: acb5faf68e112427b406b03285d1bf2aa1c42034a186c8a37b617670037f06156e8e8f5dbb222b5fac3ce97e2b6a9d69014be9ca9c9395230c0139028c91e194

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.6.7 (2021-04-26)
+
+- Fixed connection security for `--schema-first` and `--schema-only` - [more info](https://github.com/ankane/pgsync/issues/121)
+
+## 0.6.6 (2020-10-29)
+
+- Added support for tables with generated columns
+
+## 0.6.5 (2020-07-10)
+
+- Improved help
+
+## 0.6.4 (2020-06-10)
+
+- Log SQL with `--debug` option
+- Improved sequence queries
+
+## 0.6.3 (2020-06-09)
+
+- Added `--defer-constraints-v2` option
+- Ensure consistent source snapshot with `--disable-integrity`
+
 ## 0.6.2 (2020-06-09)
 
 - Added support for `--disable-integrity` on Amazon RDS

data/README.md

@@ -9,7 +9,7 @@ Sync data from one Postgres database to another (like `pg_dump`/`pg_restore`). D
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://travis-ci.org/ankane/pgsync.svg?branch=master)](https://travis-ci.org/ankane/pgsync)
+[![Build Status](https://github.com/ankane/pgsync/workflows/build/badge.svg?branch=master)](https://github.com/ankane/pgsync/actions)
 
 ## Installation
 
@@ -198,20 +198,20 @@ Rules starting with `unique_` require the table to have a single column primary
 
 Foreign keys can make it difficult to sync data. Three options are:
 
-1. Manually specify the order of tables
-2. Use deferrable constraints
-3. Disable foreign key triggers, which can silently break referential integrity
+1. Defer constraints (recommended)
+2. Manually specify the order of tables
+3. Disable foreign key triggers, which can silently break referential integrity (not recommended)
 
-When manually specifying the order, use `--jobs 1` so tables are synced one-at-a-time.
+To defer constraints, use:
 
 ```sh
-pgsync table1,table2,table3 --jobs 1
+pgsync --defer-constraints-v2
 ```
 
-If your tables have [deferrable constraints](https://begriffs.com/posts/2017-08-27-deferrable-sql-constraints.html), use:
+To manually specify the order of tables, use `--jobs 1` so tables are synced one-at-a-time.
 
 ```sh
-pgsync --defer-constraints
+pgsync table1,table2,table3 --jobs 1
 ```
 
 To disable foreign key triggers and potentially break referential integrity, use:
@@ -307,6 +307,14 @@ exclude:
   - schema_migrations
 ```
 
+## Debugging
+
+To view the SQL that’s run, use:
+
+```sh
+pgsync --debug
+```
+
 ## Other Commands
 
 Help
@@ -382,6 +390,10 @@ Also check out:
 
 Inspired by [heroku-pg-transfer](https://github.com/ddollar/heroku-pg-transfer).
 
+## History
+
+View the [changelog](https://github.com/ankane/pgsync/blob/master/CHANGELOG.md)
+
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:

data/lib/pgsync.rb

@@ -18,6 +18,7 @@ require "pgsync/client"
 require "pgsync/data_source"
 require "pgsync/init"
 require "pgsync/schema_sync"
+require "pgsync/sequence"
 require "pgsync/sync"
 require "pgsync/table"
 require "pgsync/table_sync"

data/lib/pgsync/client.rb

@@ -39,41 +39,74 @@ module PgSync
     def slop_options
       o = Slop::Options.new
       o.banner = %{Usage:
-  pgsync [options]
+    pgsync [tables,groups] [sql] [options]}
 
-Options:}
-      o.string "-d", "--db", "database"
-      o.string "-t", "--tables", "tables to sync"
-      o.string "-g", "--groups", "groups to sync"
-      o.integer "-j", "--jobs", "number of tables to sync at a time"
+      # not shown
+      o.string "-t", "--tables", "tables to sync", help: false
+      o.string "-g", "--groups", "groups to sync", help: false
+
+      o.separator ""
+      o.separator "Table options:"
+      o.string "--exclude", "tables to exclude"
       o.string "--schemas", "schemas to sync"
-      o.string "--from", "source"
-      o.string "--to", "destination"
-      o.string "--exclude", "exclude tables"
-      o.string "--config", "config file"
-      o.boolean "--to-safe", "accept danger", default: false
-      o.boolean "--debug", "debug", default: false
-      o.boolean "--list", "list", default: false
-      o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
+      o.boolean "--all-schemas", "sync all schemas", default: false
+
+      o.separator ""
+      o.separator "Row options:"
+      o.boolean "--overwrite", "overwrite existing rows", default: false
       o.boolean "--preserve", "preserve existing rows", default: false
       o.boolean "--truncate", "truncate existing rows", default: false
-      o.boolean "--schema-first", "schema first", default: false
-      o.boolean "--schema-only", "schema only", default: false
-      o.boolean "--all-schemas", "all schemas", default: false
-      o.boolean "--no-rules", "do not apply data rules", default: false
-      o.boolean "--no-sequences", "do not sync sequences", default: false
-      o.boolean "--init", "init", default: false
-      o.boolean "--in-batches", "in batches", default: false, help: false
-      o.integer "--batch-size", "batch size", default: 10000, help: false
-      o.float "--sleep", "sleep", default: 0, help: false
-      o.boolean "--fail-fast", "stop on the first failed table", default: false
-      o.boolean "--defer-constraints", "defer constraints", default: false
-      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+
+      o.separator ""
+      o.separator "Foreign key options:"
+      o.boolean "--defer-constraints-v2", "defer constraints", default: false
       o.boolean "--disable-integrity", "disable foreign key triggers", default: false
+      o.integer "-j", "--jobs", "number of tables to sync at a time"
+
+      # replaced by v2
+      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
       # private, for testing
       o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
-      o.boolean "-v", "--version", "print the version"
-      o.boolean "-h", "--help", "prints help"
+
+      o.separator ""
+      o.separator "Schema options:"
+      o.boolean "--schema-first", "sync schema first", default: false
+      o.boolean "--schema-only", "sync schema only", default: false
+
+      o.separator ""
+      o.separator "Config options:"
+      # technically, defaults to searching path for .pgsync.yml, but this is simpler
+      o.string "--config", "config file (defaults to .pgsync.yml)"
+      o.string "-d", "--db", "database-specific config file"
+
+      o.separator ""
+      o.separator "Connection options:"
+      o.string "--from", "source database URL"
+      o.string "--to", "destination database URL"
+      o.boolean "--to-safe", "confirms destination is safe (when not localhost)", default: false
+
+      o.separator ""
+      o.separator "Other options:"
+      o.boolean "--debug", "show SQL statements", default: false
+      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+      o.boolean "--fail-fast", "stop on the first failed table", default: false
+      o.boolean "--no-rules", "don't apply data rules", default: false
+      o.boolean "--no-sequences", "don't sync sequences", default: false
+
+      # not shown in help
+      # o.separator ""
+      # o.separator "Append-only table options:"
+      o.boolean "--in-batches", "sync in batches", default: false, help: false
+      o.integer "--batch-size", "batch size", default: 10000, help: false
+      o.float "--sleep", "time to sleep between batches", default: 0, help: false
+
+      o.separator ""
+      o.separator "Other commands:"
+      o.boolean "--init", "create config file", default: false
+      o.boolean "--list", "list tables", default: false
+      o.boolean "-h", "--help", "print help"
+      o.boolean "-v", "--version", "print version"
+
       o
     end
   end

data/lib/pgsync/data_source.rb

@@ -4,8 +4,10 @@ module PgSync
 
     attr_reader :url
 
-    def initialize(url)
+    def initialize(url, name:, debug:)
       @url = url
+      @name = name
+      @debug = debug
     end
 
     def exists?
@@ -50,10 +52,6 @@ module PgSync
       table_set.include?(table)
     end
 
-    def sequences(table, columns)
-      execute("SELECT #{columns.map { |f| "pg_get_serial_sequence(#{escape("#{quote_ident_full(table)}")}, #{escape(f)}) AS #{quote_ident(f)}" }.join(", ")}").first.values.compact
-    end
-
     def max_id(table, primary_key, sql_clause = nil)
       execute("SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}").first["max"].to_i
     end
@@ -62,39 +60,14 @@ module PgSync
       execute("SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}").first["min"].to_i
     end
 
-    # this value comes from pg_get_serial_sequence which is already quoted
     def last_value(seq)
-      execute("SELECT last_value FROM #{seq}").first["last_value"]
+      execute("SELECT last_value FROM #{quote_ident_full(seq)}").first["last_value"]
     end
 
     def truncate(table)
       execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
     end
 
-    # https://stackoverflow.com/a/20537829
-    # TODO can simplify with array_position in Postgres 9.5+
-    def primary_key(table)
-      query = <<~SQL
-        SELECT
-          pg_attribute.attname,
-          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
-          pg_attribute.attnum,
-          pg_index.indkey
-        FROM
-          pg_index, pg_class, pg_attribute, pg_namespace
-        WHERE
-          nspname = $1 AND
-          relname = $2 AND
-          indrelid = pg_class.oid AND
-          pg_class.relnamespace = pg_namespace.oid AND
-          pg_attribute.attrelid = pg_class.oid AND
-          pg_attribute.attnum = any(pg_index.indkey) AND
-          indisprimary
-      SQL
-      rows = execute(query, [table.schema, table.name])
-      rows.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["attname"] }
-    end
-
     def triggers(table)
       query = <<~SQL
         SELECT
@@ -148,20 +121,34 @@ module PgSync
     end
 
     def execute(query, params = [])
+      log_sql query, params
       conn.exec_params(query, params).to_a
     end
 
     def transaction
       if conn.transaction_status == 0
         # not currently in transaction
-        conn.transaction do
-          yield
-        end
+        log_sql "BEGIN"
+        result =
+          conn.transaction do
+            yield
+          end
+        log_sql "COMMIT"
+        result
       else
         yield
       end
     end
 
+    # TODO log time for each statement
+    def log_sql(query, params = {})
+      if @debug
+        message = "#{colorize("[#{@name}]", :cyan)} #{query.gsub(/\s+/, " ").strip}"
+        message = "#{message} #{params.inspect}" if params.any?
+        log message
+      end
+    end
+
     private
 
     def concurrent_id

data/lib/pgsync/schema_sync.rb

@@ -27,7 +27,7 @@ module PgSync
       # if spinner, capture lines to show on error
       lines = []
       success =
-        run_command("#{dump_command} | #{restore_command}") do |line|
+        run_command do |line|
           if show_spinner
             lines << line
           else
@@ -49,12 +49,14 @@ module PgSync
 
     private
 
-    def run_command(command)
-      Open3.popen2e(command) do |stdin, stdout, wait_thr|
-        stdout.each do |line|
+    def run_command
+      err_r, err_w = IO.pipe
+      Open3.pipeline_start(dump_command, restore_command, err: err_w) do |wait_thrs|
+        err_w.close
+        err_r.each do |line|
           yield line
         end
-        wait_thr.value.success?
+        wait_thrs.all? { |t| t.value.success? }
       end
     end
 
@@ -65,19 +67,19 @@ module PgSync
     end
 
     def dump_command
-      tables =
-        if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
-          @tasks.map { |task| "-t #{Shellwords.escape(task.quoted_table)}" }
-        else
-          []
+      cmd = ["pg_dump", "-Fc", "--verbose", "--schema-only", "--no-owner", "--no-acl"]
+      if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
+        @tasks.each do |task|
+          cmd.concat(["-t", task.quoted_table])
         end
-
-      "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables.join(" ")} -d #{@source.url}"
+      end
+      cmd.concat(["-d", @source.url])
     end
 
     def restore_command
-      if_exists = Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
-      "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@destination.url}"
+      cmd = ["pg_restore", "--verbose", "--no-owner", "--no-acl", "--clean"]
+      cmd << "--if-exists" if Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
+      cmd.concat(["-d", @destination.url])
     end
   end
 end

data/lib/pgsync/sequence.rb

@@ -0,0 +1,29 @@
+# minimal class to keep schema and sequence name separate
+module PgSync
+  class Sequence
+    attr_reader :schema, :name, :column
+
+    def initialize(schema, name, column:)
+      @schema = schema
+      @name = name
+      @column = column
+    end
+
+    def full_name
+      "#{schema}.#{name}"
+    end
+
+    def eql?(other)
+      other.schema == schema && other.name == name
+    end
+
+    # override hash when overriding eql?
+    def hash
+      [schema, name].hash
+    end
+
+    def to_s
+      full_name
+    end
+  end
+end

data/lib/pgsync/sync.rb

@@ -34,13 +34,13 @@ module PgSync
         raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
       end
 
+      print_description("From", source)
+      print_description("To", destination)
+
       if (opts[:preserve] || opts[:overwrite]) && destination.server_version_num < 90500
         raise Error, "Postgres 9.5+ is required for --preserve and --overwrite"
       end
 
-      print_description("From", source)
-      print_description("To", destination)
-
       resolver = TaskResolver.new(args: args, opts: opts, source: source, destination: destination, config: config, first_schema: first_schema)
       tasks =
         resolver.tasks.map do |task|
@@ -126,15 +126,15 @@ module PgSync
     end
 
     def source
-      @source ||= data_source(@options[:from])
+      @source ||= data_source(@options[:from], "from")
     end
 
     def destination
-      @destination ||= data_source(@options[:to])
+      @destination ||= data_source(@options[:to], "to")
     end
 
-    def data_source(url)
-      ds = DataSource.new(url)
+    def data_source(url, name)
+      ds = DataSource.new(url, name: name, debug: @options[:debug])
       ObjectSpace.define_finalizer(self, self.class.finalize(ds))
       ds
     end

data/lib/pgsync/table_sync.rb

@@ -17,6 +17,10 @@ module PgSync
 
       add_columns
 
+      add_primary_keys
+
+      add_sequences unless opts[:no_sequences]
+
       show_notes
 
       # don't sync tables with no shared fields
@@ -24,8 +28,6 @@ module PgSync
       run_tasks(tasks.reject { |task| task.shared_fields.empty? })
     end
 
-    # TODO only query specific tables
-    # TODO add sequences, primary keys, etc
     def add_columns
       source_columns = columns(source)
       destination_columns = columns(destination)
@@ -36,6 +38,79 @@ module PgSync
       end
     end
 
+    def add_primary_keys
+      destination_primary_keys = primary_keys(destination)
+
+      tasks.each do |task|
+        task.to_primary_key = destination_primary_keys[task.table] || []
+      end
+    end
+
+    def add_sequences
+      source_sequences = sequences(source)
+      destination_sequences = sequences(destination)
+
+      tasks.each do |task|
+        shared_columns = Set.new(task.shared_fields)
+
+        task.from_sequences = (source_sequences[task.table] || []).select { |s| shared_columns.include?(s.column) }
+        task.to_sequences = (destination_sequences[task.table] || []).select { |s| shared_columns.include?(s.column) }
+      end
+    end
+
+    def sequences(data_source)
+      query = <<~SQL
+        SELECT
+          nt.nspname as schema,
+          t.relname as table,
+          a.attname as column,
+          n.nspname as sequence_schema,
+          s.relname as sequence
+        FROM
+          pg_class s
+        INNER JOIN
+          pg_depend d ON d.objid = s.oid
+        INNER JOIN
+          pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
+        INNER JOIN
+          pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
+        INNER JOIN
+          pg_namespace n ON n.oid = s.relnamespace
+        INNER JOIN
+          pg_namespace nt ON nt.oid = t.relnamespace
+        WHERE
+          s.relkind = 'S'
+      SQL
+      data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|
+        [k, v.map { |r| Sequence.new(r["sequence_schema"], r["sequence"], column: r["column"]) }]
+      end.to_h
+    end
+
+    def primary_keys(data_source)
+      # https://stackoverflow.com/a/20537829
+      # TODO can simplify with array_position in Postgres 9.5+
+      query = <<~SQL
+        SELECT
+          nspname AS schema,
+          relname AS table,
+          pg_attribute.attname AS column,
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
+          pg_attribute.attnum,
+          pg_index.indkey
+        FROM
+          pg_index, pg_class, pg_attribute, pg_namespace
+        WHERE
+          indrelid = pg_class.oid AND
+          pg_class.relnamespace = pg_namespace.oid AND
+          pg_attribute.attrelid = pg_class.oid AND
+          pg_attribute.attnum = any(pg_index.indkey) AND
+          indisprimary
+      SQL
+      data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|
+        [k, v.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["column"] }]
+      end.to_h
+    end
+
     def show_notes
       # for tables
       resolver.notes.each do |note|
@@ -66,6 +141,8 @@ module PgSync
           data_type AS type
         FROM
           information_schema.columns
+        WHERE
+          is_generated = 'NEVER'
         ORDER BY 1, 2, 3
       SQL
       data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|
@@ -93,28 +170,33 @@ module PgSync
     def run_tasks(tasks, &block)
       notices = []
       failed_tables = []
-
-      spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
-      task_spinners = {}
       started_at = {}
 
+      show_spinners = output.tty? && !opts[:in_batches] && !opts[:debug]
+      if show_spinners
+        spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
+        task_spinners = {}
+      end
+
       start = lambda do |task, i|
         message = ":spinner #{display_item(task)}"
-        spinner = spinners.register(message)
-        if opts[:in_batches]
-          # log instead of spin for non-tty
-          log message.sub(":spinner", "⠋")
-        else
+
+        if show_spinners
+          spinner = spinners.register(message)
           spinner.auto_spin
+          task_spinners[task] = spinner
+        elsif opts[:in_batches]
+          log message.sub(":spinner", "⠋")
         end
-        task_spinners[task] = spinner
+
         started_at[task] = Time.now
       end
 
       finish = lambda do |task, i, result|
-        spinner = task_spinners[task]
         time = (Time.now - started_at[task]).round(1)
 
+        success = result[:status] == "success"
+
         message =
           if result[:message]
             "(#{result[:message].lines.first.to_s.strip})"
@@ -124,24 +206,31 @@ module PgSync
 
         notices.concat(result[:notices])
 
-        if result[:status] == "success"
-          spinner.success(message)
+        if show_spinners
+          spinner = task_spinners[task]
+          if success
+            spinner.success(message)
+          else
+            spinner.error(message)
+          end
         else
-          spinner.error(message)
-          failed_tables << task_name(task)
-          fail_sync(failed_tables) if opts[:fail_fast]
+          status = success ? "✔" : "✖"
+          log [status, display_item(task), message].join(" ")
         end
 
-        unless spinner.send(:tty?)
-          status = result[:status] == "success" ? "✔" : "✖"
-          log [status, display_item(task), message].join(" ")
+        unless success
+          failed_tables << task_name(task)
+          fail_sync(failed_tables) if opts[:fail_fast]
         end
       end
 
       options = {start: start, finish: finish}
 
       jobs = opts[:jobs]
-      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints]
+
+      # disable multiple jobs for defer constraints and disable integrity
+      # so we can use a transaction to ensure a consistent snapshot
+      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
         warning "--jobs ignored" if jobs
         jobs = 0
       end
@@ -171,9 +260,25 @@ module PgSync
       fail_sync(failed_tables) if failed_tables.any?
     end
 
+    # TODO add option to open transaction on source when manually specifying order of tables
     def maybe_defer_constraints
-      if opts[:defer_constraints]
+      if opts[:disable_integrity] || opts[:disable_integrity_v2]
+        # create a transaction on the source
+        # to ensure we get a consistent snapshot
+        source.transaction do
+          yield
+        end
+      elsif opts[:defer_constraints] || opts[:defer_constraints_v2]
         destination.transaction do
+          if opts[:defer_constraints_v2]
+            table_constraints = non_deferrable_constraints(destination)
+            table_constraints.each do |table, constraints|
+              constraints.each do |constraint|
+                destination.execute("ALTER TABLE #{quote_ident_full(table)} ALTER CONSTRAINT #{quote_ident(constraint)} DEFERRABLE")
+              end
+            end
+          end
+
           destination.execute("SET CONSTRAINTS ALL DEFERRED")
 
           # create a transaction on the source
@@ -181,6 +286,20 @@ module PgSync
           source.transaction do
             yield
           end
+
+          # set them back
+          # there are 3 modes: DEFERRABLE INITIALLY DEFERRED, DEFERRABLE INITIALLY IMMEDIATE, and NOT DEFERRABLE
+          # we only update NOT DEFERRABLE
+          # https://www.postgresql.org/docs/current/sql-set-constraints.html
+          if opts[:defer_constraints_v2]
+            destination.execute("SET CONSTRAINTS ALL IMMEDIATE")
+
+            table_constraints.each do |table, constraints|
+              constraints.each do |constraint|
+                destination.execute("ALTER TABLE #{quote_ident_full(table)} ALTER CONSTRAINT #{quote_ident(constraint)} NOT DEFERRABLE")
+              end
+            end
+          end
         end
       else
         yield

data/lib/pgsync/task.rb

@@ -3,7 +3,7 @@ module PgSync
     include Utils
 
     attr_reader :source, :destination, :config, :table, :opts
-    attr_accessor :from_columns, :to_columns
+    attr_accessor :from_columns, :to_columns, :from_sequences, :to_sequences, :to_primary_key
 
     def initialize(source:, destination:, config:, table:, opts:)
       @source = source
@@ -11,6 +11,8 @@ module PgSync
       @config = config
       @table = table
       @opts = opts
+      @from_sequences = []
+      @to_sequences = []
     end
 
     def quoted_table
@@ -39,14 +41,6 @@ module PgSync
       @shared_fields ||= to_fields & from_fields
     end
 
-    def from_sequences
-      @from_sequences ||= opts[:no_sequences] ? [] : source.sequences(table, shared_fields)
-    end
-
-    def to_sequences
-      @to_sequences ||= opts[:no_sequences] ? [] : destination.sequences(table, shared_fields)
-    end
-
     def shared_sequences
       @shared_sequences ||= to_sequences & from_sequences
     end
@@ -88,15 +82,10 @@ module PgSync
       sql_clause << " #{opts[:sql]}" if opts[:sql]
 
       bad_fields = opts[:no_rules] ? [] : config["data_rules"]
-      primary_key = destination.primary_key(table)
+      primary_key = to_primary_key
       copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, _| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, primary_key)} AS #{quote_ident(f)}" : "#{quoted_table}.#{quote_ident(f)}" }.join(", ")
       fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
 
-      seq_values = {}
-      shared_sequences.each do |seq|
-        seq_values[seq] = source.last_value(seq)
-      end
-
       copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quoted_table}#{sql_clause}) TO STDOUT"
       if opts[:in_batches]
         raise Error, "No primary key" if primary_key.empty?
@@ -151,20 +140,27 @@ module PgSync
             "NOTHING"
           else # overwrite or sql clause
             setter = shared_fields.reject { |f| primary_key.include?(f) }.map { |f| "#{quote_ident(f)} = EXCLUDED.#{quote_ident(f)}" }
-            "UPDATE SET #{setter.join(", ")}"
+            if setter.any?
+              "UPDATE SET #{setter.join(", ")}"
+            else
+              "NOTHING"
+            end
           end
-        destination.execute("INSERT INTO #{quoted_table} (SELECT * FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
+        destination.execute("INSERT INTO #{quoted_table} (#{fields}) (SELECT #{fields} FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
       else
         # use delete instead of truncate for foreign keys
-        if opts[:defer_constraints]
+        if opts[:defer_constraints] || opts[:defer_constraints_v2]
           destination.execute("DELETE FROM #{quoted_table}")
         else
           destination.truncate(table)
         end
         copy(copy_to_command, dest_table: table, dest_fields: fields)
       end
-      seq_values.each do |seq, value|
-        destination.execute("SELECT setval(#{escape(seq)}, #{escape(value)})")
+
+      # update sequences
+      shared_sequences.each do |seq|
+        value = source.last_value(seq)
+        destination.execute("SELECT setval(#{escape(quote_ident_full(seq))}, #{escape(value)})")
       end
 
       {status: "success"}
@@ -214,6 +210,10 @@ module PgSync
 
     def copy(source_command, dest_table:, dest_fields:)
       destination_command = "COPY #{quote_ident_full(dest_table)} (#{dest_fields}) FROM STDIN"
+
+      source.log_sql(source_command)
+      destination.log_sql(destination_command)
+
       destination.conn.copy_data(destination_command) do
         source.conn.copy_data(source_command) do
           while (row = source.conn.get_copy_data)

data/lib/pgsync/utils.rb

@@ -3,7 +3,8 @@ module PgSync
     COLOR_CODES = {
       red: 31,
       green: 32,
-      yellow: 33
+      yellow: 33,
+      cyan: 36
     }
 
     def log(message = nil)
@@ -59,7 +60,7 @@ module PgSync
     end
 
     def quote_ident_full(ident)
-      if ident.is_a?(Table)
+      if ident.is_a?(Table) || ident.is_a?(Sequence)
         [quote_ident(ident.schema), quote_ident(ident.name)].join(".")
       else # temp table names are strings
         quote_ident(ident)

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.6.2"
+  VERSION = "0.6.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.7
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-10 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.1
+        version: 4.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.1
+        version: 4.8.2
 - !ruby/object:Gem::Dependency
   name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
@@ -66,49 +66,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
+description:
 email: andrew@chartkick.com
 executables:
 - pgsync
@@ -125,6 +83,7 @@ files:
 - lib/pgsync/data_source.rb
 - lib/pgsync/init.rb
 - lib/pgsync/schema_sync.rb
+- lib/pgsync/sequence.rb
 - lib/pgsync/sync.rb
 - lib/pgsync/table.rb
 - lib/pgsync/table_sync.rb
@@ -136,7 +95,7 @@ homepage: https://github.com/ankane/pgsync
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -151,8 +110,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Sync Postgres data between databases
 test_files: []