pgsync 0.6.3 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d3ff88829338544bfe3434a16a00640e6496b7b89b1ef5dd43e66f844ea51ce
-  data.tar.gz: d5457e8c2596fdda651c9739712fe0934af14028295ea9cd6addfc70438ced16
+  metadata.gz: 0ecb467f4d3112edfcfebe19913e29daeddfe1e4cc10e18bd9e8850fbceced57
+  data.tar.gz: 4d8309dec9e8238074267baf583fe333c7b1d3300ad80398748e15905eb1e539
 SHA512:
-  metadata.gz: 6f9549d0d85cb502b5b88a3178fe617bf94f032d72787ab6dcf8f6097dc256ca2e969e2c4a434ebf33e4de17a4f11c590c57ef8cae420b20fe4f39bfed795baa
-  data.tar.gz: b10ac04af7dc26c3067c2ff73f163c69d73dfb5c1f559585adcf774312ac78ac07dcd790e1706cc571a90f0e4573060cedf01fc6ebd8b1732fbaa16b486213dd
+  metadata.gz: 145eb31e2565257a5adedc05c1308692dffb1e0785728ecd38230bb23c34c4ba6ecf08512201f707f457b136e0b7a22ca511b46fb7834f2045c041ff43dd6ccc
+  data.tar.gz: cde51dee36149f9f8c21e26eecd504de68d8a47c2524f084b74962d7ceae1a01b59841261fc6c112cf94b31a89f55b9a991c05d1af0db41c46ed615fc56e888f

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.4 (2020-06-10)
+
+- Log SQL with `--debug` option
+- Improved sequence queries
+
 ## 0.6.3 (2020-06-09)
 
 - Added `--defer-constraints-v2` option

data/README.md

@@ -307,6 +307,14 @@ exclude:
   - schema_migrations
 ```
 
+## Debugging
+
+To view the SQL that’s run, use:
+
+```sh
+pgsync --debug
+```
+
 ## Other Commands
 
 Help

data/lib/pgsync.rb

@@ -18,6 +18,7 @@ require "pgsync/client"
 require "pgsync/data_source"
 require "pgsync/init"
 require "pgsync/schema_sync"
+require "pgsync/sequence"
 require "pgsync/sync"
 require "pgsync/table"
 require "pgsync/table_sync"

data/lib/pgsync/data_source.rb

@@ -4,8 +4,10 @@ module PgSync
 
     attr_reader :url
 
-    def initialize(url)
+    def initialize(url, name:, debug:)
       @url = url
+      @name = name
+      @debug = debug
     end
 
     def exists?
@@ -50,10 +52,6 @@ module PgSync
       table_set.include?(table)
     end
 
-    def sequences(table, columns)
-      execute("SELECT #{columns.map { |f| "pg_get_serial_sequence(#{escape("#{quote_ident_full(table)}")}, #{escape(f)}) AS #{quote_ident(f)}" }.join(", ")}").first.values.compact
-    end
-
     def max_id(table, primary_key, sql_clause = nil)
       execute("SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}").first["max"].to_i
     end
@@ -62,39 +60,14 @@ module PgSync
       execute("SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}").first["min"].to_i
     end
 
-    # this value comes from pg_get_serial_sequence which is already quoted
     def last_value(seq)
-      execute("SELECT last_value FROM #{seq}").first["last_value"]
+      execute("SELECT last_value FROM #{quote_ident_full(seq)}").first["last_value"]
     end
 
     def truncate(table)
       execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
     end
 
-    # https://stackoverflow.com/a/20537829
-    # TODO can simplify with array_position in Postgres 9.5+
-    def primary_key(table)
-      query = <<~SQL
-        SELECT
-          pg_attribute.attname,
-          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
-          pg_attribute.attnum,
-          pg_index.indkey
-        FROM
-          pg_index, pg_class, pg_attribute, pg_namespace
-        WHERE
-          nspname = $1 AND
-          relname = $2 AND
-          indrelid = pg_class.oid AND
-          pg_class.relnamespace = pg_namespace.oid AND
-          pg_attribute.attrelid = pg_class.oid AND
-          pg_attribute.attnum = any(pg_index.indkey) AND
-          indisprimary
-      SQL
-      rows = execute(query, [table.schema, table.name])
-      rows.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["attname"] }
-    end
-
     def triggers(table)
       query = <<~SQL
         SELECT
@@ -148,20 +121,34 @@ module PgSync
     end
 
     def execute(query, params = [])
+      log_sql query, params
       conn.exec_params(query, params).to_a
     end
 
     def transaction
       if conn.transaction_status == 0
         # not currently in transaction
-        conn.transaction do
-          yield
-        end
+        log_sql "BEGIN"
+        result =
+          conn.transaction do
+            yield
+          end
+        log_sql "COMMIT"
+        result
       else
         yield
       end
     end
 
+    # TODO log time for each statement
+    def log_sql(query, params = {})
+      if @debug
+        message = "#{colorize("[#{@name}]", :cyan)} #{query.gsub(/\s+/, " ").strip}"
+        message = "#{message} #{params.inspect}" if params.any?
+        log message
+      end
+    end
+
     private
 
     def concurrent_id

data/lib/pgsync/sequence.rb

@@ -0,0 +1,29 @@
+# minimal class to keep schema and sequence name separate
+module PgSync
+  class Sequence
+    attr_reader :schema, :name, :column
+
+    def initialize(schema, name, column:)
+      @schema = schema
+      @name = name
+      @column = column
+    end
+
+    def full_name
+      "#{schema}.#{name}"
+    end
+
+    def eql?(other)
+      other.schema == schema && other.name == name
+    end
+
+    # override hash when overriding eql?
+    def hash
+      [schema, name].hash
+    end
+
+    def to_s
+      full_name
+    end
+  end
+end

data/lib/pgsync/sync.rb

@@ -34,13 +34,13 @@ module PgSync
         raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
       end
 
+      print_description("From", source)
+      print_description("To", destination)
+
       if (opts[:preserve] || opts[:overwrite]) && destination.server_version_num < 90500
         raise Error, "Postgres 9.5+ is required for --preserve and --overwrite"
       end
 
-      print_description("From", source)
-      print_description("To", destination)
-
       resolver = TaskResolver.new(args: args, opts: opts, source: source, destination: destination, config: config, first_schema: first_schema)
       tasks =
         resolver.tasks.map do |task|
@@ -126,15 +126,15 @@ module PgSync
     end
 
     def source
-      @source ||= data_source(@options[:from])
+      @source ||= data_source(@options[:from], "from")
     end
 
     def destination
-      @destination ||= data_source(@options[:to])
+      @destination ||= data_source(@options[:to], "to")
     end
 
-    def data_source(url)
-      ds = DataSource.new(url)
+    def data_source(url, name)
+      ds = DataSource.new(url, name: name, debug: @options[:debug])
       ObjectSpace.define_finalizer(self, self.class.finalize(ds))
       ds
     end

data/lib/pgsync/table_sync.rb

@@ -17,6 +17,10 @@ module PgSync
 
       add_columns
 
+      add_primary_keys
+
+      add_sequences unless opts[:no_sequences]
+
       show_notes
 
       # don't sync tables with no shared fields
@@ -24,8 +28,6 @@ module PgSync
       run_tasks(tasks.reject { |task| task.shared_fields.empty? })
     end
 
-    # TODO only query specific tables
-    # TODO add sequences, primary keys, etc
     def add_columns
       source_columns = columns(source)
       destination_columns = columns(destination)
@@ -36,6 +38,79 @@ module PgSync
       end
     end
 
+    def add_primary_keys
+      destination_primary_keys = primary_keys(destination)
+
+      tasks.each do |task|
+        task.to_primary_key = destination_primary_keys[task.table] || []
+      end
+    end
+
+    def add_sequences
+      source_sequences = sequences(source)
+      destination_sequences = sequences(destination)
+
+      tasks.each do |task|
+        shared_columns = Set.new(task.shared_fields)
+
+        task.from_sequences = (source_sequences[task.table] || []).select { |s| shared_columns.include?(s.column) }
+        task.to_sequences = (destination_sequences[task.table] || []).select { |s| shared_columns.include?(s.column) }
+      end
+    end
+
+    def sequences(data_source)
+      query = <<~SQL
+        SELECT
+          nt.nspname as schema,
+          t.relname as table,
+          a.attname as column,
+          n.nspname as sequence_schema,
+          s.relname as sequence
+        FROM
+          pg_class s
+        INNER JOIN
+          pg_depend d ON d.objid = s.oid
+        INNER JOIN
+          pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
+        INNER JOIN
+          pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
+        INNER JOIN
+          pg_namespace n ON n.oid = s.relnamespace
+        INNER JOIN
+          pg_namespace nt ON nt.oid = t.relnamespace
+        WHERE
+          s.relkind = 'S'
+      SQL
+      data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|
+        [k, v.map { |r| Sequence.new(r["sequence_schema"], r["sequence"], column: r["column"]) }]
+      end.to_h
+    end
+
+    def primary_keys(data_source)
+      # https://stackoverflow.com/a/20537829
+      # TODO can simplify with array_position in Postgres 9.5+
+      query = <<~SQL
+        SELECT
+          nspname AS schema,
+          relname AS table,
+          pg_attribute.attname AS column,
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
+          pg_attribute.attnum,
+          pg_index.indkey
+        FROM
+          pg_index, pg_class, pg_attribute, pg_namespace
+        WHERE
+          indrelid = pg_class.oid AND
+          pg_class.relnamespace = pg_namespace.oid AND
+          pg_attribute.attrelid = pg_class.oid AND
+          pg_attribute.attnum = any(pg_index.indkey) AND
+          indisprimary
+      SQL
+      data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|
+        [k, v.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["column"] }]
+      end.to_h
+    end
+
     def show_notes
       # for tables
       resolver.notes.each do |note|
@@ -93,28 +168,33 @@ module PgSync
     def run_tasks(tasks, &block)
       notices = []
       failed_tables = []
-
-      spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
-      task_spinners = {}
       started_at = {}
 
+      show_spinners = output.tty? && !opts[:in_batches] && !opts[:debug]
+      if show_spinners
+        spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
+        task_spinners = {}
+      end
+
       start = lambda do |task, i|
         message = ":spinner #{display_item(task)}"
-        spinner = spinners.register(message)
-        if opts[:in_batches]
-          # log instead of spin for non-tty
-          log message.sub(":spinner", "⠋")
-        else
+
+        if show_spinners
+          spinner = spinners.register(message)
           spinner.auto_spin
+          task_spinners[task] = spinner
+        elsif opts[:in_batches]
+          log message.sub(":spinner", "⠋")
         end
-        task_spinners[task] = spinner
+
         started_at[task] = Time.now
       end
 
       finish = lambda do |task, i, result|
-        spinner = task_spinners[task]
         time = (Time.now - started_at[task]).round(1)
 
+        success = result[:status] == "success"
+
         message =
           if result[:message]
             "(#{result[:message].lines.first.to_s.strip})"
@@ -124,23 +204,30 @@ module PgSync
 
         notices.concat(result[:notices])
 
-        if result[:status] == "success"
-          spinner.success(message)
+        if show_spinners
+          spinner = task_spinners[task]
+          if success
+            spinner.success(message)
+          else
+            spinner.error(message)
+          end
         else
-          spinner.error(message)
-          failed_tables << task_name(task)
-          fail_sync(failed_tables) if opts[:fail_fast]
+          status = success ? "✔" : "✖"
+          log [status, display_item(task), message].join(" ")
         end
 
-        unless spinner.send(:tty?)
-          status = result[:status] == "success" ? "✔" : "✖"
-          log [status, display_item(task), message].join(" ")
+        unless success
+          failed_tables << task_name(task)
+          fail_sync(failed_tables) if opts[:fail_fast]
         end
       end
 
       options = {start: start, finish: finish}
 
       jobs = opts[:jobs]
+
+      # disable multiple jobs for defer constraints and disable integrity
+      # so we can use a transaction to ensure a consistent snapshot
       if opts[:debug] || opts[:in_batches] || opts[:defer_constraints] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
         warning "--jobs ignored" if jobs
         jobs = 0
@@ -171,6 +258,7 @@ module PgSync
       fail_sync(failed_tables) if failed_tables.any?
     end
 
+    # TODO add option to open transaction on source when manually specifying order of tables
     def maybe_defer_constraints
       if opts[:disable_integrity] || opts[:disable_integrity_v2]
         # create a transaction on the source

data/lib/pgsync/task.rb

@@ -3,7 +3,7 @@ module PgSync
     include Utils
 
     attr_reader :source, :destination, :config, :table, :opts
-    attr_accessor :from_columns, :to_columns
+    attr_accessor :from_columns, :to_columns, :from_sequences, :to_sequences, :to_primary_key
 
     def initialize(source:, destination:, config:, table:, opts:)
       @source = source
@@ -11,6 +11,8 @@ module PgSync
       @config = config
       @table = table
       @opts = opts
+      @from_sequences = []
+      @to_sequences = []
     end
 
     def quoted_table
@@ -39,14 +41,6 @@ module PgSync
       @shared_fields ||= to_fields & from_fields
     end
 
-    def from_sequences
-      @from_sequences ||= opts[:no_sequences] ? [] : source.sequences(table, shared_fields)
-    end
-
-    def to_sequences
-      @to_sequences ||= opts[:no_sequences] ? [] : destination.sequences(table, shared_fields)
-    end
-
     def shared_sequences
       @shared_sequences ||= to_sequences & from_sequences
     end
@@ -88,15 +82,10 @@ module PgSync
       sql_clause << " #{opts[:sql]}" if opts[:sql]
 
       bad_fields = opts[:no_rules] ? [] : config["data_rules"]
-      primary_key = destination.primary_key(table)
+      primary_key = to_primary_key
       copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, _| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, primary_key)} AS #{quote_ident(f)}" : "#{quoted_table}.#{quote_ident(f)}" }.join(", ")
       fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
 
-      seq_values = {}
-      shared_sequences.each do |seq|
-        seq_values[seq] = source.last_value(seq)
-      end
-
       copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quoted_table}#{sql_clause}) TO STDOUT"
       if opts[:in_batches]
         raise Error, "No primary key" if primary_key.empty?
@@ -163,8 +152,11 @@ module PgSync
         end
         copy(copy_to_command, dest_table: table, dest_fields: fields)
       end
-      seq_values.each do |seq, value|
-        destination.execute("SELECT setval(#{escape(seq)}, #{escape(value)})")
+
+      # update sequences
+      shared_sequences.each do |seq|
+        value = source.last_value(seq)
+        destination.execute("SELECT setval(#{escape(quote_ident_full(seq))}, #{escape(value)})")
       end
 
       {status: "success"}
@@ -214,6 +206,10 @@ module PgSync
 
     def copy(source_command, dest_table:, dest_fields:)
       destination_command = "COPY #{quote_ident_full(dest_table)} (#{dest_fields}) FROM STDIN"
+
+      source.log_sql(source_command)
+      destination.log_sql(destination_command)
+
       destination.conn.copy_data(destination_command) do
         source.conn.copy_data(source_command) do
           while (row = source.conn.get_copy_data)

data/lib/pgsync/utils.rb

@@ -3,7 +3,8 @@ module PgSync
     COLOR_CODES = {
       red: 31,
       green: 32,
-      yellow: 33
+      yellow: 33,
+      cyan: 36
     }
 
     def log(message = nil)
@@ -59,7 +60,7 @@ module PgSync
     end
 
     def quote_ident_full(ident)
-      if ident.is_a?(Table)
+      if ident.is_a?(Table) || ident.is_a?(Sequence)
         [quote_ident(ident.schema), quote_ident(ident.name)].join(".")
       else # temp table names are strings
         quote_ident(ident)

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.6.3"
+  VERSION = "0.6.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-10 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -125,6 +125,7 @@ files:
 - lib/pgsync/data_source.rb
 - lib/pgsync/init.rb
 - lib/pgsync/schema_sync.rb
+- lib/pgsync/sequence.rb
 - lib/pgsync/sync.rb
 - lib/pgsync/table.rb
 - lib/pgsync/table_sync.rb