pgsync 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b754f24353de37a0b8b8800d9ea0ac010c3dcd16234a6554abab564c1fb6b839
-  data.tar.gz: 7ec8238e286e7c145934be5309e3a67ab3f7a9b0267840e046d8b7e6b0c703bb
+  metadata.gz: 855b6fc121c88a53bc7fda6243add1d965c880e4fcb33328d5cfd931cb7bb588
+  data.tar.gz: fc157143ac24e7cfad51d57bc10c2859ea8225d74a6a3376821e7ac45e8dee13
 SHA512:
-  metadata.gz: 62ee273ca358509668ce44a7b9b599e156c31ef47a457d9ff857a1384c97e3e73379f84b8d56f6347b612249725c111b62671be7ac0703d6a96a89e6a920c33f
-  data.tar.gz: 1e55d84d7a142ae3b1b79e48cbc55171e2f78265a478428427a7e8f38712d7a26f9801ff958f4e68011c758f881b365656128e8888ec9f81efac1a087e1cb03c
+  metadata.gz: 6fbdbb036ea9ca16138af36fec1a01ae31c9b7963065bf8014ae3a09d564cdc7ecb82ee36f32531435a793ae9e933a49685e348fcaed674f702b2cce1d7b7c74
+  data.tar.gz: f3ff567e7e2b0720696efe209c98cc2b983969aefe28c9ee3ac68c829ccd5a755820f1d77008f1c4e86acd184f56ec08c1762a3460b08c27679e93cb1313718c

data/CHANGELOG.md

@@ -1,26 +1,32 @@
-# 0.4.3
+## 0.5.0 (2020-03-26)
+
+- Improved output when syncing
+- Improved output on interrupt
+- Added `--no-sequences` option
+
+## 0.4.3 (2019-10-27)
 
 - Added `sslmode` to template
 
-# 0.4.2
+## 0.4.2 (2019-10-27)
 
 - Improved flexibility of commands
 - Sync all objects when no tables specified
 
-# 0.4.1
+## 0.4.1 (2018-12-15)
 
 - Made `psql` version check more robust
 - Fixed issue with non-lowercase primary key
 - Prefer `--init` over `--setup`
 - Improved data rules
 
-# 0.4.0
+## 0.4.0 (2018-02-28)
 
 - Sync all schemas in search path by default
 - Added support for socket connections
 - Added support for environment variables
 
-# 0.3.9
+## 0.3.9 (2018-02-27)
 
 - Better support for schemas
 - Added `--schemas` option
@@ -28,47 +34,47 @@
 - Added `--schema-first` option
 - Fixed issue with non-lowercase tables and partial syncs
 
-# 0.3.8
+## 0.3.8 (2017-10-01)
 
 - Added Windows support
 - Added `random_string` and `random_number` replacement options
 - Improved performance of `--in-batches` for large tables
 
-# 0.3.7
+## 0.3.7 (2017-08-30)
 
 - Fixed non-lowercase tables and columns
 - Fixed `--truncate` option with `--in-batches`
 
-# 0.3.6
+## 0.3.6 (2016-10-02)
 
 - Fixed `Table does not exist in source` error
 
-# 0.3.5
+## 0.3.5 (2016-07-23)
 
 - Support schemas other than public
 
-# 0.3.4
+## 0.3.4 (2016-04-29)
 
 - Added `--in-batches` mode for production transfers with `--batch-size` and `--sleep`
 
-# 0.3.3
+## 0.3.3 (2016-04-25)
 
 - Added `-d` option as an alias for `--db`
 - Added support for wildcard tables
 - Fixed `--schema-only` errors
 
-# 0.3.2
+## 0.3.2 (2016-04-19)
 
 - Prefer `{1}` for interpolation
 - Added `--overwrite` option
 - Deprecated `--where` and `--limit`
 
-# 0.3.1
+## 0.3.1 (2016-04-06)
 
 - Added `-t` or `--tables`, `-g` or `--groups` options
 - Deprecated `tables`, `groups`, and `setup` commands
 
-# 0.3.0
+## 0.3.0 (2016-04-06)
 
 - More powerful groups
 - Overwrite rows by default when `WHERE` clause (previously truncated)
@@ -79,34 +85,34 @@
 - Added `--setup` option
 - Added `--truncate` option
 
-# 0.2.4
+## 0.2.4 (2016-04-04)
 
 - Added `--preserve` option
 - Added `--list` option for groups and tables
 - Added `--limit` option
 
-# 0.2.3
+## 0.2.3 (2016-03-30)
 
 - Fixed `no PostgreSQL user name specified in startup packet`
 
-# 0.2.2
+## 0.2.2 (2016-03-28)
 
 - Added `--debug` option
 
-# 0.2.1
+## 0.2.1 (2016-03-27)
 
 - Do not require config file
 
-# 0.2.0
+## 0.2.0 (2016-03-26)
 
 - Fixed idle transaction timeout errors - respects `statement_timeout` as a result
 - Raise error when command exits with non-zero status
 
-# 0.1.1
+## 0.1.1 (2016-03-23)
 
 - Better support for multiple databases
 - Search parent directories for config file
 
-# 0.1.0
+## 0.1.0 (2015-12-07)
 
 - First release

data/README.md

@@ -122,6 +122,8 @@ pgsync product:123
 
 ### Schema
 
+**Note:** pgsync is designed to sync data. You should use a schema migration tool to manage schema changes. The methods in this section are provided for convenience but not recommended.
+
 Sync schema before the data
 
 ```sh
@@ -146,7 +148,7 @@ pgsync does not try to sync Postgres extensions.
 
 ## Data Protection
 
-Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to your database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
+Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
 
 ## Sensitive Information
 
@@ -183,7 +185,7 @@ Options for replacement are:
 - `null`
 - `untouched`
 
-Rules starting with `unique_` require the table to have a primary key.
+Rules starting with `unique_` require the table to have a primary key. `unique_phone` requires a numeric primary key.
 
 ## Multiple Databases
 
@@ -280,6 +282,14 @@ gem install specific_install
 gem specific_install https://github.com/ankane/pgsync.git
 ```
 
+## Related Projects
+
+Also check out:
+
+- [Dexter](https://github.com/ankane/dexter) - The automatic indexer for Postgres
+- [PgHero](https://github.com/ankane/pghero) - A performance dashboard for Postgres
+- [pgslice](https://github.com/ankane/pgslice) - Postgres partitioning as easy as pie
+
 ## Thanks
 
 Inspired by [heroku-pg-transfer](https://github.com/ddollar/heroku-pg-transfer).
@@ -293,7 +303,7 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
-To run tests, do:
+To get started with development:
 
 ```sh
 git clone https://github.com/ankane/pgsync.git
@@ -304,5 +314,5 @@ createdb pgsync_test1
 createdb pgsync_test2
 createdb pgsync_test3
 
-bundle exec rake
+bundle exec rake test
 ```

data/exe/pgsync

@@ -1,10 +1,12 @@
 #!/usr/bin/env ruby
 
+trap("SIGINT") { abort }
+
 begin
   require "pgsync"
   PgSync::Client.new(ARGV).perform
 rescue PgSync::Error => e
-  abort e.message
+  abort PgSync::Client.colorize(e.message, 31) # red
 rescue Interrupt
   abort
 end

data/lib/pgsync.rb

@@ -1,17 +1,21 @@
+# dependencies
+require "parallel"
+require "pg"
+require "slop"
+require "tty-spinner"
+
+# stdlib
 require "cgi"
 require "erb"
 require "fileutils"
-require "multiprocessing"
-require "pg"
-require "parallel"
 require "set"
 require "shellwords"
-require "slop"
 require "tempfile"
 require "thread" # windows only
 require "uri"
 require "yaml"
 
+# modules
 require "pgsync/client"
 require "pgsync/data_source"
 require "pgsync/table_list"

data/lib/pgsync/client.rb

@@ -5,7 +5,6 @@ module PgSync
       $stderr.sync = true
       @exit = false
       @arguments, @options = parse_args(args)
-      @mutex = windows? ? Mutex.new : MultiProcessing::Mutex.new
     end
 
     # TODO clean up this mess
@@ -92,8 +91,8 @@ module PgSync
         unless opts[:schema_only]
           confirm_tables_exist(destination, tables, "destination")
 
-          in_parallel(tables) do |table, table_opts|
-            TableSync.new.sync(@mutex, config, table, opts.merge(table_opts), source.url, destination.url, source.search_path.find { |sp| sp != "pg_catalog" })
+          in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts|
+            TableSync.new.sync(config, table, opts.merge(table_opts), source.url, destination.url)
           end
         end
 
@@ -181,6 +180,7 @@ Options:}
         o.boolean "--schema-only", "schema only", default: false
         o.boolean "--all-schemas", "all schemas", default: false
         o.boolean "--no-rules", "do not apply data rules", default: false
+        o.boolean "--no-sequences", "do not sync sequences", default: false
         o.boolean "--init", "init", default: false
         o.boolean "--setup", "setup", default: false, help: false
         o.boolean "--in-batches", "in batches", default: false, help: false
@@ -267,14 +267,56 @@ Options:}
       $stderr.puts message
     end
 
-    def in_parallel(tables, &block)
+    def in_parallel(tables, first_schema:, &block)
+      spinners = TTY::Spinner::Multi.new(format: :dots)
+      item_spinners = {}
+
+      start = lambda do |item, i|
+        table, opts = item
+        message = String.new(":spinner ")
+        message << table.sub("#{first_schema}.", "")
+        # maybe output later
+        # message << " #{opts[:sql]}" if opts[:sql]
+        spinner = spinners.register(message)
+        spinner.auto_spin
+        item_spinners[item] = spinner
+      end
+
+      errors = 0
+
+      finish = lambda do |item, i, result|
+        spinner = item_spinners[item]
+        if result[:status] == "success"
+          spinner.success(display_message(result))
+        else
+          # TODO add option to fail fast
+          spinner.error(display_message(result))
+          errors += 1
+        end
+
+        unless spinner.send(:tty?)
+          status = result[:status] == "success" ? "✔" : "✖"
+          log [status, item.first.sub("#{first_schema}.", ""), display_message(result)].compact.join(" ")
+        end
+      end
+
+      options = {start: start, finish: finish}
       if @options[:debug] || @options[:in_batches]
-        tables.each(&block)
+        options[:in_processes] = 0
       else
-        options = {}
         options[:in_threads] = 4 if windows?
-        Parallel.each(tables, options, &block)
       end
+
+      Parallel.each(tables, **options, &block)
+
+      raise PgSync::Error, "Synced failed for #{errors} table#{errors == 1 ? nil : "s"}" if errors > 0
+    end
+
+    def display_message(result)
+      message = String.new("")
+      message << "- #{result[:time]}s" if result[:time]
+      message << "(#{result[:message].gsub("\n", " ").strip})" if result[:message]
+      message
     end
 
     def pretty_list(items)
@@ -289,11 +331,20 @@ Options:}
 
     def log_completed(start_time)
       time = Time.now - start_time
-      log "Completed in #{time.round(1)}s"
+      message = "Completed in #{time.round(1)}s"
+      log self.class.colorize(message, 32) # green
     end
 
     def windows?
       Gem.win_platform?
     end
+
+    def self.colorize(message, color_code)
+      if $stderr.tty?
+        "\e[#{color_code}m#{message}\e[0m"
+      else
+        message
+      end
+    end
   end
 end

data/lib/pgsync/table_sync.rb

@@ -1,6 +1,6 @@
 module PgSync
   class TableSync
-    def sync(mutex, config, table, opts, source_url, destination_url, first_schema)
+    def sync(config, table, opts, source_url, destination_url)
       start_time = Time.now
       source = DataSource.new(source_url, timeout: 0)
       destination = DataSource.new(destination_url, timeout: 0)
@@ -17,30 +17,32 @@ module PgSync
         extra_fields = to_fields - from_fields
         missing_fields = from_fields - to_fields
 
-        from_sequences = source.sequences(table, shared_fields)
-        to_sequences = destination.sequences(table, shared_fields)
+        if opts[:no_sequences]
+          from_sequences = []
+          to_sequences = []
+        else
+          from_sequences = source.sequences(table, shared_fields)
+          to_sequences = destination.sequences(table, shared_fields)
+        end
+
         shared_sequences = to_sequences & from_sequences
         extra_sequences = to_sequences - from_sequences
         missing_sequences = from_sequences - to_sequences
 
         sql_clause = String.new
 
-        table_name = table.sub("#{first_schema}.", "")
+        if opts[:sql]
+          sql_clause << " #{opts[:sql]}"
+        end
 
-        mutex.synchronize do
-          log "* Syncing #{table_name}"
-          if opts[:sql]
-            log "    #{opts[:sql]}"
-            sql_clause << " #{opts[:sql]}"
-          end
-          log "    Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
-          log "    Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
-          log "    Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
-          log "    Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
+        notes = []
+        notes << "Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
+        notes << "Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
+        notes << "Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
+        notes << "Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
 
-          if shared_fields.empty?
-            log "    No fields to copy"
-          end
+        if shared_fields.empty?
+          return {status: "success", message: "No fields to copy"}
         end
 
         if shared_fields.any?
@@ -147,13 +149,19 @@ module PgSync
             to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
           end
         end
-        mutex.synchronize do
-          log "* DONE #{table_name} (#{(Time.now - start_time).round(1)}s)"
+
+        message = nil
+        if notes.any?
+          message = notes.join(", ")
         end
+
+        {status: "success", message: message, time: (Time.now - start_time).round(1)}
       ensure
         source.close
         destination.close
       end
+    rescue PgSync::Error => e
+      {status: "error", message: e.message}
     end
 
     private

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.4.3"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-28 00:00:00.000000000 Z
+date: 2020-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: multiprocessing
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,47 +25,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.2
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: 4.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.18.2
+        version: 4.2.0
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Sync Postgres data between databases