pgsync 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b145ab9d9f312a9a043e91d5d4b9e7977bb5788e339aa4f88a571a5178d9c92f
-  data.tar.gz: 8843f622d7b152fff80a2a465fddd55dcd4271f0256cd64b46a3898a38214d65
+  metadata.gz: af75f9e9eb115a889d53b10ed89d8355e3ef111868b69eec31a025a8f22db27e
+  data.tar.gz: 5da0d56267ac89f2ffc282e92275a91d3fb83d80fe969207329ec8ce1876472b
 SHA512:
-  metadata.gz: f210607011225b932975ddd77343c1ff9468ae824a642b3c0b48033853a2fa0d68f4eed4fcfcda5c36a6d7cc1b99e491a6a0b62f147d232c8d517721b08a6590
-  data.tar.gz: bb38c6291b40ad0188483889aa9696269a384456ed7425be61e624a26b797110dc363a44b9d0774fc4b2ad04122d6e8863d04b7bb1a89f986c0829ada8a2aea4
+  metadata.gz: 9f9d78aebf02f75a5fee9413b56bba68ed97334b3134cef5c28f4b5a8a15cbab45e752cb73dcd88028e2b3b2b0fefb20ef9bd4abe0c92d35998a520b0029e961
+  data.tar.gz: a2fdaee8ba32df378ed2538119817b7649fe134a33c0f6013d55d8e1cb76ae6b0a75d3dd10dff4872d0452bc2e41bb0024c5f276f16ed8f0d1c81b114574f757

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.5.2 (2020-03-27)
+
+- Added `--fail-fast` option
+- Automatically exclude tables when `--init` run inside Rails app
+- Improved error message
+- Fixed typo in error message
+
 ## 0.5.1 (2020-03-26)
 
 - Fixed Slop warning with Ruby 2.7

data/README.md

@@ -1,9 +1,10 @@
 # pgsync
 
-Sync Postgres data between databases. Designed for:
+Sync data from one Postgres database to another (like `pg_dump`/`pg_restore`). Designed for:
 
-- **speed** - up to 4x faster than traditional tools on a 4-core machine
+- **speed** - tables are transferred in parallel
 - **security** - built-in methods to prevent sensitive data from ever leaving the server
+- **flexibility** - gracefully handles schema differences, like missing columns and extra columns
 - **convenience** - sync partial tables, groups of tables, and related records
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
@@ -20,6 +21,8 @@ gem install pgsync
 
 This will give you the `pgsync` command. If installation fails, you may need to install [dependencies](#dependencies).
 
+## Setup
+
 In your project directory, run:
 
 ```sh
@@ -36,7 +39,7 @@ Sync all tables
 pgsync
 ```
 
-**Note:** pgsync assumes your schema is already set up on your local machine. See the [schema section](#schema) if that’s not the case.
+**Note:** pgsync assumes your schema is setup in your `to` database. See the [schema section](#schema) if that’s not the case.
 
 Sync specific tables
 
@@ -246,7 +249,7 @@ Use groups when possible to take advantage of parallelism.
 For Ruby scripts, you may need to do:
 
 ```rb
-Bundler.with_clean_env do
+Bundler.with_unbundled_env do
   system "pgsync ..."
 end
 ```

data/config.yml

@@ -13,10 +13,7 @@ from: $(some_command)?sslmode=require
 to: postgres://localhost:5432/myapp_development
 
 # exclude tables
-# exclude:
-#   - schema_migrations
-#   - ar_internal_metadata
-
+%{exclude}
 # define groups
 # groups:
 #   group1:

data/exe/pgsync

@@ -1,15 +1,11 @@
 #!/usr/bin/env ruby
 
 # disable Slop warning with Ruby 2.7
+# TODO remove when Slop > 4.7.0 is released
 $VERBOSE = nil
 
+# handle interrupts
 trap("SIGINT") { abort }
 
-begin
-  require "pgsync"
-  PgSync::Client.new(ARGV).perform
-rescue PgSync::Error => e
-  abort PgSync::Client.colorize(e.message, 31) # red
-rescue Interrupt
-  abort
-end
+require "pgsync"
+PgSync::Client.start

data/lib/pgsync.rb

@@ -5,19 +5,18 @@ require "slop"
 require "tty-spinner"
 
 # stdlib
-require "cgi"
-require "erb"
-require "fileutils"
 require "set"
 require "shellwords"
 require "tempfile"
-require "thread" # windows only
 require "uri"
 require "yaml"
 
 # modules
+require "pgsync/utils"
 require "pgsync/client"
 require "pgsync/data_source"
+require "pgsync/init"
+require "pgsync/sync"
 require "pgsync/table_list"
 require "pgsync/table_sync"
 require "pgsync/version"

data/lib/pgsync/client.rb

@@ -1,160 +1,38 @@
 module PgSync
   class Client
+    include Utils
+
     def initialize(args)
-      $stdout.sync = true
+      @args = args
       $stderr.sync = true
-      @exit = false
-      @arguments, @options = parse_args(args)
     end
 
-    # TODO clean up this mess
-    def perform
-      return if @exit
-
-      args, opts = @arguments, @options
-      [:to, :from, :to_safe, :exclude, :schemas].each do |opt|
-        opts[opt] ||= config[opt.to_s]
-      end
-      map_deprecations(args, opts)
+    def perform(testing: true)
+      opts = parse_args
 
-      if opts[:init]
-        setup(db_config_file(args[0]) || config_file || ".pgsync.yml")
+      if opts.version?
+        log VERSION
+      elsif opts.help?
+        log opts
+      # TODO remove deprecated conditions (last two)
+      elsif opts.init? || opts.setup? || opts.arguments[0] == "setup"
+        Init.new.perform(opts)
       else
-        sync(args, opts)
+        Sync.new.perform(opts)
       end
-
-      true
+    rescue Error, PG::ConnectionBad => e
+      raise e if testing
+      abort colorize(e.message, 31) # red
     end
 
-    protected
-
-    def sync(args, opts)
-      start_time = Time.now
-
-      if args.size > 2
-        raise PgSync::Error, "Usage:\n    pgsync [options]"
-      end
-
-      source = DataSource.new(opts[:from])
-      raise PgSync::Error, "No source" unless source.exists?
-
-      destination = DataSource.new(opts[:to])
-      raise PgSync::Error, "No destination" unless destination.exists?
-
-      begin
-        # start connections
-        source.host
-        destination.host
-
-        unless opts[:to_safe] || destination.local?
-          raise PgSync::Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
-        end
-
-        print_description("From", source)
-        print_description("To", destination)
-      ensure
-        source.close
-        destination.close
-      end
-
-      tables = nil
-      begin
-        tables = TableList.new(args, opts, source, config).tables
-      ensure
-        source.close
-      end
-
-      confirm_tables_exist(source, tables, "source")
-
-      if opts[:list]
-        confirm_tables_exist(destination, tables, "destination")
-
-        list_items =
-          if args[0] == "groups"
-            (config["groups"] || {}).keys
-          else
-            tables.keys
-          end
-
-        pretty_list list_items
-      else
-        if opts[:schema_first] || opts[:schema_only]
-          if opts[:preserve]
-            raise PgSync::Error, "Cannot use --preserve with --schema-first or --schema-only"
-          end
-
-          log "* Dumping schema"
-          schema_tables = tables if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude]
-          sync_schema(source, destination, schema_tables)
-        end
-
-        unless opts[:schema_only]
-          confirm_tables_exist(destination, tables, "destination")
-
-          in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts|
-            TableSync.new.sync(config, table, opts.merge(table_opts), source.url, destination.url)
-          end
-        end
-
-        log_completed(start_time)
-      end
-    end
-
-    def confirm_tables_exist(data_source, tables, description)
-      tables.keys.each do |table|
-        unless data_source.table_exists?(table)
-          raise PgSync::Error, "Table does not exist in #{description}: #{table}"
-        end
-      end
-    ensure
-      data_source.close
-    end
-
-    def map_deprecations(args, opts)
-      command = args[0]
-
-      case command
-      when "setup"
-        args.shift
-        opts[:init] = true
-        deprecated "Use `psync --init` instead"
-      when "schema"
-        args.shift
-        opts[:schema_only] = true
-        deprecated "Use `psync --schema-only` instead"
-      when "tables"
-        args.shift
-        opts[:tables] = args.shift
-        deprecated "Use `pgsync #{opts[:tables]}` instead"
-      when "groups"
-        args.shift
-        opts[:groups] = args.shift
-        deprecated "Use `pgsync #{opts[:groups]}` instead"
-      end
-
-      if opts[:where]
-        opts[:sql] ||= String.new
-        opts[:sql] << " WHERE #{opts[:where]}"
-        deprecated "Use `\"WHERE #{opts[:where]}\"` instead"
-      end
-
-      if opts[:limit]
-        opts[:sql] ||= String.new
-        opts[:sql] << " LIMIT #{opts[:limit]}"
-        deprecated "Use `\"LIMIT #{opts[:limit]}\"` instead"
-      end
+    def self.start
+      new(ARGV).perform(testing: false)
     end
 
-    def sync_schema(source, destination, tables = nil)
-      dump_command = source.dump_command(tables)
-      restore_command = destination.restore_command
-      unless system("#{dump_command} | #{restore_command}")
-        raise PgSync::Error, "Schema sync returned non-zero exit code"
-      end
-    end
+    protected
 
-    def parse_args(args)
-      opts = Slop.parse(args) do |o|
+    def parse_args
+      Slop.parse(@args) do |o|
         o.banner = %{Usage:
     pgsync [options]
 
@@ -186,165 +64,13 @@ Options:}
         o.boolean "--in-batches", "in batches", default: false, help: false
         o.integer "--batch-size", "batch size", default: 10000, help: false
         o.float "--sleep", "sleep", default: 0, help: false
-        o.on "-v", "--version", "print the version" do
-          log PgSync::VERSION
-          @exit = true
-        end
-        o.on "-h", "--help", "prints help" do
-          log o
-          @exit = true
-        end
+        o.boolean "--fail-fast", "stop on the first failed table", default: false
+        # o.array "--var", "pass a variable"
+        o.boolean "-v", "--version", "print the version"
+        o.boolean "-h", "--help", "prints help"
       end
-
-      opts_hash = opts.to_hash
-      opts_hash[:init] = opts_hash[:setup] if opts_hash[:setup]
-
-      [opts.arguments, opts_hash]
     rescue Slop::Error => e
-      raise PgSync::Error, e.message
-    end
-
-    def config
-      @config ||= begin
-        if config_file
-          begin
-            YAML.load_file(config_file) || {}
-          rescue Psych::SyntaxError => e
-            raise PgSync::Error, e.message
-          end
-        else
-          {}
-        end
-      end
-    end
-
-    def setup(config_file)
-      if File.exist?(config_file)
-        raise PgSync::Error, "#{config_file} exists."
-      else
-        FileUtils.cp(File.dirname(__FILE__) + "/../../config.yml", config_file)
-        log "#{config_file} created. Add your database credentials."
-      end
-    end
-
-    def db_config_file(db)
-      return unless db
-      ".pgsync-#{db}.yml"
-    end
-
-    def print_description(prefix, source)
-      location = " on #{source.host}:#{source.port}" if source.host
-      log "#{prefix}: #{source.dbname}#{location}"
-    end
-
-    def search_tree(file)
-      path = Dir.pwd
-      # prevent infinite loop
-      20.times do
-        absolute_file = File.join(path, file)
-        if File.exist?(absolute_file)
-          break absolute_file
-        end
-        path = File.dirname(path)
-        break if path == "/"
-      end
-    end
-
-    def config_file
-      return @config_file if instance_variable_defined?(:@config_file)
-
-      @config_file =
-        search_tree(
-          if @options[:db]
-            db_config_file(@options[:db])
-          else
-            @options[:config] || ".pgsync.yml"
-          end
-        )
-    end
-
-    def log(message = nil)
-      $stderr.puts message
-    end
-
-    def in_parallel(tables, first_schema:, &block)
-      spinners = TTY::Spinner::Multi.new(format: :dots)
-      item_spinners = {}
-
-      start = lambda do |item, i|
-        table, opts = item
-        message = String.new(":spinner ")
-        message << table.sub("#{first_schema}.", "")
-        # maybe output later
-        # message << " #{opts[:sql]}" if opts[:sql]
-        spinner = spinners.register(message)
-        spinner.auto_spin
-        item_spinners[item] = spinner
-      end
-
-      errors = 0
-
-      finish = lambda do |item, i, result|
-        spinner = item_spinners[item]
-        if result[:status] == "success"
-          spinner.success(display_message(result))
-        else
-          # TODO add option to fail fast
-          spinner.error(display_message(result))
-          errors += 1
-        end
-
-        unless spinner.send(:tty?)
-          status = result[:status] == "success" ? "✔" : "✖"
-          log [status, item.first.sub("#{first_schema}.", ""), display_message(result)].compact.join(" ")
-        end
-      end
-
-      options = {start: start, finish: finish}
-      if @options[:debug] || @options[:in_batches]
-        options[:in_processes] = 0
-      else
-        options[:in_threads] = 4 if windows?
-      end
-
-      Parallel.each(tables, **options, &block)
-
-      raise PgSync::Error, "Synced failed for #{errors} table#{errors == 1 ? nil : "s"}" if errors > 0
-    end
-
-    def display_message(result)
-      message = String.new("")
-      message << "- #{result[:time]}s" if result[:time]
-      message << "(#{result[:message].gsub("\n", " ").strip})" if result[:message]
-      message
-    end
-
-    def pretty_list(items)
-      items.each do |item|
-        log item
-      end
-    end
-
-    def deprecated(message)
-      log "[DEPRECATED] #{message}"
-    end
-
-    def log_completed(start_time)
-      time = Time.now - start_time
-      message = "Completed in #{time.round(1)}s"
-      log self.class.colorize(message, 32) # green
-    end
-
-    def windows?
-      Gem.win_platform?
-    end
-
-    def self.colorize(message, color_code)
-      if $stderr.tty?
-        "\e[#{color_code}m#{message}\e[0m"
-      else
-        message
-      end
+      raise Error, e.message
     end
   end
 end

data/lib/pgsync/data_source.rb

@@ -95,10 +95,8 @@ module PgSync
             config = {dbname: @url}
           end
           PG::Connection.new(config)
-        rescue PG::ConnectionBad => e
-          raise PgSync::Error, e.message
         rescue URI::InvalidURIError
-          raise PgSync::Error, "Invalid connection string"
+          raise Error, "Invalid connection string"
         end
       end
     end
@@ -116,8 +114,7 @@ module PgSync
     end
 
     def restore_command
-      psql_version = `psql --version`.lines[0].chomp.split(" ")[-1].split(/[^\d.]/)[0]
-      if_exists = Gem::Version.new(psql_version) >= Gem::Version.new("9.4.0")
+      if_exists = Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
       "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@url}"
     end
 
@@ -137,6 +134,12 @@ module PgSync
 
     private
 
+    def pg_restore_version
+      `pg_restore --version`.lines[0].chomp.split(" ")[-1].split(/[^\d.]/)[0]
+    rescue Errno::ENOENT
+      raise Error, "pg_restore not found"
+    end
+
     def table_set
       @table_set ||= Set.new(tables)
     end
@@ -177,7 +180,7 @@ module PgSync
           command = m[2..-2]
           result = `#{command}`.chomp
           unless $?.success?
-            raise PgSync::Error, "Command exited with non-zero status:\n#{command}"
+            raise Error, "Command exited with non-zero status:\n#{command}"
           end
           result
         end

data/lib/pgsync/init.rb

@@ -0,0 +1,42 @@
+module PgSync
+  class Init
+    include Utils
+
+    def perform(opts)
+      # needed for config_file method
+      @options = opts.to_hash
+
+      file = db_config_file(opts.arguments[0]) || config_file || ".pgsync.yml"
+
+      if File.exist?(file)
+        raise Error, "#{file} exists."
+      else
+        exclude =
+          if rails_app?
+            <<~EOS
+              exclude:
+                - schema_migrations
+                - ar_internal_metadata
+            EOS
+          else
+            <<~EOS
+              # exclude:
+              #   - schema_migrations
+              #   - ar_internal_metadata
+            EOS
+          end
+
+        # create file
+        contents = File.read(__dir__ + "/../../config.yml")
+        File.write(file, contents % {exclude: exclude})
+
+        log "#{file} created. Add your database credentials."
+      end
+    end
+
+    # TODO maybe check parent directories
+    def rails_app?
+      File.exist?("bin/rails")
+    end
+  end
+end

data/lib/pgsync/sync.rb

@@ -0,0 +1,240 @@
+module PgSync
+  class Sync
+    include Utils
+
+    def perform(options)
+      args = options.arguments
+      opts = options.to_hash
+      @options = opts
+
+      # merge config
+      [:to, :from, :to_safe, :exclude, :schemas].each do |opt|
+        opts[opt] ||= config[opt.to_s]
+      end
+
+      # TODO remove deprecations
+      map_deprecations(args, opts)
+
+      # start
+      start_time = Time.now
+
+      if args.size > 2
+        raise Error, "Usage:\n    pgsync [options]"
+      end
+
+      source = DataSource.new(opts[:from])
+      raise Error, "No source" unless source.exists?
+
+      destination = DataSource.new(opts[:to])
+      raise Error, "No destination" unless destination.exists?
+
+      begin
+        # start connections
+        source.host
+        destination.host
+
+        unless opts[:to_safe] || destination.local?
+          raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
+        end
+
+        print_description("From", source)
+        print_description("To", destination)
+      ensure
+        source.close
+        destination.close
+      end
+
+      tables = nil
+      begin
+        tables = TableList.new(args, opts, source, config).tables
+      ensure
+        source.close
+      end
+
+      confirm_tables_exist(source, tables, "source")
+
+      if opts[:list]
+        confirm_tables_exist(destination, tables, "destination")
+
+        list_items =
+          if args[0] == "groups"
+            (config["groups"] || {}).keys
+          else
+            tables.keys
+          end
+
+        pretty_list list_items
+      else
+        if opts[:schema_first] || opts[:schema_only]
+          if opts[:preserve]
+            raise Error, "Cannot use --preserve with --schema-first or --schema-only"
+          end
+
+          log "* Dumping schema"
+          schema_tables = tables if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude]
+          sync_schema(source, destination, schema_tables)
+        end
+
+        unless opts[:schema_only]
+          confirm_tables_exist(destination, tables, "destination")
+
+          in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts|
+            TableSync.new.sync(config, table, opts.merge(table_opts), source.url, destination.url)
+          end
+        end
+
+        log_completed(start_time)
+      end
+    end
+
+    def confirm_tables_exist(data_source, tables, description)
+      tables.keys.each do |table|
+        unless data_source.table_exists?(table)
+          raise Error, "Table does not exist in #{description}: #{table}"
+        end
+      end
+    ensure
+      data_source.close
+    end
+
+    def map_deprecations(args, opts)
+      command = args[0]
+
+      case command
+      when "schema"
+        args.shift
+        opts[:schema_only] = true
+        deprecated "Use `psync --schema-only` instead"
+      when "tables"
+        args.shift
+        opts[:tables] = args.shift
+        deprecated "Use `pgsync #{opts[:tables]}` instead"
+      when "groups"
+        args.shift
+        opts[:groups] = args.shift
+        deprecated "Use `pgsync #{opts[:groups]}` instead"
+      end
+
+      if opts[:where]
+        opts[:sql] ||= String.new
+        opts[:sql] << " WHERE #{opts[:where]}"
+        deprecated "Use `\"WHERE #{opts[:where]}\"` instead"
+      end
+
+      if opts[:limit]
+        opts[:sql] ||= String.new
+        opts[:sql] << " LIMIT #{opts[:limit]}"
+        deprecated "Use `\"LIMIT #{opts[:limit]}\"` instead"
+      end
+    end
+
+    def sync_schema(source, destination, tables = nil)
+      dump_command = source.dump_command(tables)
+      restore_command = destination.restore_command
+      unless system("#{dump_command} | #{restore_command}")
+        raise Error, "Schema sync returned non-zero exit code"
+      end
+    end
+
+    def config
+      @config ||= begin
+        if config_file
+          begin
+            YAML.load_file(config_file) || {}
+          rescue Psych::SyntaxError => e
+            raise Error, e.message
+          end
+        else
+          {}
+        end
+      end
+    end
+
+    def print_description(prefix, source)
+      location = " on #{source.host}:#{source.port}" if source.host
+      log "#{prefix}: #{source.dbname}#{location}"
+    end
+
+    def in_parallel(tables, first_schema:, &block)
+      spinners = TTY::Spinner::Multi.new(format: :dots)
+      item_spinners = {}
+
+      start = lambda do |item, i|
+        table, opts = item
+        message = String.new(":spinner ")
+        message << table.sub("#{first_schema}.", "")
+        # maybe output later
+        # message << " #{opts[:sql]}" if opts[:sql]
+        spinner = spinners.register(message)
+        spinner.auto_spin
+        item_spinners[item] = spinner
+      end
+
+      failed_tables = []
+
+      finish = lambda do |item, i, result|
+        spinner = item_spinners[item]
+        table_name = item.first.sub("#{first_schema}.", "")
+
+        if result[:status] == "success"
+          spinner.success(display_message(result))
+        else
+          # TODO add option to fail fast
+          spinner.error(display_message(result))
+          failed_tables << table_name
+          fail_sync(failed_tables) if @options[:fail_fast]
+        end
+
+        unless spinner.send(:tty?)
+          status = result[:status] == "success" ? "✔" : "✖"
+          log [status, table_name, display_message(result)].compact.join(" ")
+        end
+      end
+
+      options = {start: start, finish: finish}
+      if @options[:debug] || @options[:in_batches]
+        options[:in_processes] = 0
+      else
+        options[:in_threads] = 4 if windows?
+      end
+
+      # could try to use `raise Parallel::Kill` to fail faster with --fail-fast
+      # see `fast_faster` branch
+      # however, need to make sure connections are cleaned up properly
+      Parallel.each(tables, **options, &block)
+
+      fail_sync(failed_tables) if failed_tables.any?
+    end
+
+    def fail_sync(failed_tables)
+      raise Error, "Sync failed for #{failed_tables.size} table#{failed_tables.size == 1 ? nil : "s"}: #{failed_tables.join(", ")}"
+    end
+
+    def display_message(result)
+      messages = []
+      messages << "- #{result[:time]}s" if result[:time]
+      messages << "(#{result[:message].gsub("\n", " ").strip})" if result[:message]
+      messages.join(" ")
+    end
+
+    def pretty_list(items)
+      items.each do |item|
+        log item
+      end
+    end
+
+    def deprecated(message)
+      log colorize("[DEPRECATED] #{message}", 33) # yellow
+    end
+
+    def log_completed(start_time)
+      time = Time.now - start_time
+      message = "Completed in #{time.round(1)}s"
+      log colorize(message, 32) # green
+    end
+
+    def windows?
+      Gem.win_platform?
+    end
+  end
+end

data/lib/pgsync/table_list.rb

@@ -1,5 +1,7 @@
 module PgSync
   class TableList
+    include Utils
+
     attr_reader :args, :opts, :source, :config
 
     def initialize(args, options, source, config)
@@ -20,7 +22,7 @@ module PgSync
           if (t = (config["groups"] || {})[group])
             add_tables(tables, t, id, args[1])
           else
-            raise PgSync::Error, "Group not found: #{group}"
+            raise Error, "Group not found: #{group}"
           end
         end
       end
@@ -29,7 +31,7 @@ module PgSync
         tables ||= Hash.new { |hash, key| hash[key] = {} }
         to_arr(opts[:tables]).each do |tag|
           table, id = tag.split(":", 2)
-          raise PgSync::Error, "Cannot use parameters with tables" if id
+          raise Error, "Cannot use parameters with tables" if id
           add_table(tables, table, id, args[1])
         end
       end
@@ -43,7 +45,7 @@ module PgSync
           if (t = (config["groups"] || {})[group])
             add_tables(tables, t, id, args[1])
           else
-            raise PgSync::Error, "Cannot use parameters with tables" if id
+            raise Error, "Cannot use parameters with tables" if id
             add_table(tables, group, id, args[1])
           end
         end
@@ -78,6 +80,12 @@ module PgSync
     end
 
     def add_tables(tables, t, id, boom)
+      # if id
+      #   # TODO show group name and value
+      #   log colorize("`pgsync group:value` is deprecated and will have a different function in 0.6.0.", 33) # yellow
+      #   log colorize("Use `pgsync group --var 1=value` instead.", 33) # yellow
+      # end
+
       t.each do |table|
         sql = nil
         if table.is_a?(Array)
@@ -96,10 +104,38 @@ module PgSync
         end
       else
         tables[table] = {}
-        tables[table][:sql] = boom.gsub("{id}", cast(id)).gsub("{1}", cast(id)) if boom
+        if boom
+          sql = boom.dup
+          # vars must match \w
+          missing_vars = sql.scan(/{\w+}/).map { |v| v[1..-2] }
+
+          vars = {}
+
+          # legacy
+          if id
+            vars["id"] = cast(id)
+            vars["1"] = cast(id)
+          end
+
+          # opts[:var].each do |value|
+          #   k, v = value.split("=", 2)
+          #   vars[k] = v
+          # end
+
+          sql = boom.dup
+          vars.each do |k, v|
+            # only sub if in var list
+            sql.gsub!("{#{k}}", cast(v)) if missing_vars.delete(k)
+          end
+
+          raise Error, "Missing variables: #{missing_vars.uniq.join(", ")}" if missing_vars.any?
+
+          tables[table][:sql] = sql
+        end
       end
     end
 
+    # TODO quote vars in next major version
     def cast(value)
       value.to_s.gsub(/\A\"|\"\z/, '')
     end

data/lib/pgsync/table_sync.rb

@@ -1,5 +1,7 @@
 module PgSync
   class TableSync
+    include Utils
+
     def sync(config, table, opts, source_url, destination_url)
       start_time = Time.now
       source = DataSource.new(source_url, timeout: 0)
@@ -57,8 +59,8 @@ module PgSync
 
           copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{sql_clause}) TO STDOUT"
           if opts[:in_batches]
-            raise PgSync::Error, "Cannot use --overwrite with --in-batches" if opts[:overwrite]
-            raise PgSync::Error, "No primary key" unless primary_key
+            raise Error, "Cannot use --overwrite with --in-batches" if opts[:overwrite]
+            raise Error, "No primary key" unless primary_key
 
             destination.truncate(table) if opts[:truncate]
 
@@ -99,8 +101,10 @@ module PgSync
                 sleep(opts[:sleep])
               end
             end
+
+            log # add extra line for spinner
           elsif !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
-            raise PgSync::Error, "No primary key" unless primary_key
+            raise Error, "No primary key" unless primary_key
 
             temp_table = "pgsync_#{rand(1_000_000_000)}"
             file = Tempfile.new(temp_table)
@@ -160,8 +164,20 @@ module PgSync
         source.close
         destination.close
       end
-    rescue PgSync::Error => e
-      {status: "error", message: e.message}
+    rescue => e
+      message =
+        case e
+        when PG::Error
+          # likely fine to show simplified message here
+          # the full message will be shown when first trying to connect
+          "Connection failed"
+        when Error
+          e.message
+        else
+          "#{e.class.name}: #{e.message}"
+        end
+
+      {status: "error", message: message}
     end
 
     private
@@ -180,7 +196,7 @@ module PgSync
         elsif rule.key?("statement")
           rule["statement"]
         else
-          raise PgSync::Error, "Unknown rule #{rule.inspect} for column #{column}"
+          raise Error, "Unknown rule #{rule.inspect} for column #{column}"
         end
       else
         case rule
@@ -207,7 +223,7 @@ module PgSync
         when "null", nil
           "NULL"
         else
-          raise PgSync::Error, "Unknown rule #{rule} for column #{column}"
+          raise Error, "Unknown rule #{rule} for column #{column}"
         end
       end
     end
@@ -217,10 +233,6 @@ module PgSync
       "#{quote_ident_full(table)}.#{quote_ident(primary_key)}"
     end
 
-    def log(message = nil)
-      $stderr.puts message
-    end
-
     def quote_ident_full(ident)
       ident.split(".").map { |v| quote_ident(v) }.join(".")
     end

data/lib/pgsync/utils.rb

@@ -0,0 +1,46 @@
+module PgSync
+  module Utils
+    def log(message = nil)
+      $stderr.puts message
+    end
+
+    def colorize(message, color_code)
+      if $stderr.tty?
+        "\e[#{color_code}m#{message}\e[0m"
+      else
+        message
+      end
+    end
+
+    def config_file
+      return @config_file if instance_variable_defined?(:@config_file)
+
+      @config_file =
+        search_tree(
+          if @options[:db]
+            db_config_file(@options[:db])
+          else
+            @options[:config] || ".pgsync.yml"
+          end
+        )
+    end
+
+    def db_config_file(db)
+      return unless db
+      ".pgsync-#{db}.yml"
+    end
+
+    def search_tree(file)
+      path = Dir.pwd
+      # prevent infinite loop
+      20.times do
+        absolute_file = File.join(path, file)
+        if File.exist?(absolute_file)
+          break absolute_file
+        end
+        path = File.dirname(path)
+        break if path == "/"
+      end
+    end
+  end
+end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-27 00:00:00.000000000 Z
+date: 2020-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -123,8 +123,11 @@ files:
 - lib/pgsync.rb
 - lib/pgsync/client.rb
 - lib/pgsync/data_source.rb
+- lib/pgsync/init.rb
+- lib/pgsync/sync.rb
 - lib/pgsync/table_list.rb
 - lib/pgsync/table_sync.rb
+- lib/pgsync/utils.rb
 - lib/pgsync/version.rb
 homepage: https://github.com/ankane/pgsync
 licenses: