pgsync 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b145ab9d9f312a9a043e91d5d4b9e7977bb5788e339aa4f88a571a5178d9c92f
-  data.tar.gz: 8843f622d7b152fff80a2a465fddd55dcd4271f0256cd64b46a3898a38214d65
+  metadata.gz: f6c20cf99ebc0961d8699228883f6c2e9c95c9798cfb6f101cb78803df6b1b43
+  data.tar.gz: 7c719442c07e6db1d4704199aa2b95d32e1ba528797b2d21b15be85f2de71d79
 SHA512:
-  metadata.gz: f210607011225b932975ddd77343c1ff9468ae824a642b3c0b48033853a2fa0d68f4eed4fcfcda5c36a6d7cc1b99e491a6a0b62f147d232c8d517721b08a6590
-  data.tar.gz: bb38c6291b40ad0188483889aa9696269a384456ed7425be61e624a26b797110dc363a44b9d0774fc4b2ad04122d6e8863d04b7bb1a89f986c0829ada8a2aea4
+  metadata.gz: 1e3d735e6e3002e2fcb74de574f63f534472821aca0154c13d9d7ce6e9fa94426239cef875cd33724fec5a6cf2567b5656a80a9d92f89d9678ee6bee2c2ec70c
+  data.tar.gz: 3ab5d416f4c2364644c015d106631898a3615cc671179f8b1193cda5b1e894512fb39906ec1c27570ca9fbd186c9eb19af3afb88eba093bb21e8bc471c28f18b

data/CHANGELOG.md

@@ -1,3 +1,43 @@
+## 0.6.0 (2020-06-07)
+
+- Added messages for different column types and non-deferrable constraints
+- Added support for wildcards to `--exclude`
+- Improved `--overwrite` and `--preserve` options for foreign keys
+- Improved output for schema sync
+- Fixed `--overwrite` and `--preserve` options for multicolumn primary keys
+- Fixed output for notices
+
+Breaking
+
+- Syncs shared tables instead of raising an error when tables missing in destination
+- Raise an error when `--config` or `--db` option provided and config not found
+- Removed deprecated options
+- Dropped support for Postgres < 9.5
+
+## 0.5.5 (2020-05-13)
+
+- Added `--jobs` option
+- Added `--defer-constraints` option
+- Added `--disable-user-triggers` option
+- Added `--disable-integrity` option
+- Improved error message for older libpq
+
+## 0.5.4 (2020-05-09)
+
+- Fixed output for `--in-batches`
+
+## 0.5.3 (2020-04-03)
+
+- Improved Postgres error messages
+- Fixed behavior of wildcard without schema
+
+## 0.5.2 (2020-03-27)
+
+- Added `--fail-fast` option
+- Automatically exclude tables when `--init` run inside Rails app
+- Improved error message
+- Fixed typo in error message
+
 ## 0.5.1 (2020-03-26)
 
 - Fixed Slop warning with Ruby 2.7

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2019 Andrew Kane
+Copyright (c) 2015-2020 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,9 +1,10 @@
 # pgsync
 
-Sync Postgres data between databases. Designed for:
+Sync data from one Postgres database to another (like `pg_dump`/`pg_restore`). Designed for:
 
-- **speed** - up to 4x faster than traditional tools on a 4-core machine
+- **speed** - tables are transferred in parallel
 - **security** - built-in methods to prevent sensitive data from ever leaving the server
+- **flexibility** - gracefully handles schema differences, like missing columns and extra columns
 - **convenience** - sync partial tables, groups of tables, and related records
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
@@ -20,6 +21,8 @@ gem install pgsync
 
 This will give you the `pgsync` command. If installation fails, you may need to install [dependencies](#dependencies).
 
+## Setup
+
 In your project directory, run:
 
 ```sh
@@ -30,20 +33,26 @@ This creates `.pgsync.yml` for you to customize. We recommend checking this into
 
 ## How to Use
 
+First, make sure your schema is set up in both databases. We recommend using a schema migration tool for this, but pgsync also provides a few [convenience methods](#schema). Once that’s done, you’re ready to sync data.
+
 Sync all tables
 
 ```sh
 pgsync
 ```
 
-**Note:** pgsync assumes your schema is already set up on your local machine. See the [schema section](#schema) if that’s not the case.
-
 Sync specific tables
 
 ```sh
 pgsync table1,table2
 ```
 
+Works with wildcards as well
+
+```sh
+pgsync "table*"
+```
+
 Sync specific rows (existing rows are overwritten)
 
 ```sh
@@ -62,13 +71,15 @@ Or truncate them
 pgsync products "where store_id = 1" --truncate
 ```
 
-### Exclude Tables
+## Tables
+
+Exclude specific tables
 
 ```sh
-pgsync --exclude users
+pgsync --exclude table1,table2
 ```
 
-To always exclude, add to `.pgsync.yml`.
+Add to `.pgsync.yml` to exclude by default
 
 ```yml
 exclude:
@@ -76,15 +87,17 @@ exclude:
   - table2
 ```
 
-For Rails, you probably want to exclude schema migrations and ActiveRecord metadata.
+Sync tables from all schemas or specific schemas (by default, only the search path is synced)
 
-```yml
-exclude:
-  - schema_migrations
-  - ar_internal_metadata
+```sh
+pgsync --all-schemas
+# or
+pgsync --schemas public,other
+# or
+pgsync public.table1,other.table2
 ```
 
-### Groups
+## Groups
 
 Define groups in `.pgsync.yml`:
 
@@ -101,6 +114,8 @@ And run:
 pgsync group1
 ```
 
+## Variables
+
 You can also use groups to sync a specific record and associated records in other tables.
 
 To get product `123` with its reviews, last 10 coupons, and store, use:
@@ -120,18 +135,14 @@ And run:
 pgsync product:123
 ```
 
-### Schema
+## Schema
 
-**Note:** pgsync is designed to sync data. You should use a schema migration tool to manage schema changes. The methods in this section are provided for convenience but not recommended.
-
-Sync schema before the data
+Sync schema before the data (this wipes out existing data)
 
 ```sh
 pgsync --schema-first
 ```
 
-**Note:** This wipes out existing data
-
 Specify tables
 
 ```sh
@@ -146,13 +157,9 @@ pgsync --schema-only
 
 pgsync does not try to sync Postgres extensions.
 
-## Data Protection
+## Sensitive Data
 
-Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
-
-## Sensitive Information
-
-Prevent sensitive information like email addresses from leaving the remote server.
+Prevent sensitive data like email addresses from leaving the remote server.
 
 Define rules in `.pgsync.yml`:
 
@@ -185,31 +192,45 @@ Options for replacement are:
 - `null`
 - `untouched`
 
-Rules starting with `unique_` require the table to have a primary key. `unique_phone` requires a numeric primary key.
+Rules starting with `unique_` require the table to have a single column primary key. `unique_phone` requires a numeric primary key.
 
-## Multiple Databases
+## Foreign Keys
 
-To use with multiple databases, run:
+Foreign keys can make it difficult to sync data. Three options are:
+
+1. Manually specify the order of tables
+2. Use deferrable constraints
+3. Disable foreign key triggers, which can silently break referential integrity
+
+When manually specifying the order, use `--jobs 1` so tables are synced one-at-a-time.
 
 ```sh
-pgsync --init db2
+pgsync table1,table2,table3 --jobs 1
 ```
 
-This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands with:
+If your tables have [deferrable constraints](https://begriffs.com/posts/2017-08-27-deferrable-sql-constraints.html), use:
 
 ```sh
-pgsync --db db2
+pgsync --defer-constraints
 ```
 
-## Safety
+To disable foreign key triggers and potentially break referential integrity, use:
 
-To keep you from accidentally overwriting production, the destination is limited to `localhost` or `127.0.0.1` by default.
+```sh
+pgsync --disable-integrity
+```
 
-To use another host, add `to_safe: true` to your `.pgsync.yml`.
+## Triggers
+
+Disable user triggers with:
 
-## Large Tables
+```sh
+pgsync --disable-user-triggers
+```
 
-For extremely large tables, sync in batches.
+## Append-Only Tables
+
+For extremely large, append-only tables, sync in batches.
 
 ```sh
 pgsync large_table --in-batches
@@ -217,15 +238,31 @@ pgsync large_table --in-batches
 
 The script will resume where it left off when run again, making it great for backfills.
 
-## Foreign Keys
+## Connection Security
+
+Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
+
+## Safety
+
+To keep you from accidentally overwriting production, the destination is limited to `localhost` or `127.0.0.1` by default.
+
+To use another host, add `to_safe: true` to your `.pgsync.yml`.
 
-By default, tables are copied in parallel. If you use foreign keys, this can cause violations. You can specify tables to be copied serially with:
+## Multiple Databases
+
+To use with multiple databases, run:
+
+```sh
+pgsync --init db2
+```
+
+This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands with:
 
 ```sh
-pgsync group1 --debug
+pgsync --db db2
 ```
 
-## Reference
+## Other Commands
 
 Help
 
@@ -239,6 +276,12 @@ Version
 pgsync --version
 ```
 
+List tables
+
+```sh
+pgsync --list
+```
+
 ## Scripts
 
 Use groups when possible to take advantage of parallelism.
@@ -246,7 +289,7 @@ Use groups when possible to take advantage of parallelism.
 For Ruby scripts, you may need to do:
 
 ```rb
-Bundler.with_clean_env do
+Bundler.with_unbundled_env do
   system "pgsync ..."
 end
 ```

data/config.yml

@@ -13,16 +13,17 @@ from: $(some_command)?sslmode=require
 to: postgres://localhost:5432/myapp_development
 
 # exclude tables
-# exclude:
-#   - schema_migrations
-#   - ar_internal_metadata
-
+%{exclude}
 # define groups
 # groups:
 #   group1:
 #     - table1
 #     - table2
 
+# sync specific schemas
+# schemas:
+#   - public
+
 # protect sensitive information
 data_rules:
   email: unique_email

data/exe/pgsync

@@ -1,15 +1,7 @@
 #!/usr/bin/env ruby
 
-# disable Slop warning with Ruby 2.7
-$VERBOSE = nil
-
+# handle interrupts
 trap("SIGINT") { abort }
 
-begin
-  require "pgsync"
-  PgSync::Client.new(ARGV).perform
-rescue PgSync::Error => e
-  abort PgSync::Client.colorize(e.message, 31) # red
-rescue Interrupt
-  abort
-end
+require "pgsync"
+PgSync::Client.start

data/lib/pgsync.rb

@@ -5,21 +5,24 @@ require "slop"
 require "tty-spinner"
 
 # stdlib
-require "cgi"
-require "erb"
-require "fileutils"
 require "set"
 require "shellwords"
 require "tempfile"
-require "thread" # windows only
 require "uri"
 require "yaml"
+require "open3"
 
 # modules
+require "pgsync/utils"
 require "pgsync/client"
 require "pgsync/data_source"
-require "pgsync/table_list"
+require "pgsync/init"
+require "pgsync/schema_sync"
+require "pgsync/sync"
+require "pgsync/table"
 require "pgsync/table_sync"
+require "pgsync/task"
+require "pgsync/task_resolver"
 require "pgsync/version"
 
 module PgSync

data/lib/pgsync/client.rb

@@ -1,350 +1,78 @@
 module PgSync
   class Client
+    include Utils
+
     def initialize(args)
-      $stdout.sync = true
-      $stderr.sync = true
-      @exit = false
-      @arguments, @options = parse_args(args)
+      @args = args
+      output.sync = true
     end
 
-    # TODO clean up this mess
     def perform
-      return if @exit
-
-      args, opts = @arguments, @options
-      [:to, :from, :to_safe, :exclude, :schemas].each do |opt|
-        opts[opt] ||= config[opt.to_s]
-      end
-      map_deprecations(args, opts)
-
-      if opts[:init]
-        setup(db_config_file(args[0]) || config_file || ".pgsync.yml")
-      else
-        sync(args, opts)
-      end
-
-      true
-    end
-
-    protected
-
-    def sync(args, opts)
-      start_time = Time.now
-
-      if args.size > 2
-        raise PgSync::Error, "Usage:\n    pgsync [options]"
-      end
-
-      source = DataSource.new(opts[:from])
-      raise PgSync::Error, "No source" unless source.exists?
-
-      destination = DataSource.new(opts[:to])
-      raise PgSync::Error, "No destination" unless destination.exists?
-
-      begin
-        # start connections
-        source.host
-        destination.host
-
-        unless opts[:to_safe] || destination.local?
-          raise PgSync::Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
-        end
-
-        print_description("From", source)
-        print_description("To", destination)
-      ensure
-        source.close
-        destination.close
-      end
-
-      tables = nil
-      begin
-        tables = TableList.new(args, opts, source, config).tables
-      ensure
-        source.close
-      end
-
-      confirm_tables_exist(source, tables, "source")
-
-      if opts[:list]
-        confirm_tables_exist(destination, tables, "destination")
-
-        list_items =
-          if args[0] == "groups"
-            (config["groups"] || {}).keys
-          else
-            tables.keys
-          end
-
-        pretty_list list_items
+      result = Slop::Parser.new(slop_options).parse(@args)
+      arguments = result.arguments
+      options = result.to_h
+
+      raise Error, "Specify either --db or --config, not both" if options[:db] && options[:config]
+      raise Error, "Cannot use --overwrite with --in-batches" if options[:overwrite] && options[:in_batches]
+
+      if options[:version]
+        log VERSION
+      elsif options[:help]
+        log slop_options
+      elsif options[:init]
+        Init.new(arguments, options).perform
       else
-        if opts[:schema_first] || opts[:schema_only]
-          if opts[:preserve]
-            raise PgSync::Error, "Cannot use --preserve with --schema-first or --schema-only"
-          end
-
-          log "* Dumping schema"
-          schema_tables = tables if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude]
-          sync_schema(source, destination, schema_tables)
-        end
-
-        unless opts[:schema_only]
-          confirm_tables_exist(destination, tables, "destination")
-
-          in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts|
-            TableSync.new.sync(config, table, opts.merge(table_opts), source.url, destination.url)
-          end
-        end
-
-        log_completed(start_time)
+        Sync.new(arguments, options).perform
       end
+    rescue => e
+      # Error, PG::ConnectionBad, Slop::Error
+      raise e if options && options[:debug]
+      abort colorize(e.message.strip, :red)
     end
 
-    def confirm_tables_exist(data_source, tables, description)
-      tables.keys.each do |table|
-        unless data_source.table_exists?(table)
-          raise PgSync::Error, "Table does not exist in #{description}: #{table}"
-        end
-      end
-    ensure
-      data_source.close
+    def self.start
+      new(ARGV).perform
     end
 
-    def map_deprecations(args, opts)
-      command = args[0]
-
-      case command
-      when "setup"
-        args.shift
-        opts[:init] = true
-        deprecated "Use `psync --init` instead"
-      when "schema"
-        args.shift
-        opts[:schema_only] = true
-        deprecated "Use `psync --schema-only` instead"
-      when "tables"
-        args.shift
-        opts[:tables] = args.shift
-        deprecated "Use `pgsync #{opts[:tables]}` instead"
-      when "groups"
-        args.shift
-        opts[:groups] = args.shift
-        deprecated "Use `pgsync #{opts[:groups]}` instead"
-      end
-
-      if opts[:where]
-        opts[:sql] ||= String.new
-        opts[:sql] << " WHERE #{opts[:where]}"
-        deprecated "Use `\"WHERE #{opts[:where]}\"` instead"
-      end
-
-      if opts[:limit]
-        opts[:sql] ||= String.new
-        opts[:sql] << " LIMIT #{opts[:limit]}"
-        deprecated "Use `\"LIMIT #{opts[:limit]}\"` instead"
-      end
-    end
-
-    def sync_schema(source, destination, tables = nil)
-      dump_command = source.dump_command(tables)
-      restore_command = destination.restore_command
-      unless system("#{dump_command} | #{restore_command}")
-        raise PgSync::Error, "Schema sync returned non-zero exit code"
-      end
-    end
+    protected
 
-    def parse_args(args)
-      opts = Slop.parse(args) do |o|
-        o.banner = %{Usage:
-    pgsync [options]
+    def slop_options
+      o = Slop::Options.new
+      o.banner = %{Usage:
+  pgsync [options]
 
 Options:}
-        o.string "-d", "--db", "database"
-        o.string "-t", "--tables", "tables to sync"
-        o.string "-g", "--groups", "groups to sync"
-        o.string "--schemas", "schemas to sync"
-        o.string "--from", "source"
-        o.string "--to", "destination"
-        o.string "--where", "where", help: false
-        o.integer "--limit", "limit", help: false
-        o.string "--exclude", "exclude tables"
-        o.string "--config", "config file"
-        # TODO much better name for this option
-        o.boolean "--to-safe", "accept danger", default: false
-        o.boolean "--debug", "debug", default: false
-        o.boolean "--list", "list", default: false
-        o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
-        o.boolean "--preserve", "preserve existing rows", default: false
-        o.boolean "--truncate", "truncate existing rows", default: false
-        o.boolean "--schema-first", "schema first", default: false
-        o.boolean "--schema-only", "schema only", default: false
-        o.boolean "--all-schemas", "all schemas", default: false
-        o.boolean "--no-rules", "do not apply data rules", default: false
-        o.boolean "--no-sequences", "do not sync sequences", default: false
-        o.boolean "--init", "init", default: false
-        o.boolean "--setup", "setup", default: false, help: false
-        o.boolean "--in-batches", "in batches", default: false, help: false
-        o.integer "--batch-size", "batch size", default: 10000, help: false
-        o.float "--sleep", "sleep", default: 0, help: false
-        o.on "-v", "--version", "print the version" do
-          log PgSync::VERSION
-          @exit = true
-        end
-        o.on "-h", "--help", "prints help" do
-          log o
-          @exit = true
-        end
-      end
-
-      opts_hash = opts.to_hash
-      opts_hash[:init] = opts_hash[:setup] if opts_hash[:setup]
-
-      [opts.arguments, opts_hash]
-    rescue Slop::Error => e
-      raise PgSync::Error, e.message
-    end
-
-    def config
-      @config ||= begin
-        if config_file
-          begin
-            YAML.load_file(config_file) || {}
-          rescue Psych::SyntaxError => e
-            raise PgSync::Error, e.message
-          end
-        else
-          {}
-        end
-      end
-    end
-
-    def setup(config_file)
-      if File.exist?(config_file)
-        raise PgSync::Error, "#{config_file} exists."
-      else
-        FileUtils.cp(File.dirname(__FILE__) + "/../../config.yml", config_file)
-        log "#{config_file} created. Add your database credentials."
-      end
-    end
-
-    def db_config_file(db)
-      return unless db
-      ".pgsync-#{db}.yml"
-    end
-
-    def print_description(prefix, source)
-      location = " on #{source.host}:#{source.port}" if source.host
-      log "#{prefix}: #{source.dbname}#{location}"
-    end
-
-    def search_tree(file)
-      path = Dir.pwd
-      # prevent infinite loop
-      20.times do
-        absolute_file = File.join(path, file)
-        if File.exist?(absolute_file)
-          break absolute_file
-        end
-        path = File.dirname(path)
-        break if path == "/"
-      end
-    end
-
-    def config_file
-      return @config_file if instance_variable_defined?(:@config_file)
-
-      @config_file =
-        search_tree(
-          if @options[:db]
-            db_config_file(@options[:db])
-          else
-            @options[:config] || ".pgsync.yml"
-          end
-        )
-    end
-
-    def log(message = nil)
-      $stderr.puts message
-    end
-
-    def in_parallel(tables, first_schema:, &block)
-      spinners = TTY::Spinner::Multi.new(format: :dots)
-      item_spinners = {}
-
-      start = lambda do |item, i|
-        table, opts = item
-        message = String.new(":spinner ")
-        message << table.sub("#{first_schema}.", "")
-        # maybe output later
-        # message << " #{opts[:sql]}" if opts[:sql]
-        spinner = spinners.register(message)
-        spinner.auto_spin
-        item_spinners[item] = spinner
-      end
-
-      errors = 0
-
-      finish = lambda do |item, i, result|
-        spinner = item_spinners[item]
-        if result[:status] == "success"
-          spinner.success(display_message(result))
-        else
-          # TODO add option to fail fast
-          spinner.error(display_message(result))
-          errors += 1
-        end
-
-        unless spinner.send(:tty?)
-          status = result[:status] == "success" ? "✔" : "✖"
-          log [status, item.first.sub("#{first_schema}.", ""), display_message(result)].compact.join(" ")
-        end
-      end
-
-      options = {start: start, finish: finish}
-      if @options[:debug] || @options[:in_batches]
-        options[:in_processes] = 0
-      else
-        options[:in_threads] = 4 if windows?
-      end
-
-      Parallel.each(tables, **options, &block)
-
-      raise PgSync::Error, "Synced failed for #{errors} table#{errors == 1 ? nil : "s"}" if errors > 0
-    end
-
-    def display_message(result)
-      message = String.new("")
-      message << "- #{result[:time]}s" if result[:time]
-      message << "(#{result[:message].gsub("\n", " ").strip})" if result[:message]
-      message
-    end
-
-    def pretty_list(items)
-      items.each do |item|
-        log item
-      end
-    end
-
-    def deprecated(message)
-      log "[DEPRECATED] #{message}"
-    end
-
-    def log_completed(start_time)
-      time = Time.now - start_time
-      message = "Completed in #{time.round(1)}s"
-      log self.class.colorize(message, 32) # green
-    end
-
-    def windows?
-      Gem.win_platform?
-    end
-
-    def self.colorize(message, color_code)
-      if $stderr.tty?
-        "\e[#{color_code}m#{message}\e[0m"
-      else
-        message
-      end
+      o.string "-d", "--db", "database"
+      o.string "-t", "--tables", "tables to sync"
+      o.string "-g", "--groups", "groups to sync"
+      o.integer "-j", "--jobs", "number of tables to sync at a time"
+      o.string "--schemas", "schemas to sync"
+      o.string "--from", "source"
+      o.string "--to", "destination"
+      o.string "--exclude", "exclude tables"
+      o.string "--config", "config file"
+      o.boolean "--to-safe", "accept danger", default: false
+      o.boolean "--debug", "debug", default: false
+      o.boolean "--list", "list", default: false
+      o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
+      o.boolean "--preserve", "preserve existing rows", default: false
+      o.boolean "--truncate", "truncate existing rows", default: false
+      o.boolean "--schema-first", "schema first", default: false
+      o.boolean "--schema-only", "schema only", default: false
+      o.boolean "--all-schemas", "all schemas", default: false
+      o.boolean "--no-rules", "do not apply data rules", default: false
+      o.boolean "--no-sequences", "do not sync sequences", default: false
+      o.boolean "--init", "init", default: false
+      o.boolean "--in-batches", "in batches", default: false, help: false
+      o.integer "--batch-size", "batch size", default: 10000, help: false
+      o.float "--sleep", "sleep", default: 0, help: false
+      o.boolean "--fail-fast", "stop on the first failed table", default: false
+      o.boolean "--defer-constraints", "defer constraints", default: false
+      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+      o.boolean "--disable-integrity", "disable foreign key triggers", default: false
+      o.boolean "-v", "--version", "print the version"
+      o.boolean "-h", "--help", "prints help"
+      o
     end
   end
 end