pgsync 0.3.8 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50eb1d7496a1a6025abd64cb19087781c87deed7
-  data.tar.gz: 6f5c579ff553997d664bc142f91524b2263bf622
+  metadata.gz: f8d5362420e8f65878dcb39bfcb945db069d585f
+  data.tar.gz: a0d4d2b74ca25148f6ddcb2b4bdfa0bea7ef2c99
 SHA512:
-  metadata.gz: c814c85dc2e5d4f9eab683954b0a9473d693e4fa0299b52c75e1525f380380630956d2c50bb0a3212e62eb053bfe4391a108bcb97b4ae51fc3e275b0f1258df7
-  data.tar.gz: 482f1923f915f5123f334b92d0ce2a3444892ed314994aec73b5b8c551ead29dc365e21275b3ec257d48f5c1d4aa0b922db1bcb11f0ccea6478bfeae8ce631bc
+  metadata.gz: ed08a53660ea4973932176da8ea2bb1354f457c67dc9dd9cff832a67eb2b785d399f84a1840e21485579c0d595e81657303213d8831ae8ae72cba6cede713dd2
+  data.tar.gz: 4f56fdbec5b5b93b4ff6ffe781edba94076107203e5f620a5708c3487d8a1cd77e8388a06cf0c9ddbd403afefa06832dd648382dfb3dadde9fc857b83ddea089

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+# 0.3.9
+
+- Better support for schemas
+- Added `--schemas` option
+- Added `--all-schemas` option
+- Added `--schema-first` option
+- Fixed issue with non-lowercase tables and partial syncs
+
 # 0.3.8
 
 - Added Windows support

data/README.md

@@ -122,16 +122,24 @@ pgsync product:123
 
 ### Schema
 
-Sync schema
+Sync schema before the data
 
 ```sh
-pgsync --schema-only
+pgsync --schema-first
 ```
 
+**Note:** This wipes out existing data
+
 Specify tables
 
 ```sh
-pgsync table1,table2 --schema-only
+pgsync table1,table2 --schema-first
+```
+
+Or just the schema
+
+```sh
+pgsync --schema-only
 ```
 
 ## Sensitive Information
@@ -161,6 +169,7 @@ Options for replacement are:
 - statement
 - unique_email
 - unique_phone
+- unique_secret
 - random_letter
 - random_int
 - random_date
@@ -246,7 +255,7 @@ To use master, run:
 
 ```sh
 gem install specific_install
-gem specific_install ankane/pgsync
+gem specific_install https://github.com/ankane/pgsync.git
 ```
 
 ## Thanks

data/lib/pgsync/client.rb

@@ -0,0 +1,280 @@
+module PgSync
+  class Client
+    def initialize(args)
+      $stdout.sync = true
+      @exit = false
+      @arguments, @options = parse_args(args)
+      @mutex = windows? ? Mutex.new : MultiProcessing::Mutex.new
+    end
+
+    # TODO clean up this mess
+    def perform
+      return if @exit
+
+      args, opts = @arguments, @options
+      [:to, :from, :to_safe, :exclude].each do |opt|
+        opts[opt] ||= config[opt.to_s]
+      end
+      map_deprecations(args, opts)
+
+      if opts[:setup]
+        setup(db_config_file(args[0]) || config_file || ".pgsync.yml")
+      else
+        sync(args, opts)
+      end
+
+      true
+    end
+
+    protected
+
+    def sync(args, opts)
+      start_time = Time.now
+
+      if args.size > 2
+        raise PgSync::Error, "Usage:\n    pgsync [options]"
+      end
+
+      source = DataSource.new(opts[:from])
+      raise PgSync::Error, "No source" unless source.exists?
+
+      destination = DataSource.new(opts[:to])
+      raise PgSync::Error, "No destination" unless destination.exists?
+
+      unless opts[:to_safe] || destination.local?
+        raise PgSync::Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
+      end
+
+      print_description("From", source)
+      print_description("To", destination)
+
+      tables = nil
+      begin
+        tables = TableList.new(args, opts, source, config).tables
+      ensure
+        source.close
+      end
+
+      confirm_tables_exist(source, tables, "source")
+
+      if opts[:list]
+        confirm_tables_exist(destination, tables, "destination")
+
+        list_items =
+          if args[0] == "groups"
+            (config["groups"] || {}).keys
+          else
+            tables.keys
+          end
+
+        pretty_list list_items
+      else
+        if opts[:schema_first] || opts[:schema_only]
+          if opts[:preserve]
+            raise PgSync::Error, "Cannot use --preserve with --schema-first or --schema-only"
+          end
+
+          log "* Dumping schema"
+          sync_schema(source, destination, tables)
+        end
+
+        unless opts[:schema_only]
+          confirm_tables_exist(destination, tables, "destination")
+
+          in_parallel(tables) do |table, table_opts|
+            TableSync.new.sync_with_benchmark(@mutex, config, table, opts.merge(table_opts), source.url, destination.url)
+          end
+        end
+
+        log_completed(start_time)
+      end
+    end
+
+    def confirm_tables_exist(destination, tables, description)
+      tables.keys.each do |table|
+        unless destination.table_exists?(table)
+          raise PgSync::Error, "Table does not exist in #{description}: #{table}"
+        end
+      end
+    ensure
+      destination.close
+    end
+
+    def map_deprecations(args, opts)
+      command = args[0]
+
+      case command
+      when "setup"
+        args.shift
+        opts[:setup] = true
+        deprecated "Use `psync --setup` instead"
+      when "schema"
+        args.shift
+        opts[:schema_only] = true
+        deprecated "Use `psync --schema-only` instead"
+      when "tables"
+        args.shift
+        opts[:tables] = args.shift
+        deprecated "Use `pgsync #{opts[:tables]}` instead"
+      when "groups"
+        args.shift
+        opts[:groups] = args.shift
+        deprecated "Use `pgsync #{opts[:groups]}` instead"
+      end
+
+      if opts[:where]
+        opts[:sql] ||= String.new
+        opts[:sql] << " WHERE #{opts[:where]}"
+        deprecated "Use `\"WHERE #{opts[:where]}\"` instead"
+      end
+
+      if opts[:limit]
+        opts[:sql] ||= String.new
+        opts[:sql] << " LIMIT #{opts[:limit]}"
+        deprecated "Use `\"LIMIT #{opts[:limit]}\"` instead"
+      end
+    end
+
+    def sync_schema(source, destination, tables)
+      dump_command = source.dump_command(tables)
+      restore_command = destination.restore_command
+      system("#{dump_command} | #{restore_command}")
+    end
+
+    def parse_args(args)
+      opts = Slop.parse(args) do |o|
+        o.banner = %{Usage:
+    pgsync [options]
+
+Options:}
+        o.string "-d", "--db", "database"
+        o.string "-t", "--tables", "tables to sync"
+        o.string "-g", "--groups", "groups to sync"
+        o.string "--schemas", "schemas to sync"
+        o.string "--from", "source"
+        o.string "--to", "destination"
+        o.string "--where", "where", help: false
+        o.integer "--limit", "limit", help: false
+        o.string "--exclude", "exclude tables"
+        o.string "--config", "config file"
+        # TODO much better name for this option
+        o.boolean "--to-safe", "accept danger", default: false
+        o.boolean "--debug", "debug", default: false
+        o.boolean "--list", "list", default: false
+        o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
+        o.boolean "--preserve", "preserve existing rows", default: false
+        o.boolean "--truncate", "truncate existing rows", default: false
+        o.boolean "--schema-first", "schema first", default: false
+        o.boolean "--schema-only", "schema only", default: false
+        o.boolean "--all-schemas", "all schemas", default: false
+        o.boolean "--no-rules", "do not apply data rules", default: false
+        o.boolean "--setup", "setup", default: false
+        o.boolean "--in-batches", "in batches", default: false, help: false
+        o.integer "--batch-size", "batch size", default: 10000, help: false
+        o.float "--sleep", "sleep", default: 0, help: false
+        o.on "-v", "--version", "print the version" do
+          log PgSync::VERSION
+          @exit = true
+        end
+        o.on "-h", "--help", "prints help" do
+          log o
+          @exit = true
+        end
+      end
+      [opts.arguments, opts.to_hash]
+    rescue Slop::Error => e
+      raise PgSync::Error, e.message
+    end
+
+    def config
+      @config ||= begin
+        if config_file
+          begin
+            YAML.load_file(config_file) || {}
+          rescue Psych::SyntaxError => e
+            raise PgSync::Error, e.message
+          end
+        else
+          {}
+        end
+      end
+    end
+
+    def setup(config_file)
+      if File.exist?(config_file)
+        raise PgSync::Error, "#{config_file} exists."
+      else
+        FileUtils.cp(File.dirname(__FILE__) + "/../../config.yml", config_file)
+        log "#{config_file} created. Add your database credentials."
+      end
+    end
+
+    def db_config_file(db)
+      return unless db
+      ".pgsync-#{db}.yml"
+    end
+
+    def print_description(prefix, source)
+      log "#{prefix}: #{source.uri.path.sub(/\A\//, '')} on #{source.uri.host}:#{source.uri.port}"
+    end
+
+    def search_tree(file)
+      path = Dir.pwd
+      # prevent infinite loop
+      20.times do
+        absolute_file = File.join(path, file)
+        if File.exist?(absolute_file)
+          break absolute_file
+        end
+        path = File.dirname(path)
+        break if path == "/"
+      end
+    end
+
+    def config_file
+      return @config_file if instance_variable_defined?(:@config_file)
+
+      @config_file =
+        search_tree(
+          if @options[:db]
+            db_config_file(@options[:db])
+          else
+            @options[:config] || ".pgsync.yml"
+          end
+        )
+    end
+
+    def log(message = nil)
+      $stderr.puts message
+    end
+
+    def in_parallel(tables, &block)
+      if @options[:debug] || @options[:in_batches]
+        tables.each(&block)
+      else
+        options = {}
+        options[:in_threads] = 4 if windows?
+        Parallel.each(tables, options, &block)
+      end
+    end
+
+    def pretty_list(items)
+      items.each do |item|
+        log item
+      end
+    end
+
+    def deprecated(message)
+      log "[DEPRECATED] #{message}"
+    end
+
+    def log_completed(start_time)
+      time = Time.now - start_time
+      log "Completed in #{time.round(1)}s"
+    end
+
+    def windows?
+      Gem.win_platform?
+    end
+  end
+end

data/lib/pgsync/data_source.rb

@@ -0,0 +1,191 @@
+module PgSync
+  class DataSource
+    attr_reader :url
+
+    def initialize(source)
+      @url = resolve_url(source)
+    end
+
+    def exists?
+      @url && @url.size > 0
+    end
+
+    def local?
+      %w(localhost 127.0.0.1).include?(uri.host)
+    end
+
+    def uri
+      @uri ||= begin
+        uri = URI.parse(@url)
+        uri.scheme ||= "postgres"
+        uri.host ||= "localhost"
+        uri.port ||= 5432
+        uri.path = "/#{uri.path}" if uri.path && uri.path[0] != "/"
+        uri
+      end
+    end
+
+    def schema
+      @schema ||= CGI.parse(uri.query.to_s)["schema"][0]
+    end
+
+    def tables
+      query = "SELECT schemaname, tablename FROM pg_catalog.pg_tables WHERE schemaname NOT IN ('information_schema', 'pg_catalog') ORDER BY 1, 2"
+      execute(query).map { |row| "#{row["schemaname"]}.#{row["tablename"]}" }
+    end
+
+    def table_exists?(table)
+      query = "SELECT 1 FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2"
+      execute(query, table.split(".", 2)).size > 0
+    end
+
+    def close
+      if @conn
+        conn.close
+        @conn = nil
+      end
+    end
+
+    def to_url
+      uri = self.uri.dup
+      uri.query = nil
+      uri.to_s
+    end
+
+    def columns(table)
+      query = "SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2"
+      execute(query, table.split(".", 2)).map { |row| row["column_name"] }
+    end
+
+    def sequences(table, columns)
+      execute("SELECT #{columns.map { |f| "pg_get_serial_sequence(#{escape("#{quote_ident_full(table)}")}, #{escape(f)}) AS #{f}" }.join(", ")}")[0].values.compact
+    end
+
+    def max_id(table, primary_key, sql_clause = nil)
+      execute("SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}")[0]["max"].to_i
+    end
+
+    def min_id(table, primary_key, sql_clause = nil)
+      execute("SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_ident_full(table)}#{sql_clause}")[0]["min"].to_i
+    end
+
+    def last_value(seq)
+      execute("select last_value from #{seq}")[0]["last_value"]
+    end
+
+    def truncate(table)
+      execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
+    end
+
+    # http://stackoverflow.com/a/20537829
+    def primary_key(table)
+      query = <<-SQL
+        SELECT
+          pg_attribute.attname,
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
+        FROM
+          pg_index, pg_class, pg_attribute, pg_namespace
+        WHERE
+          pg_class.oid = $2::regclass AND
+          indrelid = pg_class.oid AND
+          nspname = $1 AND
+          pg_class.relnamespace = pg_namespace.oid AND
+          pg_attribute.attrelid = pg_class.oid AND
+          pg_attribute.attnum = any(pg_index.indkey) AND
+          indisprimary
+      SQL
+      row = execute(query, [table.split(".", 2)[0], quote_ident_full(table)])[0]
+      row && row["attname"]
+    end
+
+    # borrowed from
+    # ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver
+    def conn
+      @conn ||= begin
+        begin
+          uri_parser = URI::Parser.new
+          config = {
+              host: uri.host,
+              port: uri.port,
+              dbname: uri.path.sub(/\A\//, ""),
+              user: uri.user,
+              password: uri.password,
+              connect_timeout: 3
+          }.reject { |_, value| value.to_s.empty? }
+          config.map { |key, value| config[key] = uri_parser.unescape(value) if value.is_a?(String) }
+          conn = PG::Connection.new(config)
+        rescue PG::ConnectionBad => e
+          log
+          raise PgSync::Error, e.message
+        end
+      end
+    end
+
+    def dump_command(tables)
+      tables = tables.keys.map { |t| "-t #{Shellwords.escape(quote_ident_full(t))}" }.join(" ")
+      dump_command = "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables} #{to_url}"
+    end
+
+    def restore_command
+      psql_version = Gem::Version.new(`psql --version`.lines[0].chomp.split(" ")[-1].sub(/beta\d/, ""))
+      if_exists = psql_version >= Gem::Version.new("9.4.0")
+      restore_command = "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{to_url}"
+    end
+
+    def fully_resolve_tables(tables)
+      no_schema_tables = {}
+      search_path_index = Hash[search_path.map.with_index.to_a]
+      self.tables.group_by { |t| t.split(".", 2)[-1] }.each do |group, t2|
+        no_schema_tables[group] = t2.sort_by { |t| [search_path_index[t.split(".", 2)[0]] || 1000000, t] }[0]
+      end
+
+      Hash[tables.map { |k, v| [no_schema_tables[k] || k, v] }]
+    end
+
+    def search_path
+      execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
+    end
+
+    private
+
+    def quote_ident_full(ident)
+      ident.split(".", 2).map { |v| quote_ident(v) }.join(".")
+    end
+
+    def execute(query, params = [])
+      conn.exec_params(query, params).to_a
+    end
+
+    def log(message = nil)
+      $stderr.puts message
+    end
+
+    def quote_ident(value)
+      PG::Connection.quote_ident(value)
+    end
+
+    def escape(value)
+      if value.is_a?(String)
+        "'#{quote_string(value)}'"
+      else
+        value
+      end
+    end
+
+    # activerecord
+    def quote_string(s)
+      s.gsub(/\\/, '\&\&').gsub(/'/, "''")
+    end
+
+    def resolve_url(source)
+      if source && source[0..1] == "$(" && source[-1] == ")"
+        command = source[2..-2]
+        source = `#{command}`.chomp
+        unless $?.success?
+          raise PgSync::Error, "Command exited with non-zero status:\n#{command}"
+        end
+      end
+      source
+    end
+  end
+end

data/lib/pgsync/table_list.rb

@@ -0,0 +1,105 @@
+module PgSync
+  class TableList
+    attr_reader :args, :opts, :source, :config
+
+    def initialize(args, options, source, config)
+      @args = args
+      @opts = options
+      @source = source
+      @config = config
+    end
+
+    def tables
+      tables = nil
+
+      if opts[:groups]
+        tables ||= Hash.new { |hash, key| hash[key] = {} }
+        specified_groups = to_arr(opts[:groups])
+        specified_groups.map do |tag|
+          group, id = tag.split(":", 2)
+          if (t = (config["groups"] || {})[group])
+            add_tables(tables, t, id, args[1])
+          else
+            raise PgSync::Error, "Group not found: #{group}"
+          end
+        end
+      end
+
+      if opts[:tables]
+        tables ||= Hash.new { |hash, key| hash[key] = {} }
+        to_arr(opts[:tables]).each do |tag|
+          table, id = tag.split(":", 2)
+          add_table(tables, table, id, args[1])
+        end
+      end
+
+      if args[0]
+        # could be a group, table, or mix
+        tables ||= Hash.new { |hash, key| hash[key] = {} }
+        specified_groups = to_arr(args[0])
+        specified_groups.map do |tag|
+          group, id = tag.split(":", 2)
+          if (t = (config["groups"] || {})[group])
+            add_tables(tables, t, id, args[1])
+          else
+            add_table(tables, group, id, args[1])
+          end
+        end
+      end
+
+      tables ||= begin
+        exclude = to_arr(opts[:exclude])
+        exclude = source.fully_resolve_tables(exclude).keys if exclude.any?
+
+        tabs = source.tables
+        unless opts[:all_schemas]
+          schemas = Set.new(opts[:schemas] ? to_arr(opts[:schemas]) : [source.schema || "public"])
+          tabs.select! { |t| schemas.include?(t.split(".", 2)[0]) }
+        end
+
+        Hash[(tabs - exclude).map { |k| [k, {}] }]
+      end
+
+      source.fully_resolve_tables(tables)
+    end
+
+    private
+
+    def to_arr(value)
+      if value.is_a?(Array)
+        value
+      else
+        # Split by commas, but don't use commas inside double quotes
+        # http://stackoverflow.com/questions/21105360/regex-find-comma-not-inside-quotes
+        value.to_s.split(/(?!\B"[^"]*),(?![^"]*"\B)/)
+      end
+    end
+
+    def add_tables(tables, t, id, boom)
+      t.each do |table|
+        sql = nil
+        if table.is_a?(Array)
+          table, sql = table
+        end
+        add_table(tables, table, id, boom || sql)
+      end
+    end
+
+    def add_table(tables, table, id, boom, wildcard = false)
+      if table.include?("*") && !wildcard
+        regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
+        t2 = source.tables.select { |t| regex.match(t) }
+        t2.each do |tab|
+          add_table(tables, tab, id, boom, true)
+        end
+      else
+        tables[table] = {}
+        tables[table][:sql] = boom.gsub("{id}", cast(id)).gsub("{1}", cast(id)) if boom
+      end
+    end
+
+    def cast(value)
+      value.to_s.gsub(/\A\"|\"\z/, '')
+    end
+  end
+end