pgsync 0.3.9 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8d5362420e8f65878dcb39bfcb945db069d585f
-  data.tar.gz: a0d4d2b74ca25148f6ddcb2b4bdfa0bea7ef2c99
+  metadata.gz: 4000209f5f79dad8a7c6f427be2cdcf25abe219e
+  data.tar.gz: 4980413b816f0d0aa150aedf7bbcb1141fb7caad
 SHA512:
-  metadata.gz: ed08a53660ea4973932176da8ea2bb1354f457c67dc9dd9cff832a67eb2b785d399f84a1840e21485579c0d595e81657303213d8831ae8ae72cba6cede713dd2
-  data.tar.gz: 4f56fdbec5b5b93b4ff6ffe781edba94076107203e5f620a5708c3487d8a1cd77e8388a06cf0c9ddbd403afefa06832dd648382dfb3dadde9fc857b83ddea089
+  metadata.gz: 3122cf01ba1dcda75172bc823507c5db684561797dbdb88be42a93866d3fbfb0147e068ab501d1c7a74f09fd88341f623f61e4a2751183a051ba5cefecbdcbb4
+  data.tar.gz: cb8531e15f2ba2c714d1e022b286efe09208abd4b88c997139a23d6ff583e45c460bae46961735716978dd34b7448f42dc2b8f1cbdb4b06592b0d604e070a219

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.4.0
+
+- Sync all schemas in search path by default
+- Added support for socket connections
+- Added support for environment variables
+
 # 0.3.9
 
 - Better support for schemas

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 Andrew Kane
+Copyright (c) 2015-2018 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/exe/pgsync

@@ -5,6 +5,6 @@ begin
   PgSync::Client.new(ARGV).perform
 rescue PgSync::Error => e
   abort e.message
-rescue Interrupt => e
+rescue Interrupt
   # do nothing
 end

data/lib/pgsync.rb

@@ -1,16 +1,16 @@
-require "yaml"
-require "slop"
-require "uri"
+require "cgi"
 require "erb"
+require "fileutils"
+require "multiprocessing"
 require "pg"
 require "parallel"
-require "multiprocessing"
-require "fileutils"
-require "tempfile"
-require "cgi"
-require "shellwords"
 require "set"
+require "shellwords"
+require "slop"
+require "tempfile"
 require "thread" # windows only
+require "uri"
+require "yaml"
 
 require "pgsync/client"
 require "pgsync/data_source"
@@ -18,13 +18,6 @@ require "pgsync/table_list"
 require "pgsync/table_sync"
 require "pgsync/version"
 
-module URI
-  class POSTGRESQL < Generic
-    DEFAULT_PORT = 5432
-  end
-  @@schemes["POSTGRESQL"] = @@schemes["POSTGRES"] = POSTGRESQL
-end
-
 module PgSync
   class Error < StandardError; end
 end

data/lib/pgsync/client.rb

@@ -2,6 +2,7 @@ module PgSync
   class Client
     def initialize(args)
       $stdout.sync = true
+      $stderr.sync = true
       @exit = false
       @arguments, @options = parse_args(args)
       @mutex = windows? ? Mutex.new : MultiProcessing::Mutex.new
@@ -12,7 +13,7 @@ module PgSync
       return if @exit
 
       args, opts = @arguments, @options
-      [:to, :from, :to_safe, :exclude].each do |opt|
+      [:to, :from, :to_safe, :exclude, :schemas].each do |opt|
         opts[opt] ||= config[opt.to_s]
       end
       map_deprecations(args, opts)
@@ -41,12 +42,21 @@ module PgSync
       destination = DataSource.new(opts[:to])
       raise PgSync::Error, "No destination" unless destination.exists?
 
-      unless opts[:to_safe] || destination.local?
-        raise PgSync::Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
-      end
+      begin
+        # start connections
+        source.host
+        destination.host
+
+        unless opts[:to_safe] || destination.local?
+          raise PgSync::Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
+        end
 
-      print_description("From", source)
-      print_description("To", destination)
+        print_description("From", source)
+        print_description("To", destination)
+      ensure
+        source.close
+        destination.close
+      end
 
       tables = nil
       begin
@@ -82,7 +92,7 @@ module PgSync
           confirm_tables_exist(destination, tables, "destination")
 
           in_parallel(tables) do |table, table_opts|
-            TableSync.new.sync_with_benchmark(@mutex, config, table, opts.merge(table_opts), source.url, destination.url)
+            TableSync.new.sync(@mutex, config, table, opts.merge(table_opts), source.url, destination.url, source.search_path.find { |sp| sp != "pg_catalog" })
           end
         end
 
@@ -90,14 +100,14 @@ module PgSync
       end
     end
 
-    def confirm_tables_exist(destination, tables, description)
+    def confirm_tables_exist(data_source, tables, description)
       tables.keys.each do |table|
-        unless destination.table_exists?(table)
+        unless data_source.table_exists?(table)
           raise PgSync::Error, "Table does not exist in #{description}: #{table}"
         end
       end
     ensure
-      destination.close
+      data_source.close
     end
 
     def map_deprecations(args, opts)
@@ -215,7 +225,8 @@ Options:}
     end
 
     def print_description(prefix, source)
-      log "#{prefix}: #{source.uri.path.sub(/\A\//, '')} on #{source.uri.host}:#{source.uri.port}"
+      location = " on #{source.host}:#{source.port}" if source.host
+      log "#{prefix}: #{source.dbname}#{location}"
     end
 
     def search_tree(file)

data/lib/pgsync/data_source.rb

@@ -2,8 +2,9 @@ module PgSync
   class DataSource
     attr_reader :url
 
-    def initialize(source)
+    def initialize(source, timeout: 3)
       @url = resolve_url(source)
+      @timeout = timeout
     end
 
     def exists?
@@ -11,45 +12,31 @@ module PgSync
     end
 
     def local?
-      %w(localhost 127.0.0.1).include?(uri.host)
+      !host || %w(localhost 127.0.0.1).include?(host)
     end
 
-    def uri
-      @uri ||= begin
-        uri = URI.parse(@url)
-        uri.scheme ||= "postgres"
-        uri.host ||= "localhost"
-        uri.port ||= 5432
-        uri.path = "/#{uri.path}" if uri.path && uri.path[0] != "/"
-        uri
-      end
-    end
-
-    def schema
-      @schema ||= CGI.parse(uri.query.to_s)["schema"][0]
+    def host
+      @host ||= conninfo[:host]
     end
 
-    def tables
-      query = "SELECT schemaname, tablename FROM pg_catalog.pg_tables WHERE schemaname NOT IN ('information_schema', 'pg_catalog') ORDER BY 1, 2"
-      execute(query).map { |row| "#{row["schemaname"]}.#{row["tablename"]}" }
+    def port
+      @port ||= conninfo[:port]
     end
 
-    def table_exists?(table)
-      query = "SELECT 1 FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2"
-      execute(query, table.split(".", 2)).size > 0
+    def dbname
+      @dbname ||= conninfo[:dbname]
     end
 
-    def close
-      if @conn
-        conn.close
-        @conn = nil
+    # gets visible tables
+    def tables
+      @tables ||= begin
+        query = "SELECT table_schema, table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema NOT IN ('information_schema', 'pg_catalog') ORDER BY 1, 2"
+        execute(query).map { |row| "#{row["table_schema"]}.#{row["table_name"]}" }
       end
     end
 
-    def to_url
-      uri = self.uri.dup
-      uri.query = nil
-      uri.to_s
+    def table_exists?(table)
+      table_set.include?(table)
     end
 
     def columns(table)
@@ -98,38 +85,40 @@ module PgSync
       row && row["attname"]
     end
 
-    # borrowed from
-    # ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver
     def conn
       @conn ||= begin
         begin
-          uri_parser = URI::Parser.new
-          config = {
-              host: uri.host,
-              port: uri.port,
-              dbname: uri.path.sub(/\A\//, ""),
-              user: uri.user,
-              password: uri.password,
-              connect_timeout: 3
-          }.reject { |_, value| value.to_s.empty? }
-          config.map { |key, value| config[key] = uri_parser.unescape(value) if value.is_a?(String) }
-          conn = PG::Connection.new(config)
+          ENV["PGCONNECT_TIMEOUT"] ||= @timeout.to_s
+          if @url =~ /\Apostgres(ql)?:\/\//
+            config = @url
+          else
+            config = {dbname: @url}
+          end
+          PG::Connection.new(config)
         rescue PG::ConnectionBad => e
-          log
           raise PgSync::Error, e.message
+        rescue URI::InvalidURIError
+          raise PgSync::Error, "Invalid connection string"
         end
       end
     end
 
+    def close
+      if @conn
+        @conn.close
+        @conn = nil
+      end
+    end
+
     def dump_command(tables)
       tables = tables.keys.map { |t| "-t #{Shellwords.escape(quote_ident_full(t))}" }.join(" ")
-      dump_command = "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables} #{to_url}"
+      "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables} -d #{@url}"
     end
 
     def restore_command
       psql_version = Gem::Version.new(`psql --version`.lines[0].chomp.split(" ")[-1].sub(/beta\d/, ""))
       if_exists = psql_version >= Gem::Version.new("9.4.0")
-      restore_command = "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{to_url}"
+      "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@url}"
     end
 
     def fully_resolve_tables(tables)
@@ -143,11 +132,19 @@ module PgSync
     end
 
     def search_path
-      execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
+      @search_path ||= execute("SELECT current_schemas(true)")[0]["current_schemas"][1..-2].split(",")
     end
 
     private
 
+    def table_set
+      @table_set ||= Set.new(tables)
+    end
+
+    def conninfo
+      @conninfo ||= conn.conninfo_hash
+    end
+
     def quote_ident_full(ident)
       ident.split(".", 2).map { |v| quote_ident(v) }.join(".")
     end
@@ -156,10 +153,6 @@ module PgSync
       conn.exec_params(query, params).to_a
     end
 
-    def log(message = nil)
-      $stderr.puts message
-    end
-
     def quote_ident(value)
       PG::Connection.quote_ident(value)
     end

data/lib/pgsync/table_list.rb

@@ -53,7 +53,7 @@ module PgSync
 
         tabs = source.tables
         unless opts[:all_schemas]
-          schemas = Set.new(opts[:schemas] ? to_arr(opts[:schemas]) : [source.schema || "public"])
+          schemas = Set.new(opts[:schemas] ? to_arr(opts[:schemas]) : source.search_path)
           tabs.select! { |t| schemas.include?(t.split(".", 2)[0]) }
         end
 

data/lib/pgsync/table_sync.rb

@@ -1,24 +1,14 @@
 module PgSync
   class TableSync
-    def sync_with_benchmark(mutex, config, table, opts, source_url, destination_url)
-      time =
-        benchmark do
-          sync(mutex, config, table, opts, source_url, destination_url)
-        end
-
-      mutex.synchronize do
-        log "* DONE #{table} (#{time.round(1)}s)"
-      end
-    end
-
-    def sync(mutex, config, table, opts, source_url, destination_url)
-      source = DataSource.new(source_url)
-      destination = DataSource.new(destination_url)
-
-      from_connection = source.conn
-      to_connection = destination.conn
+    def sync(mutex, config, table, opts, source_url, destination_url, first_schema)
+      start_time = Time.now
+      source = DataSource.new(source_url, timeout: 0)
+      destination = DataSource.new(destination_url, timeout: 0)
 
       begin
+        from_connection = source.conn
+        to_connection = destination.conn
+
         bad_fields = opts[:no_rules] ? [] : config["data_rules"]
 
         from_fields = source.columns(table)
@@ -35,8 +25,10 @@ module PgSync
 
         sql_clause = String.new
 
+        table_name = table.sub("#{first_schema}.", "")
+
         mutex.synchronize do
-          log "* Syncing #{table}"
+          log "* Syncing #{table_name}"
           if opts[:sql]
             log "    #{opts[:sql]}"
             sql_clause << " #{opts[:sql]}"
@@ -93,7 +85,7 @@ module PgSync
               batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{batch_sql_clause}) TO STDOUT"
               to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
                 from_connection.copy_data batch_copy_to_command do
-                  while row = from_connection.get_copy_data
+                  while (row = from_connection.get_copy_data)
                     to_connection.put_copy_data(row)
                   end
                 end
@@ -157,6 +149,9 @@ module PgSync
             to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
           end
         end
+        mutex.synchronize do
+          log "* DONE #{table_name} (#{(Time.now - start_time).round(1)}s)"
+        end
       ensure
         source.close
         destination.close
@@ -205,12 +200,6 @@ module PgSync
       end
     end
 
-    def benchmark
-      start_time = Time.now
-      yield
-      Time.now - start_time
-    end
-
     def log(message = nil)
       $stderr.puts message
     end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.3.9"
+  VERSION = "0.4.0"
 end

data/pgsync.gemspec

@@ -18,12 +18,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "slop", ">= 4.2.0"
-  spec.add_dependency "pg"
-  spec.add_dependency "parallel"
   spec.add_dependency "multiprocessing"
+  spec.add_dependency "parallel"
+  spec.add_dependency "pg"
+  spec.add_dependency "slop", ">= 4.2.0"
 
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "rake"
 end

metadata

@@ -1,31 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.4.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-27 00:00:00.000000000 Z
+date: 2018-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: multiprocessing
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,19 +53,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: multiprocessing
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="