pgsync 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1c5a725040f7f317495776e4a7e8b7d9940c27c
-  data.tar.gz: 02319b35c1c94acbeee08a5ae644d397c63d3fab
+  metadata.gz: 68da22be015e5ff66ecc722c7e2cf19a91b77b3c
+  data.tar.gz: 76498a3252a89668dc9b1e9f65dd152dedc4a727
 SHA512:
-  metadata.gz: f99fd4c76bdbde44ee5998a8dccb54c55edc67d61b3e9764617077f2d368b1bbecc29a80e06d876827312a2e573d35aef71cffc9dcd1c9d7df532656e420cb37
-  data.tar.gz: dc25dd88ab0192228b78b9667b9b5cafc2260fffa0a3f4ee3b4271c8e9aab1c6ac6e160119fd1102f96cf8213b897dfb87f0a8076e056967b5fd5842fceb1944
+  metadata.gz: 71f0e8b384f7a07706e6ad796c407f212cd0eb1dfe04bebffb278e0208e26e755a3352ef1ffd8d7681d868818df30312297d9fd60a5373048524236cdf2d6238
+  data.tar.gz: cdae25c85d546e866cd1a8558ab1c76bd548ed0a8d41ff049644e2cf29661e5f565a803f69a848217f482163aa4c3e49bc1680098f45fcbdb125ae443be50040

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+rvm: 2.4.1
+cache: bundler
+sudo: false
+script: bundle exec rake test
+addons:
+  postgresql: "9.4"
+notifications:
+  email:
+    on_success: never
+    on_failure: change

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.3.7
+
+- Fixed non-lowercase tables and columns
+- Fixed `--truncate` option with `--in-batches`
+
 # 0.3.6
 
 - Fixed `Table does not exist in source` error

data/README.md

@@ -1,6 +1,6 @@
 # pgsync
 
-Sync Postgres data to your local machine. Designed for:
+Sync Postgres data between databases. Designed for:
 
 - **speed** - up to 4x faster than traditional tools on a 4-core machine
 - **security** - built-in methods to prevent sensitive data from ever leaving the server
@@ -8,6 +8,8 @@ Sync Postgres data to your local machine. Designed for:
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
+[![Build Status](https://travis-ci.org/ankane/pgsync.svg?branch=master)](https://travis-ci.org/ankane/pgsync)
+
 ## Installation
 
 pgsync is a command line tool. To install, run:
@@ -109,7 +111,7 @@ groups:
     products: "where id = {1}"
     reviews: "where product_id = {1}"
     coupons: "where product_id = {1} order by created_at desc limit 10"
-    stores: "where id in (select store_id from products where id = {1})
+    stores: "where id in (select store_id from products where id = {1})"
 ```
 
 And run:

data/Rakefile

@@ -1,8 +1,9 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
 
-task :default => :test
+task default: :test
 Rake::TestTask.new do |t|
   t.libs << "test"
   t.pattern = "test/**/*_test.rb"
+  t.warning = false
 end

data/lib/pgsync.rb

@@ -9,6 +9,7 @@ require "multiprocessing"
 require "fileutils"
 require "tempfile"
 require "cgi"
+require "shellwords"
 
 module URI
   class POSTGRESQL < Generic
@@ -97,8 +98,8 @@ module PgSync
 
         if opts[:schema_only]
           log "* Dumping schema"
-          tables = tables.keys.map { |t| "-t #{t}" }.join(" ")
-          psql_version = Gem::Version.new(`psql --version`.split(" ").last)
+          tables = tables.keys.map { |t| "-t #{Shellwords.escape(quote_ident(t))}" }.join(" ")
+          psql_version = Gem::Version.new(`psql --version`.lines[0].chomp.split(" ")[-1].sub(/beta\d/, ""))
           if_exists = psql_version >= Gem::Version.new("9.4.0")
           dump_command = "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables} #{to_url(source_uri)}"
           restore_command = "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{to_url(destination_uri)}"
@@ -172,19 +173,23 @@ module PgSync
               end
 
               if shared_fields.any?
-                copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, from_connection)} AS #{escape_identifier(f)}" : "#{table}.#{escape_identifier(f)}" }.join(", ")
-                fields = shared_fields.map { |f| escape_identifier(f) }.join(", ")
+                copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, from_connection)} AS #{quote_ident(f)}" : "#{quote_ident(table)}.#{quote_ident(f)}" }.join(", ")
+                fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
 
                 seq_values = {}
                 shared_sequences.each do |seq|
                   seq_values[seq] = from_connection.exec("select last_value from #{seq}").to_a[0]["last_value"]
                 end
 
-                copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{sql_clause}) TO STDOUT"
+                copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident(table)}#{sql_clause}) TO STDOUT"
                 if opts[:in_batches]
+                  abort "Cannot use --overwrite with --in-batches" if opts[:overwrite]
+
                   primary_key = self.primary_key(from_connection, table, from_schema)
                   abort "No primary key" unless primary_key
 
+                  to_connection.exec("TRUNCATE #{quote_ident(table)} CASCADE") if opts[:truncate]
+
                   from_max_id = max_id(from_connection, table, primary_key, sql_clause)
                   to_max_id = max_id(to_connection, table, primary_key, sql_clause) + 1
 
@@ -200,14 +205,14 @@ module PgSync
                   batch_count = ((from_max_id - starting_id + 1) / batch_size.to_f).ceil
 
                   while starting_id <= from_max_id
-                    where = "#{primary_key} >= #{starting_id} AND #{primary_key} < #{starting_id + batch_size}"
+                    where = "#{quote_ident(primary_key)} >= #{starting_id} AND #{quote_ident(primary_key)} < #{starting_id + batch_size}"
                     log "    #{i}/#{batch_count}: #{where}"
 
                     # TODO be smarter for advance sql clauses
                     batch_sql_clause = " #{sql_clause.length > 0 ? "#{sql_clause} AND" : "WHERE"} #{where}"
 
-                    batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{batch_sql_clause}) TO STDOUT"
-                    to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
+                    batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident(table)}#{batch_sql_clause}) TO STDOUT"
+                    to_connection.copy_data "COPY #{quote_ident(table)} (#{fields}) FROM STDIN" do
                       from_connection.copy_data batch_copy_to_command do
                         while row = from_connection.get_copy_data
                           to_connection.put_copy_data(row)
@@ -238,10 +243,10 @@ module PgSync
 
                     to_connection.transaction do
                       # create a temp table
-                      to_connection.exec("CREATE TABLE #{temp_table} AS SELECT * FROM #{table} WITH NO DATA")
+                      to_connection.exec("CREATE TABLE #{quote_ident(temp_table)} AS SELECT * FROM #{quote_ident(table)} WITH NO DATA")
 
                       # load file
-                      to_connection.copy_data "COPY #{temp_table} (#{fields}) FROM STDIN" do
+                      to_connection.copy_data "COPY #{quote_ident(temp_table)} (#{fields}) FROM STDIN" do
                         file.each do |row|
                           to_connection.put_copy_data(row)
                         end
@@ -249,22 +254,22 @@ module PgSync
 
                       if opts[:preserve]
                         # insert into
-                        to_connection.exec("INSERT INTO #{table} (SELECT * FROM #{temp_table} WHERE NOT EXISTS (SELECT 1 FROM #{table} WHERE #{table}.#{primary_key} = #{temp_table}.#{primary_key}))")
+                        to_connection.exec("INSERT INTO #{quote_ident(table)} (SELECT * FROM #{quote_ident(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident(table)} WHERE #{quote_ident(table)}.#{primary_key} = #{quote_ident(temp_table)}.#{quote_ident(primary_key)}))")
                       else
-                        to_connection.exec("DELETE FROM #{table} WHERE #{primary_key} IN (SELECT #{primary_key} FROM #{temp_table})")
-                        to_connection.exec("INSERT INTO #{table} (SELECT * FROM #{temp_table})")
+                        to_connection.exec("DELETE FROM #{quote_ident(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident(temp_table)})")
+                        to_connection.exec("INSERT INTO #{quote_ident(table)} (SELECT * FROM #{quote_ident(temp_table)})")
                       end
 
                       # delete temp table
-                      to_connection.exec("DROP TABLE #{temp_table}")
+                      to_connection.exec("DROP TABLE #{quote_ident(temp_table)}")
                     end
                   ensure
                      file.close
                      file.unlink
                   end
                 else
-                  to_connection.exec("TRUNCATE #{table} CASCADE")
-                  to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
+                  to_connection.exec("TRUNCATE #{quote_ident(table)} CASCADE")
+                  to_connection.copy_data "COPY #{quote_ident(table)} (#{fields}) FROM STDIN" do
                     from_connection.copy_data copy_to_command do
                       while row = from_connection.get_copy_data
                         to_connection.put_copy_data(row)
@@ -452,7 +457,7 @@ Options:}
       else
         strategies = {
           "unique_email" => "'email' || #{table}.id || '@example.org'",
-          "untouched" => escape_identifier(column),
+          "untouched" => quote_ident(column),
           "unique_phone" => "(#{table}.id + 1000000000)::text",
           "random_int" => "(RAND() * 10)::int",
           "random_date" => "'1970-01-01'",
@@ -471,7 +476,7 @@ Options:}
       end
     end
 
-    def escape_identifier(value)
+    def quote_ident(value)
       PG::Connection.quote_ident(value)
     end
 
@@ -492,7 +497,9 @@ Options:}
       if value.is_a?(Array)
         value
       else
-        value.to_s.split(",")
+        # Split by commas, but don't use commas inside double quotes
+        # http://stackoverflow.com/questions/21105360/regex-find-comma-not-inside-quotes
+        value.to_s.split(/(?!\B"[^"]*),(?![^"]*"\B)/)
       end
     end
 
@@ -502,8 +509,8 @@ Options:}
       uri.host ||= "localhost"
       uri.port ||= 5432
       uri.path = "/#{uri.path}" if uri.path && uri.path[0] != "/"
-      schema = ((uri.query && CGI::parse(uri.query)["schema"]) || [])[0]
-      [uri, schema || "public"]
+      schema = CGI.parse(uri.query.to_s)["schema"][0] || "public"
+      [uri, schema]
     end
 
     def print_uri(prefix, uri)
@@ -551,7 +558,7 @@ Options:}
     end
 
     def sequences(conn, table, columns)
-      conn.exec("SELECT #{columns.map { |f| "pg_get_serial_sequence(#{escape(table)}, #{escape(f)}) AS #{f}" }.join(", ")}").to_a[0].values.compact
+      conn.exec("SELECT #{columns.map { |f| "pg_get_serial_sequence(#{escape(quote_ident(table))}, #{escape(f)}) AS #{f}" }.join(", ")}").to_a[0].values.compact
     end
 
     def in_parallel(tables, &block)
@@ -582,8 +589,8 @@ Options:}
       if table.include?("*") && !wildcard
         regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
         t2 = with_connection(from_uri) { |conn| self.tables(conn, from_schema) }.select { |t| regex.match(t) }
-        t2.each do |table|
-          add_table(tables, table, id, boom, from_uri, from_schema, true)
+        t2.each do |tab|
+          add_table(tables, tab, id, boom, from_uri, from_schema, true)
         end
       else
         tables[table] = {}
@@ -643,15 +650,15 @@ Options:}
     end
 
     def max_id(conn, table, primary_key, sql_clause = nil)
-      conn.exec("SELECT MAX(#{escape_identifier(primary_key)}) FROM #{escape_identifier(table)}#{sql_clause}").to_a[0]["max"].to_i
+      conn.exec("SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_ident(table)}#{sql_clause}").to_a[0]["max"].to_i
     end
 
     def min_id(conn, table, primary_key, sql_clause = nil)
-      conn.exec("SELECT MIN(#{escape_identifier(primary_key)}) FROM #{escape_identifier(table)}#{sql_clause}").to_a[0]["min"].to_i
+      conn.exec("SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_ident(table)}#{sql_clause}").to_a[0]["min"].to_i
     end
 
     def cast(value)
-      value.to_s
+      value.to_s.gsub(/\A\"|\"\z/, '')
     end
 
     def deprecated(message)

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.3.6"
+  VERSION = "0.3.7"
 end

data/pgsync.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Andrew Kane"]
   spec.email         = ["andrew@chartkick.com"]
 
-  spec.summary       = "Quickly and securely sync data between environments"
+  spec.summary       = "Sync Postgres data between databases"
   spec.homepage      = "https://github.com/ankane/pgsync"
   spec.license       = "MIT"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-03 00:00:00.000000000 Z
+date: 2017-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -117,6 +117,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -147,8 +148,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: Quickly and securely sync data between environments
+summary: Sync Postgres data between databases
 test_files: []