pgsync 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4000209f5f79dad8a7c6f427be2cdcf25abe219e
-  data.tar.gz: 4980413b816f0d0aa150aedf7bbcb1141fb7caad
+SHA256:
+  metadata.gz: 6dd029f7881055cb38f098ec23ecab2e70e06f118c7d50cfd09d43d5c8c6ad28
+  data.tar.gz: 24eff3b8de6a4a59b55dd749dcc3134785bfb2e1361aaec2fec1edc30a579875
 SHA512:
-  metadata.gz: 3122cf01ba1dcda75172bc823507c5db684561797dbdb88be42a93866d3fbfb0147e068ab501d1c7a74f09fd88341f623f61e4a2751183a051ba5cefecbdcbb4
-  data.tar.gz: cb8531e15f2ba2c714d1e022b286efe09208abd4b88c997139a23d6ff583e45c460bae46961735716978dd34b7448f42dc2b8f1cbdb4b06592b0d604e070a219
+  metadata.gz: e6eca6142c30b5bcf43608eaf338aea4c6a71b37781b979e34f9028e1418f50c3188a02998e6f4b74111ec58c85eff6b04ad005c8dfa9a1085f653b10e9b37db
+  data.tar.gz: d39a13d5a83b168f0ccb90b04309ca1f616dab646963a6a6b873da06fdbc883eeef7e758890aa486f4df06c211417ac00afd60ff843ee17e17421eb65bb6abff

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.4.1
+
+- Made `psql` version check more robust
+- Fixed issue with non-lowercase primary key
+- Prefer `--init` over `--setup`
+- Improved data rules
+
 # 0.4.0
 
 - Sync all schemas in search path by default

data/README.md

@@ -18,12 +18,12 @@ pgsync is a command line tool. To install, run:
 gem install pgsync
 ```
 
-This will give you the `pgsync` command.
+This will give you the `pgsync` command. If installation fails, you may need to install [dependencies](#dependencies).
 
 In your project directory, run:
 
 ```sh
-pgsync --setup
+pgsync --init
 ```
 
 This creates `.pgsync.yml` for you to customize. We recommend checking this into your version control (assuming it doesn’t contain sensitive information). `pgsync` commands can be run from this directory or any subdirectory.
@@ -142,9 +142,15 @@ Or just the schema
 pgsync --schema-only
 ```
 
+pgsync does not try to sync Postgres extensions.
+
+## Data Protection
+
+Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to your database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
+
 ## Sensitive Information
 
-Prevent sensitive information - like passwords and email addresses - from leaving the remote server.
+Prevent sensitive information like email addresses from leaving the remote server.
 
 Define rules in `.pgsync.yml`:
 
@@ -164,27 +170,27 @@ data_rules:
 
 Options for replacement are:
 
-- null
-- value
-- statement
-- unique_email
-- unique_phone
-- unique_secret
-- random_letter
-- random_int
-- random_date
-- random_time
-- random_ip
-- random_string
-- random_number
-- untouched
+- `unique_email`
+- `unique_phone`
+- `unique_secret`
+- `random_letter`
+- `random_int`
+- `random_date`
+- `random_time`
+- `random_ip`
+- `value`
+- `statement`
+- `null`
+- `untouched`
+
+Rules starting with `unique_` require the table to have a primary key.
 
 ## Multiple Databases
 
 To use with multiple databases, run:
 
 ```sh
-pgsync --setup db2
+pgsync --init db2
 ```
 
 This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands with:
@@ -231,7 +237,7 @@ Version
 pgsync --version
 ```
 
-## Setup Scripts
+## Scripts
 
 Use groups when possible to take advantage of parallelism.
 
@@ -243,6 +249,22 @@ Bundler.with_clean_env do
 end
 ```
 
+## Dependencies
+
+If installation fails, your system may be missing Ruby or libpq.
+
+On Mac, run:
+
+```sh
+brew install postgresql
+```
+
+On Ubuntu, run:
+
+```sh
+sudo apt-get install ruby-dev libpq-dev build-essential
+```
+
 ## Upgrading
 
 Run:
@@ -270,3 +292,16 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Fix bugs and [submit pull requests](https://github.com/ankane/pgsync/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
+
+To run tests, do:
+
+```sh
+git clone https://github.com/ankane/pgsync.git
+cd pgsync
+bundle install
+
+createdb pgsync_test1
+createdb pgsync_test2
+
+bundle exec rake
+```

data/config.yml

@@ -3,7 +3,7 @@
 #   postgres://user:password@host:port/dbname
 # we recommend a command which outputs a database URL
 # so sensitive information is not included in this file
-from: $(heroku config:get DATABASE_URL)
+from: $(some_command)
 
 # destination database URL
 to: postgres://localhost:5432/myapp_development

data/exe/pgsync

@@ -1,10 +1,10 @@
 #!/usr/bin/env ruby
 
-require "pgsync"
 begin
+  require "pgsync"
   PgSync::Client.new(ARGV).perform
 rescue PgSync::Error => e
   abort e.message
 rescue Interrupt
-  # do nothing
+  abort
 end

data/lib/pgsync/client.rb

@@ -18,7 +18,7 @@ module PgSync
       end
       map_deprecations(args, opts)
 
-      if opts[:setup]
+      if opts[:init]
         setup(db_config_file(args[0]) || config_file || ".pgsync.yml")
       else
         sync(args, opts)
@@ -116,8 +116,8 @@ module PgSync
       case command
       when "setup"
         args.shift
-        opts[:setup] = true
-        deprecated "Use `psync --setup` instead"
+        opts[:init] = true
+        deprecated "Use `psync --init` instead"
       when "schema"
         args.shift
         opts[:schema_only] = true
@@ -178,7 +178,8 @@ Options:}
         o.boolean "--schema-only", "schema only", default: false
         o.boolean "--all-schemas", "all schemas", default: false
         o.boolean "--no-rules", "do not apply data rules", default: false
-        o.boolean "--setup", "setup", default: false
+        o.boolean "--init", "init", default: false
+        o.boolean "--setup", "setup", default: false, help: false
         o.boolean "--in-batches", "in batches", default: false, help: false
         o.integer "--batch-size", "batch size", default: 10000, help: false
         o.float "--sleep", "sleep", default: 0, help: false
@@ -191,7 +192,11 @@ Options:}
           @exit = true
         end
       end
-      [opts.arguments, opts.to_hash]
+
+      opts_hash = opts.to_hash
+      opts_hash[:init] = opts_hash[:setup] if opts_hash[:setup]
+
+      [opts.arguments, opts_hash]
     rescue Slop::Error => e
       raise PgSync::Error, e.message
     end

data/lib/pgsync/data_source.rb

@@ -116,8 +116,8 @@ module PgSync
     end
 
     def restore_command
-      psql_version = Gem::Version.new(`psql --version`.lines[0].chomp.split(" ")[-1].sub(/beta\d/, ""))
-      if_exists = psql_version >= Gem::Version.new("9.4.0")
+      psql_version = `psql --version`.lines[0].chomp.split(" ")[-1].split(/[^\d.]/)[0]
+      if_exists = Gem::Version.new(psql_version) >= Gem::Version.new("9.4.0")
       "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@url}"
     end
 

data/lib/pgsync/table_list.rb

@@ -29,6 +29,7 @@ module PgSync
         tables ||= Hash.new { |hash, key| hash[key] = {} }
         to_arr(opts[:tables]).each do |tag|
           table, id = tag.split(":", 2)
+          raise PgSync::Error, "Cannot use parameters with tables" if id
           add_table(tables, table, id, args[1])
         end
       end
@@ -42,6 +43,7 @@ module PgSync
           if (t = (config["groups"] || {})[group])
             add_tables(tables, t, id, args[1])
           else
+            raise PgSync::Error, "Cannot use parameters with tables" if id
             add_table(tables, group, id, args[1])
           end
         end
@@ -70,7 +72,7 @@ module PgSync
         value
       else
         # Split by commas, but don't use commas inside double quotes
-        # http://stackoverflow.com/questions/21105360/regex-find-comma-not-inside-quotes
+        # https://stackoverflow.com/questions/21105360/regex-find-comma-not-inside-quotes
         value.to_s.split(/(?!\B"[^"]*),(?![^"]*"\B)/)
       end
     end

data/lib/pgsync/table_sync.rb

@@ -44,7 +44,8 @@ module PgSync
         end
 
         if shared_fields.any?
-          copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f)} AS #{quote_ident(f)}" : "#{quote_ident_full(table)}.#{quote_ident(f)}" }.join(", ")
+          primary_key = destination.primary_key(table)
+          copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, _| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, primary_key)} AS #{quote_ident(f)}" : "#{quote_ident_full(table)}.#{quote_ident(f)}" }.join(", ")
           fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
 
           seq_values = {}
@@ -55,8 +56,6 @@ module PgSync
           copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{sql_clause}) TO STDOUT"
           if opts[:in_batches]
             raise PgSync::Error, "Cannot use --overwrite with --in-batches" if opts[:overwrite]
-
-            primary_key = source.primary_key(table)
             raise PgSync::Error, "No primary key" unless primary_key
 
             destination.truncate(table) if opts[:truncate]
@@ -99,14 +98,13 @@ module PgSync
               end
             end
           elsif !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
-            primary_key = destination.primary_key(table)
             raise PgSync::Error, "No primary key" unless primary_key
 
             temp_table = "pgsync_#{rand(1_000_000_000)}"
             file = Tempfile.new(temp_table)
             begin
               from_connection.copy_data copy_to_command do
-                while row = from_connection.get_copy_data
+                while (row = from_connection.get_copy_data)
                   file.write(row)
                 end
               end
@@ -124,7 +122,7 @@ module PgSync
 
               if opts[:preserve]
                 # insert into
-                to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident_full(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident_full(table)} WHERE #{quote_ident_full(table)}.#{primary_key} = #{quote_ident_full(temp_table)}.#{quote_ident(primary_key)}))")
+                to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident_full(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident_full(table)} WHERE #{quote_ident_full(table)}.#{quote_ident(primary_key)} = #{quote_ident_full(temp_table)}.#{quote_ident(primary_key)}))")
               else
                 to_connection.transaction do
                   to_connection.exec("DELETE FROM #{quote_ident_full(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident_full(temp_table)})")
@@ -139,7 +137,7 @@ module PgSync
             destination.truncate(table)
             to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
               from_connection.copy_data copy_to_command do
-                while row = from_connection.get_copy_data
+                while (row = from_connection.get_copy_data)
                   to_connection.put_copy_data(row)
                 end
               end
@@ -167,7 +165,7 @@ module PgSync
     end
 
     # TODO wildcard rules
-    def apply_strategy(rule, table, column)
+    def apply_strategy(rule, table, column, primary_key)
       if rule.is_a?(Hash)
         if rule.key?("value")
           escape(rule["value"])
@@ -177,29 +175,40 @@ module PgSync
           raise PgSync::Error, "Unknown rule #{rule.inspect} for column #{column}"
         end
       else
-        strategies = {
-          "unique_email" => "'email' || #{table}.id || '@example.org'",
-          "untouched" => quote_ident(column),
-          "unique_phone" => "(#{table}.id + 1000000000)::text",
-          "random_int" => "(RAND() * 10)::int",
-          "random_date" => "'1970-01-01'",
-          "random_time" => "NOW()",
-          "unique_secret" => "'secret' || #{table}.id",
-          "random_ip" => "'127.0.0.1'",
-          "random_letter" => "'A'",
-          "random_string" => "right(md5(random()::text),10)",
-          "random_number" => "(RANDOM() * 1000000)::int",
-          "null" => "NULL",
-          nil => "NULL"
-        }
-        if strategies[rule]
-          strategies[rule]
+        case rule
+        when "untouched"
+          quote_ident(column)
+        when "unique_email"
+          "'email' || #{quoted_primary_key(table, primary_key, rule)}::text || '@example.org'"
+        when "unique_phone"
+          "(#{quoted_primary_key(table, primary_key, rule)}::bigint + 1000000000)::text"
+        when "unique_secret"
+          "'secret' || #{quoted_primary_key(table, primary_key, rule)}::text"
+        when "random_int", "random_number"
+          "(RANDOM() * 100)::int"
+        when "random_date"
+          "date '1970-01-01' + (RANDOM() * 10000)::int"
+        when "random_time"
+          "NOW() - (RANDOM() * 100000000)::int * INTERVAL '1 second'"
+        when "random_ip"
+          "(1 + RANDOM() * 254)::int::text || '.0.0.1'"
+        when "random_letter"
+          "chr(65 + (RANDOM() * 26)::int)"
+        when "random_string"
+          "RIGHT(MD5(RANDOM()::text), 10)"
+        when "null", nil
+          "NULL"
         else
           raise PgSync::Error, "Unknown rule #{rule} for column #{column}"
         end
       end
     end
 
+    def quoted_primary_key(table, primary_key, rule)
+      raise "Primary key required for this data rule: #{rule}" unless primary_key
+      "#{quote_ident_full(table)}.#{quote_ident(primary_key)}"
+    end
+
     def log(message = nil)
       $stderr.puts message
     end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-01 00:00:00.000000000 Z
+date: 2018-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multiprocessing
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.2
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -109,20 +109,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: 
-email:
-- andrew@chartkick.com
+email: andrew@chartkick.com
 executables:
 - pgsync
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
-- Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
 - config.yml
 - exe/pgsync
 - lib/pgsync.rb
@@ -131,7 +126,6 @@ files:
 - lib/pgsync/table_list.rb
 - lib/pgsync/table_sync.rb
 - lib/pgsync/version.rb
-- pgsync.gemspec
 homepage: https://github.com/ankane/pgsync
 licenses:
 - MIT
@@ -144,7 +138,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -152,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Sync Postgres data between databases

data/.gitignore

@@ -1,10 +0,0 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-/.pgsync.yml

data/.travis.yml

@@ -1,11 +0,0 @@
-language: ruby
-rvm: 2.4.1
-cache: bundler
-sudo: false
-script: bundle exec rake test
-addons:
-  postgresql: "9.4"
-notifications:
-  email:
-    on_success: never
-    on_failure: change

data/Gemfile

@@ -1,4 +0,0 @@
-source "https://rubygems.org"
-
-# Specify your gem's dependencies in pgsync.gemspec
-gemspec

data/Rakefile

@@ -1,9 +0,0 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
-
-task default: :test
-Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.pattern = "test/**/*_test.rb"
-  t.warning = false
-end

data/pgsync.gemspec

@@ -1,29 +0,0 @@
-# coding: utf-8
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "pgsync/version"
-
-Gem::Specification.new do |spec|
-  spec.name          = "pgsync"
-  spec.version       = PgSync::VERSION
-  spec.authors       = ["Andrew Kane"]
-  spec.email         = ["andrew@chartkick.com"]
-
-  spec.summary       = "Sync Postgres data between databases"
-  spec.homepage      = "https://github.com/ankane/pgsync"
-  spec.license       = "MIT"
-
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_dependency "multiprocessing"
-  spec.add_dependency "parallel"
-  spec.add_dependency "pg"
-  spec.add_dependency "slop", ">= 4.2.0"
-
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "minitest"
-  spec.add_development_dependency "rake"
-end