pgsync 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d12ffec2fc5681c48eb5f13eafca045330d3615f
-  data.tar.gz: f8ed258ee86c6cc099299b1e962856500431d6ac
+  metadata.gz: 12edffcbf1375d06f4a56916e2581053711b3a81
+  data.tar.gz: c1b08b53a4f0dd07d32d87d921400f3718aaa07e
 SHA512:
-  metadata.gz: f19de0157cf9e4ccadea5e787c6413995cfbcb9bd086750512c3392be495df3b7bc1868bd55fdc27d0e65c7a81adefbda6c9118dd9cf4f17b9102b2cc0628994
-  data.tar.gz: 1248be9efc810bf5eb14685696e0b5214740549af940aa67cb9f6484651b72bed4ed8ca82d85a4ea26b840f69208606064c6eb9935f027ce64079c02b95f2baa
+  metadata.gz: e6484259d1dd62dde1b95b8472ae487298dae7848a0e14a95f68efc75bc954a370eb8b2ed74121c26dcc98c46c365a00a823386c88abe27e3ca02e81bbfa363a
+  data.tar.gz: e510ca2717985f8d2f1fd0fc2e16219f81a63f6a55171911508e4def6bc22769267c99d393df02dea15713fe80e030dab67690808517179f5a717953fda99308

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.2
+
+- Prefer `{1}` for interpolation
+- Added `--overwrite` option
+- Deprecated `--where` and `--limit`
+
 # 0.3.1
 
 - Added `-t` or `--tables`, `-g` or `--groups` options

data/README.md

@@ -1,6 +1,10 @@
 # pgsync
 
-Quickly and securely sync data between environments
+Sync Postgres data to your local machine. Designed for:
+
+- **speed** - up to 4x faster than traditional tools on a 4-core machine
+- **security** - built-in methods to prevent sensitive data from ever leaving the server
+- **convenience** - sync partial tables, groups of tables, and related records
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
@@ -26,16 +30,30 @@ Sync all tables
 pgsync
 ```
 
+**Note:** pgsync assumes your schema is already set up on your local machine. See the [schema section](#schema) if that’s not the case.
+
 Sync specific tables
 
 ```sh
 pgsync table1,table2
 ```
 
-Sync specific rows
+Sync specific rows (existing rows are overwritten)
+
+```sh
+pgsync products "where store_id = 1"
+```
+
+You can also preserve existing rows
 
 ```sh
-pgsync products "WHERE id < 1000"
+pgsync products "where store_id = 1" --preserve
+```
+
+Or truncate them
+
+```sh
+pgsync products "where store_id = 1" --truncate
 ```
 
 ### Exclude Tables
@@ -78,19 +96,21 @@ pgsync group1
 
 You can also use groups to sync a specific record and associated records in other tables.
 
-To get user `123` and his or her orders, use:
+To get product `123` with its reviews, last 10 coupons, and store, use:
 
 ```yml
 groups:
-  user:
-    users: "WHERE id = {id}"
-    orders: "WHERE user_id = {id}"
+  product:
+    products: "where id = {1}"
+    reviews: "where product_id = {1}"
+    coupons: "where product_id = {1} order by created_at desc limit 10"
+    stores: "where id in (select store_id from products where id = {1})
 ```
 
 And run:
 
 ```sh
-pgsync user:123
+pgsync product:123
 ```
 
 ### Schema
@@ -161,6 +181,18 @@ To keep you from accidentally overwriting production, the destination is limited
 
 To use another host, add `to_safe: true` to your `.pgsync.yml`.
 
+## Setup Scripts
+
+Use groups when possible to take advantage of parallelism.
+
+For Ruby scripts, you may need to do:
+
+```rb
+Bundler.with_clean_env do
+  system "pgsync ..."
+end
+```
+
 ## Upgrading
 
 Run:

data/lib/pgsync.rb

@@ -18,7 +18,6 @@ end
 
 module PgSync
   class Error < StandardError; end
-  class Rollback < StandardError; end
 
   class Client
     def initialize(args)
@@ -45,6 +44,10 @@ module PgSync
         args.shift
         opts[:setup] = true
         deprecated "Use `psync --setup` instead"
+      when "schema"
+        args.shift
+        opts[:schema_only] = true
+        deprecated "Use `psync --schema-only` instead"
       when "tables"
         args.shift
         opts[:tables] = args.shift
@@ -55,6 +58,18 @@ module PgSync
         deprecated "Use `pgsync #{opts[:groups]}` instead"
       end
 
+      if opts[:where]
+        opts[:sql] ||= String.new
+        opts[:sql] << " WHERE #{opts[:where]}"
+        deprecated "Use `\"WHERE #{opts[:where]}\"` instead"
+      end
+
+      if opts[:limit]
+        opts[:sql] ||= String.new
+        opts[:sql] << " LIMIT #{opts[:limit]}"
+        deprecated "Use `\"LIMIT #{opts[:limit]}\"` instead"
+      end
+
       if opts[:setup]
         setup(db_config_file(args[0]) || config_file || ".pgsync.yml")
       else
@@ -75,17 +90,14 @@ module PgSync
 
         tables = table_list(args, opts, from_uri)
 
-        if args[0] == "schema" || opts[:schema_only]
-          time =
-            benchmark do
-              log "* Dumping schema"
-              tables = tables.keys.map { |t| "-t #{t}" }.join(" ")
-              dump_command = "pg_dump --verbose --schema-only --no-owner --no-acl --clean #{tables} #{to_url(source_uri)}"
-              restore_command = "psql -q -d #{to_url(destination_uri)}"
-              system("#{dump_command} | #{restore_command}")
-            end
+        if opts[:schema_only]
+          log "* Dumping schema"
+          tables = tables.keys.map { |t| "-t #{t}" }.join(" ")
+          dump_command = "pg_dump --verbose --schema-only --no-owner --no-acl --clean #{tables} #{to_url(source_uri)}"
+          restore_command = "psql -q -d #{to_url(destination_uri)}"
+          system("#{dump_command} | #{restore_command}")
 
-          log "* DONE (#{time.round(1)}s)"
+          log_completed(start_time)
         else
           with_connection(to_uri, timeout: 3) do |conn|
             tables.keys.each do |table|
@@ -106,8 +118,7 @@ module PgSync
               sync_table(table, opts.merge(table_opts), from_uri, to_uri)
             end
 
-            time = Time.now - start_time
-            log "Completed in #{time.round(1)}s"
+            log_completed(start_time)
           end
         end
       end
@@ -135,8 +146,6 @@ module PgSync
               extra_sequences = to_sequences - from_sequences
               missing_sequences = from_sequences - to_sequences
 
-              where = opts[:where]
-              limit = opts[:limit]
               sql_clause = String.new
 
               @mutex.synchronize do
@@ -145,14 +154,6 @@ module PgSync
                   log "    #{opts[:sql]}"
                   sql_clause << " #{opts[:sql]}"
                 end
-                if where
-                  log "    #{where}"
-                  sql_clause << " WHERE #{opts[:where]}"
-                end
-                if limit
-                  log "    LIMIT #{limit}"
-                  sql_clause << " LIMIT #{limit}"
-                end
                 log "    Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
                 log "    Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
                 log "    Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
@@ -173,7 +174,7 @@ module PgSync
                 end
 
                 copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{sql_clause}) TO STDOUT"
-                if !opts[:truncate] && (opts[:preserve] || !sql_clause.empty?)
+                if !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
                   primary_key = self.primary_key(from_connection, table, "public")
                   abort "No primary key" unless primary_key
 
@@ -246,8 +247,8 @@ Options:}
         o.string "-g", "--groups", "groups"
         o.string "--from", "source"
         o.string "--to", "destination"
-        o.string "--where", "where"
-        o.integer "--limit", "limit"
+        o.string "--where", "where", help: false
+        o.integer "--limit", "limit", help: false
         o.string "--exclude", "exclude tables"
         o.string "--config", "config file"
         o.string "--db", "database"
@@ -255,6 +256,7 @@ Options:}
         o.boolean "--to-safe", "accept danger", default: false
         o.boolean "--debug", "debug", default: false
         o.boolean "--list", "list", default: false
+        o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
         o.boolean "--preserve", "preserve existing rows", default: false
         o.boolean "--truncate", "truncate existing rows", default: false
         o.boolean "--schema-only", "schema only", default: false
@@ -520,11 +522,15 @@ Options:}
         if table.is_a?(Array)
           table, sql = table
         end
-        tables[table] = {}
-        tables[table][:sql] = (boom || sql).to_s.gsub("{id}", cast(id)) if boom || sql
+        add_table(tables, table, id, boom || sql)
       end
     end
 
+    def add_table(tables, table, id, boom)
+      tables[table] = {}
+      tables[table][:sql] = boom.gsub("{id}", cast(id)).gsub("{1}", cast(id)) if boom
+    end
+
     def table_list(args, opts, from_uri)
       tables = nil
 
@@ -545,8 +551,7 @@ Options:}
         tables ||= Hash.new { |hash, key| hash[key] = {} }
         to_arr(opts[:tables]).each do |tag|
           table, id = tag.split(":", 2)
-          tables[table] = {}
-          tables[table][:sql] = args[1].to_s.gsub("{id}", cast(id)) if args[1]
+          add_table(tables, table, id, args[1])
         end
       end
 
@@ -559,8 +564,7 @@ Options:}
           if (t = (config["groups"] || {})[group])
             add_tables(tables, t, id, args[1])
           else
-            tables[group] = {}
-            tables[group][:sql] = args[1].to_s.gsub("{id}", cast(id)) if args[1]
+            add_table(tables, group, id, args[1])
           end
         end
       end
@@ -585,5 +589,10 @@ Options:}
     def deprecated(message)
       log "[DEPRECATED] #{message}"
     end
+
+    def log_completed(start_time)
+      time = Time.now - start_time
+      log "Completed in #{time.round(1)}s"
+    end
   end
 end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/pgsync.gemspec

@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "bin"
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
   spec.add_dependency "slop", ">= 4.2.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2016-04-07 00:00:00.000000000 Z
+date: 2016-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -122,8 +122,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/pgsync
 - config.yml
+- exe/pgsync
 - lib/pgsync.rb
 - lib/pgsync/version.rb
 - pgsync.gemspec