pgsync 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 792aeac71e01e8b28c8f33eab3392f2afb8cf1ac
-  data.tar.gz: e47f1aed5ef45cdb3af86d2a52a93c16f332e079
+  metadata.gz: 06dcfff629b2d6d8e5fb56719c8527b908aa41d8
+  data.tar.gz: c3afdf8c5ac9b352c38169c65fe59ff1d55e61a5
 SHA512:
-  metadata.gz: 4c6c37c246796179708f08abd3e43d09a3608545cc02ee4eab03cfeb3138b893e60be4ca8fceb6773ac295346fa194d6275aae660517c870be96b6d93cfb3358
-  data.tar.gz: 0fa9979a1a15d6a5e0da7298eed998f9a04a5067ed2be35d0b67d84ef5ffd2fa85b94cdf1d7f52d9ab42185ce476b57cac89b0f5a182eaacfd011a8ad8b12861
+  metadata.gz: fb13e085bd8bb12d027166481a64cf28d6c7dda07aee78996d65dcf8f8beba91af9850d6ebf544d0cd937abfed07f4c209147c08b010306f3ef4223b49591718
+  data.tar.gz: 2019ba48fac3b89f17d0c70069f1c975f1c052bed67bd2f8c63b7d8eaca5b396c2dbafe434dd7f0887a2b062a154210ce41bf76fd5067e190f2121209e7d6312

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+# 0.3.0
+
+- More powerful groups
+- Overwrite rows by default when `WHERE` clause (previously truncated)
+- Added `pgsync users "WHERE id = 1`
+- Added `pgsync group1`, shorthand for `pgsync groups group1`
+- Added `--schema-only` option
+- Added `--no-rules` option
+- Added `--setup` option
+- Added `--truncate` option
+
 # 0.2.4
 
 - Added `--preserve` option

data/README.md

@@ -2,6 +2,8 @@
 
 Quickly and securely sync data between environments
 
+:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
+
 ## Installation
 
 ```sh
@@ -11,7 +13,7 @@ gem install pgsync
 And in your project directory, run:
 
 ```sh
-pgsync setup
+pgsync --setup
 ```
 
 This creates `.pgsync.yml` for you to customize. We recommend checking this into your version control (assuming it doesn’t contain sensitive information). `pgsync` commands can be run from this directory or any subdirectory.
@@ -30,16 +32,10 @@ Fetch specific tables
 pgsync table1,table2
 ```
 
-Fetch specific rows (truncates destination table first)
-
-```sh
-pgsync products --where "id < 100"
-```
-
-To preserve existing rows, use:
+Fetch specific rows
 
 ```sh
-pgsync products --where "id < 100" --preserve
+pgsync products "WHERE id < 1000"
 ```
 
 ### Exclude Tables
@@ -63,38 +59,52 @@ exclude:
   - schema_migrations
 ```
 
-### Schema
+### Groups
 
-Fetch schema
+Define groups in `.pgsync.yml`:
 
-```sh
-pgsync schema
+```yml
+groups:
+  group1:
+    - table1
+    - table2
 ```
 
-Specify tables
+And run:
 
 ```sh
-pgsync schema table1,table2
+pgsync group1
 ```
 
-### Groups
-
-Define groups in `.pgsync.yml`:
+You can also sync specific rows:
 
 ```yml
 groups:
-  group1:
-    - table1
-    - table2
-  group2:
-    - table3
-    - table4
+  user:
+    users: "WHERE id = {id}"
+    orders: "WHERE user_id = {id}"
 ```
 
 And run:
 
 ```sh
-pgsync groups group1,group2
+pgsync user:123
+```
+
+to get rows associated with user `123`.
+
+### Schema
+
+Fetch schema
+
+```sh
+pgsync --schema-only
+```
+
+Specify tables
+
+```sh
+pgsync table1,table2 --schema-only
 ```
 
 ## Sensitive Information
@@ -136,7 +146,7 @@ Options for replacement are:
 To use with multiple databases, run:
 
 ```sh
-pgsync setup db2
+pgsync --setup db2
 ```
 
 This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands with:

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.2.4"
+  VERSION = "0.3.0"
 end

data/lib/pgsync.rb

@@ -40,8 +40,13 @@ module PgSync
       end
       command = args[0]
 
+      # setup hack
+      if opts[:setup]
+        command = "setup"
+        args[1] = args[0]
+      end
       if command == "setup"
-        setup(db_config_file(args[1]) || config_file)
+        setup(db_config_file(args[1]) || config_file || ".pgsync.yml")
       else
         source = parse_source(opts[:from])
         abort "No source" unless source
@@ -55,11 +60,16 @@ module PgSync
         print_uri("From", source_uri)
         print_uri("To", destination_uri)
 
-        if args[0] == "schema"
+        from_uri = source_uri
+        to_uri = destination_uri
+
+        tables = table_list(args, opts, from_uri)
+
+        if args[0] == "schema" || opts[:schema_only]
           time =
             benchmark do
               log "* Dumping schema"
-              tables = to_arr(args[1]).map { |t| "-t #{t}" }.join(" ")
+              tables = tables.keys.map { |t| "-t #{t}" }.join(" ")
               dump_command = "pg_dump --verbose --schema-only --no-owner --no-acl --clean #{tables} #{to_url(source_uri)}"
               restore_command = "psql -q -d #{to_url(destination_uri)}"
               system("#{dump_command} | #{restore_command}")
@@ -67,39 +77,8 @@ module PgSync
 
           log "* DONE (#{time.round(1)}s)"
         else
-          from_uri = source_uri
-          to_uri = destination_uri
-
-          tables =
-            if args[0] == "groups"
-              specified_groups = to_arr(args[1])
-              specified_groups.map do |group|
-                if (tables = config["groups"][group])
-                  tables
-                else
-                  abort "Group not found: #{group}"
-                end
-              end.flatten
-            elsif args[0] == "tables"
-              to_arr(args[1])
-            elsif args[0]
-              to_arr(args[0])
-            else
-              nil
-            end
-
-          with_connection(from_uri, timeout: 3) do |conn|
-            tables ||= self.tables(conn, "public") - to_arr(opts[:exclude])
-
-            tables.each do |table|
-              unless table_exists?(conn, table, "public")
-                abort "Table does not exist in source: #{table}"
-              end
-            end
-          end
-
           with_connection(to_uri, timeout: 3) do |conn|
-            tables.each do |table|
+            tables.keys.each do |table|
               unless table_exists?(conn, table, "public")
                 abort "Table does not exist in destination: #{table}"
               end
@@ -110,130 +89,143 @@ module PgSync
             if args[0] == "groups"
               pretty_list (config["groups"] || {}).keys
             else
-              pretty_list tables
+              pretty_list tables.keys
             end
           else
-            in_parallel(tables) do |table|
-              time =
-                benchmark do
-                  with_connection(from_uri) do |from_connection|
-                    with_connection(to_uri) do |to_connection|
-                      bad_fields = config["data_rules"]
-
-                      from_fields = columns(from_connection, table, "public")
-                      to_fields = columns(to_connection, table, "public")
-                      shared_fields = to_fields & from_fields
-                      extra_fields = to_fields - from_fields
-                      missing_fields = from_fields - to_fields
-
-                      from_sequences = sequences(from_connection, table, shared_fields)
-                      to_sequences = sequences(to_connection, table, shared_fields)
-                      shared_sequences = to_sequences & from_sequences
-                      extra_sequences = to_sequences - from_sequences
-                      missing_sequences = from_sequences - to_sequences
-
-                      where = opts[:where]
-                      limit = opts[:limit]
-                      sql_clause = String.new
-
-                      @mutex.synchronize do
-                        log "* Syncing #{table}"
-                        if where
-                          log "    #{where}"
-                          sql_clause << " WHERE #{opts[:where]}"
-                        end
-                        if limit
-                          log "    LIMIT #{limit}"
-                          sql_clause << " LIMIT #{limit}"
-                        end
-                        log "    Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
-                        log "    Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
-                        log "    Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
-                        log "    Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
+            in_parallel(tables) do |table, table_opts|
+              sync_table(table, opts.merge(table_opts), from_uri, to_uri)
+            end
 
-                        if shared_fields.empty?
-                          log "    No fields to copy"
-                        end
+            time = Time.now - start_time
+            log "Completed in #{time.round(1)}s"
+          end
+        end
+      end
+      true
+    end
+
+    protected
+
+    def sync_table(table, opts, from_uri, to_uri)
+      time =
+        benchmark do
+          with_connection(from_uri) do |from_connection|
+            with_connection(to_uri) do |to_connection|
+              bad_fields = opts[:no_rules] ? [] : config["data_rules"]
+
+              from_fields = columns(from_connection, table, "public")
+              to_fields = columns(to_connection, table, "public")
+              shared_fields = to_fields & from_fields
+              extra_fields = to_fields - from_fields
+              missing_fields = from_fields - to_fields
+
+              from_sequences = sequences(from_connection, table, shared_fields)
+              to_sequences = sequences(to_connection, table, shared_fields)
+              shared_sequences = to_sequences & from_sequences
+              extra_sequences = to_sequences - from_sequences
+              missing_sequences = from_sequences - to_sequences
+
+              where = opts[:where]
+              limit = opts[:limit]
+              sql_clause = String.new
+
+              @mutex.synchronize do
+                log "* Syncing #{table}"
+                if opts[:sql]
+                  log "    #{opts[:sql]}"
+                  sql_clause << " #{opts[:sql]}"
+                end
+                if where
+                  log "    #{where}"
+                  sql_clause << " WHERE #{opts[:where]}"
+                end
+                if limit
+                  log "    LIMIT #{limit}"
+                  sql_clause << " LIMIT #{limit}"
+                end
+                log "    Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
+                log "    Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
+                log "    Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
+                log "    Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
+
+                if shared_fields.empty?
+                  log "    No fields to copy"
+                end
+              end
+
+              if shared_fields.any?
+                copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, from_connection)} AS #{escape_identifier(f)}" : "#{table}.#{escape_identifier(f)}" }.join(", ")
+                fields = shared_fields.map { |f| escape_identifier(f) }.join(", ")
+
+                seq_values = {}
+                shared_sequences.each do |seq|
+                  seq_values[seq] = from_connection.exec("select last_value from #{seq}").to_a[0]["last_value"]
+                end
+
+                copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{sql_clause}) TO STDOUT"
+                if !opts[:truncate] && (opts[:preserve] || !sql_clause.empty?)
+                  primary_key = self.primary_key(from_connection, table, "public")
+                  abort "No primary key" unless primary_key
+
+                  temp_table = "pgsync_#{rand(1_000_000_000)}"
+                  file = Tempfile.new(temp_table)
+                  begin
+                    from_connection.copy_data copy_to_command do
+                      while row = from_connection.get_copy_data
+                        file.write(row)
                       end
+                    end
+                    file.rewind
 
-                      if shared_fields.any?
-                        copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], f, from_connection)} AS #{escape_identifier(f)}" : escape_identifier(f) }.join(", ")
-                        fields = shared_fields.map { |f| escape_identifier(f) }.join(", ")
+                    to_connection.transaction do
+                      # create a temp table
+                      to_connection.exec("CREATE TABLE #{temp_table} AS SELECT * FROM #{table} WITH NO DATA")
 
-                        seq_values = {}
-                        shared_sequences.each do |seq|
-                          seq_values[seq] = from_connection.exec("select last_value from #{seq}").to_a[0]["last_value"]
+                      # load file
+                      to_connection.copy_data "COPY #{temp_table} (#{fields}) FROM STDIN" do
+                        file.each do |row|
+                          to_connection.put_copy_data(row)
                         end
+                      end
 
-                        copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{sql_clause}) TO STDOUT"
-                        if opts[:preserve]
-                          primary_key = self.primary_key(from_connection, table, "public")
-                          abort "No primary key" unless primary_key
-
-                          temp_table = "pgsync_#{rand(1_000_000_000)}"
-                          file = Tempfile.new(temp_table)
-                          begin
-                            from_connection.copy_data copy_to_command do
-                              while row = from_connection.get_copy_data
-                                file.write(row)
-                              end
-                            end
-                            file.rewind
-
-                            to_connection.transaction do
-                              # create a temp table
-                              to_connection.exec("CREATE TABLE #{temp_table} AS SELECT * FROM #{table} WITH NO DATA")
-
-                              # load file
-                              to_connection.copy_data "COPY #{temp_table} (#{fields}) FROM STDIN" do
-                                file.each do |row|
-                                  to_connection.put_copy_data(row)
-                                end
-                              end
-
-                              # insert into
-                              to_connection.exec("INSERT INTO #{table} (SELECT * FROM #{temp_table} WHERE NOT EXISTS (SELECT 1 FROM #{table} WHERE #{table}.#{primary_key} = #{temp_table}.#{primary_key}))")
-
-                              # delete temp table
-                              to_connection.exec("DROP TABLE #{temp_table}")
-                            end
-                          ensure
-                             file.close
-                             file.unlink
-                          end
-                        else
-                          to_connection.exec("TRUNCATE #{table} CASCADE")
-                          to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
-                            from_connection.copy_data copy_to_command do
-                              while row = from_connection.get_copy_data
-                                to_connection.put_copy_data(row)
-                              end
-                            end
-                          end
-                        end
-                        seq_values.each do |seq, value|
-                          to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
-                        end
+                      if opts[:preserve]
+                        # insert into
+                        to_connection.exec("INSERT INTO #{table} (SELECT * FROM #{temp_table} WHERE NOT EXISTS (SELECT 1 FROM #{table} WHERE #{table}.#{primary_key} = #{temp_table}.#{primary_key}))")
+                      else
+                        to_connection.exec("DELETE FROM #{table} WHERE #{primary_key} IN (SELECT #{primary_key} FROM #{temp_table})")
+                        to_connection.exec("INSERT INTO #{table} (SELECT * FROM #{temp_table})")
+                      end
+
+                      # delete temp table
+                      to_connection.exec("DROP TABLE #{temp_table}")
+                    end
+                  ensure
+                     file.close
+                     file.unlink
+                  end
+                else
+                  to_connection.exec("TRUNCATE #{table} CASCADE")
+                  to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
+                    from_connection.copy_data copy_to_command do
+                      while row = from_connection.get_copy_data
+                        to_connection.put_copy_data(row)
                       end
                     end
                   end
                 end
-
-              @mutex.synchronize do
-                log "* DONE #{table} (#{time.round(1)}s)"
+                seq_values.each do |seq, value|
+                  to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
+                end
               end
             end
-
-            time = Time.now - start_time
-            log "Completed in #{time.round(1)}s"
           end
         end
+
+      @mutex.synchronize do
+        log "* DONE #{table} (#{time.round(1)}s)"
       end
-      true
     end
 
-    protected
-
     def parse_args(args)
       opts = Slop.parse(args) do |o|
         o.banner = %{Usage:
@@ -257,7 +249,11 @@ Options:}
         o.boolean "--to-safe", "accept danger", default: false
         o.boolean "--debug", "debug", default: false
         o.boolean "--list", "list", default: false
-        o.boolean "--preserve", "preserve", default: false
+        o.boolean "--preserve", "preserve existing rows", default: false
+        o.boolean "--truncate", "truncate existing rows", default: false
+        o.boolean "--schema-only", "schema only", default: false
+        o.boolean "--no-rules", "do not apply data rules", default: false
+        o.boolean "--setup", "setup", default: false
         o.on "-v", "--version", "print the version" do
           log PgSync::VERSION
           @exit = true
@@ -385,7 +381,7 @@ Options:}
     end
 
     # TODO wildcard rules
-    def apply_strategy(rule, column, conn)
+    def apply_strategy(rule, table, column, conn)
       if rule.is_a?(Hash)
         if rule.key?("value")
           escape(rule["value"])
@@ -396,13 +392,13 @@ Options:}
         end
       else
         strategies = {
-          "unique_email" => "'email' || id || '@example.org'",
+          "unique_email" => "'email' || #{table}.id || '@example.org'",
           "untouched" => escape_identifier(column),
-          "unique_phone" => "(id + 1000000000)::text",
+          "unique_phone" => "(#{table}.id + 1000000000)::text",
           "random_int" => "(RAND() * 10)::int",
           "random_date" => "'1970-01-01'",
           "random_time" => "NOW()",
-          "unique_secret" => "'secret' || id",
+          "unique_secret" => "'secret' || #{table}.id",
           "random_ip" => "'127.0.0.1'",
           "random_letter" => "'A'",
           "null" => "NULL",
@@ -511,5 +507,68 @@ Options:}
         log item
       end
     end
+
+    def table_list(args, opts, from_uri)
+      tables = nil
+
+      if args[0] == "groups"
+        tables = Hash.new { |hash, key| hash[key] = {} }
+        specified_groups = to_arr(args[1])
+        specified_groups.map do |tag|
+          group, id = tag.split(":", 2)
+          if (t = config["goups"][group])
+            t.each do |table|
+              tables[table] = {}
+              tables[table][:sql] = args[2].to_s.gsub("{id}", cast(id)) if args[2]
+            end
+          else
+            abort "Group not found: #{group}"
+          end
+        end
+      elsif args[0] == "tables"
+        tables = Hash.new { |hash, key| hash[key] = {} }
+        to_arr(args[1]).each do |tag|
+          table, id = tag.split(":", 2)
+          tables[table] = {}
+          tables[table][:sql] = args[2].to_s.gsub("{id}", cast(id)) if args[2]
+        end
+      elsif args[0]
+        # could be a group, table, or mix
+        tables = Hash.new { |hash, key| hash[key] = {} }
+        specified_groups = to_arr(args[0])
+        specified_groups.map do |tag|
+          group, id = tag.split(":", 2)
+          if (t = config["groups"][group])
+            t.each do |table|
+              sql = nil
+              if table.is_a?(Array)
+                table, sql = table
+              end
+              tables[table] = {}
+              tables[table][:sql] = (args[1] || sql).to_s.gsub("{id}", cast(id)) if args[1] || sql
+            end
+          else
+            tables[group] = {}
+            tables[group][:sql] = args[1].to_s.gsub("{id}", cast(id)) if args[1]
+          end
+        end
+      end
+
+      with_connection(from_uri, timeout: 3) do |conn|
+        tables ||= Hash[(self.tables(conn, "public") - to_arr(opts[:exclude])).map { |k| [k, {}] }]
+
+        tables.keys.each do |table|
+          unless table_exists?(conn, table, "public")
+            abort "Table does not exist in source: #{table}"
+          end
+        end
+      end
+
+      tables
+    end
+
+    def cast(value)
+      value.to_s
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-05 00:00:00.000000000 Z
+date: 2016-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop