pgsync 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d815251b4114bab3d2f124185c3f4d0a6d7ea01b
-  data.tar.gz: 46838871758b47ed72b72de2544e8dca5c4d0448
+  metadata.gz: 792aeac71e01e8b28c8f33eab3392f2afb8cf1ac
+  data.tar.gz: e47f1aed5ef45cdb3af86d2a52a93c16f332e079
 SHA512:
-  metadata.gz: 19599ac80d68278b56ebe96d6ed1b6d5065df06d3384bb900d4876974d250048c8ca1bf9e0ba13fb3a7af4a6981c4f7a132d7d47c52636dab0180ec55995b7ca
-  data.tar.gz: 19e780c6b2006d3c9b96149bfa73b27e7b366f46b160cb0a6ad9f309530f75b36ffa484c956c7de1c3658623c376ef0e2589d4be42807160a618603ebeb6e196
+  metadata.gz: 4c6c37c246796179708f08abd3e43d09a3608545cc02ee4eab03cfeb3138b893e60be4ca8fceb6773ac295346fa194d6275aae660517c870be96b6d93cfb3358
+  data.tar.gz: 0fa9979a1a15d6a5e0da7298eed998f9a04a5067ed2be35d0b67d84ef5ffd2fa85b94cdf1d7f52d9ab42185ce476b57cac89b0f5a182eaacfd011a8ad8b12861

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/.pgsync.yml

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.2.4
+
+- Added `--preserve` option
+- Added `--list` option for groups and tables
+- Added `--limit` option
+
 # 0.2.3
 
 - Fixed `no PostgreSQL user name specified in startup packet`

data/README.md

@@ -14,7 +14,7 @@ And in your project directory, run:
 pgsync setup
 ```
 
-This creates `.pgsync.yml` for you to customize. We recommend checking this into your version control (assuming it doesn’t contain sensitive information).
+This creates `.pgsync.yml` for you to customize. We recommend checking this into your version control (assuming it doesn’t contain sensitive information). `pgsync` commands can be run from this directory or any subdirectory.
 
 ## How to Use
 
@@ -36,6 +36,12 @@ Fetch specific rows (truncates destination table first)
 pgsync products --where "id < 100"
 ```
 
+To preserve existing rows, use:
+
+```sh
+pgsync products --where "id < 100" --preserve
+```
+
 ### Exclude Tables
 
 ```sh

data/lib/pgsync.rb

@@ -7,6 +7,14 @@ require "pg"
 require "parallel"
 require "multiprocessing"
 require "fileutils"
+require "tempfile"
+
+module URI
+  class POSTGRESQL < Generic
+    DEFAULT_PORT = 5432
+  end
+  @@schemes["POSTGRESQL"] = @@schemes["POSTGRES"] = POSTGRESQL
+end
 
 module PgSync
   class Error < StandardError; end
@@ -98,75 +106,127 @@ module PgSync
             end
           end
 
-          in_parallel(tables) do |table|
-            time =
-              benchmark do
-                with_connection(from_uri) do |from_connection|
-                  with_connection(to_uri) do |to_connection|
-                    bad_fields = config["data_rules"]
-
-                    from_fields = columns(from_connection, table, "public")
-                    to_fields = columns(to_connection, table, "public")
-                    shared_fields = to_fields & from_fields
-                    extra_fields = to_fields - from_fields
-                    missing_fields = from_fields - to_fields
-
-                    from_sequences = sequences(from_connection, table, shared_fields)
-                    to_sequences = sequences(to_connection, table, shared_fields)
-                    shared_sequences = to_sequences & from_sequences
-                    extra_sequences = to_sequences - from_sequences
-                    missing_sequences = from_sequences - to_sequences
-
-                    where = opts[:where]
-
-                    @mutex.synchronize do
-                      log "* Syncing #{table}"
-                      if where
-                        log "    #{where}"
-                        where = " WHERE #{opts[:where]}"
-                      end
-                      log "    Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
-                      log "    Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
-                      log "    Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
-                      log "    Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
+          if opts[:list]
+            if args[0] == "groups"
+              pretty_list (config["groups"] || {}).keys
+            else
+              pretty_list tables
+            end
+          else
+            in_parallel(tables) do |table|
+              time =
+                benchmark do
+                  with_connection(from_uri) do |from_connection|
+                    with_connection(to_uri) do |to_connection|
+                      bad_fields = config["data_rules"]
+
+                      from_fields = columns(from_connection, table, "public")
+                      to_fields = columns(to_connection, table, "public")
+                      shared_fields = to_fields & from_fields
+                      extra_fields = to_fields - from_fields
+                      missing_fields = from_fields - to_fields
+
+                      from_sequences = sequences(from_connection, table, shared_fields)
+                      to_sequences = sequences(to_connection, table, shared_fields)
+                      shared_sequences = to_sequences & from_sequences
+                      extra_sequences = to_sequences - from_sequences
+                      missing_sequences = from_sequences - to_sequences
+
+                      where = opts[:where]
+                      limit = opts[:limit]
+                      sql_clause = String.new
+
+                      @mutex.synchronize do
+                        log "* Syncing #{table}"
+                        if where
+                          log "    #{where}"
+                          sql_clause << " WHERE #{opts[:where]}"
+                        end
+                        if limit
+                          log "    LIMIT #{limit}"
+                          sql_clause << " LIMIT #{limit}"
+                        end
+                        log "    Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
+                        log "    Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
+                        log "    Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
+                        log "    Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
 
-                      if shared_fields.empty?
-                        log "    No fields to copy"
+                        if shared_fields.empty?
+                          log "    No fields to copy"
+                        end
                       end
-                    end
 
-                    if shared_fields.any?
-                      copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], f, from_connection)} AS #{escape_identifier(f)}" : escape_identifier(f) }.join(", ")
-                      fields = shared_fields.map { |f| escape_identifier(f) }.join(", ")
+                      if shared_fields.any?
+                        copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], f, from_connection)} AS #{escape_identifier(f)}" : escape_identifier(f) }.join(", ")
+                        fields = shared_fields.map { |f| escape_identifier(f) }.join(", ")
 
-                      seq_values = {}
-                      shared_sequences.each do |seq|
-                        seq_values[seq] = from_connection.exec("select last_value from #{seq}").to_a[0]["last_value"]
-                      end
+                        seq_values = {}
+                        shared_sequences.each do |seq|
+                          seq_values[seq] = from_connection.exec("select last_value from #{seq}").to_a[0]["last_value"]
+                        end
 
-                      to_connection.exec("TRUNCATE #{table} CASCADE")
-                      to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
-                        from_connection.copy_data "COPY (SELECT #{copy_fields} FROM #{table}#{where}) TO STDOUT" do
-                          while row = from_connection.get_copy_data
-                            to_connection.put_copy_data(row)
+                        copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{sql_clause}) TO STDOUT"
+                        if opts[:preserve]
+                          primary_key = self.primary_key(from_connection, table, "public")
+                          abort "No primary key" unless primary_key
+
+                          temp_table = "pgsync_#{rand(1_000_000_000)}"
+                          file = Tempfile.new(temp_table)
+                          begin
+                            from_connection.copy_data copy_to_command do
+                              while row = from_connection.get_copy_data
+                                file.write(row)
+                              end
+                            end
+                            file.rewind
+
+                            to_connection.transaction do
+                              # create a temp table
+                              to_connection.exec("CREATE TABLE #{temp_table} AS SELECT * FROM #{table} WITH NO DATA")
+
+                              # load file
+                              to_connection.copy_data "COPY #{temp_table} (#{fields}) FROM STDIN" do
+                                file.each do |row|
+                                  to_connection.put_copy_data(row)
+                                end
+                              end
+
+                              # insert into
+                              to_connection.exec("INSERT INTO #{table} (SELECT * FROM #{temp_table} WHERE NOT EXISTS (SELECT 1 FROM #{table} WHERE #{table}.#{primary_key} = #{temp_table}.#{primary_key}))")
+
+                              # delete temp table
+                              to_connection.exec("DROP TABLE #{temp_table}")
+                            end
+                          ensure
+                             file.close
+                             file.unlink
+                          end
+                        else
+                          to_connection.exec("TRUNCATE #{table} CASCADE")
+                          to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
+                            from_connection.copy_data copy_to_command do
+                              while row = from_connection.get_copy_data
+                                to_connection.put_copy_data(row)
+                              end
+                            end
                           end
                         end
-                      end
-                      seq_values.each do |seq, value|
-                        to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
+                        seq_values.each do |seq, value|
+                          to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
+                        end
                       end
                     end
                   end
                 end
-              end
 
-            @mutex.synchronize do
-              log "* DONE #{table} (#{time.round(1)}s)"
+              @mutex.synchronize do
+                log "* DONE #{table} (#{time.round(1)}s)"
+              end
             end
-          end
 
-          time = Time.now - start_time
-          log "Completed in #{time.round(1)}s"
+            time = Time.now - start_time
+            log "Completed in #{time.round(1)}s"
+          end
         end
       end
       true
@@ -189,12 +249,15 @@ Options:}
         o.string "--from", "source"
         o.string "--to", "destination"
         o.string "--where", "where"
+        o.integer "--limit", "limit"
         o.string "--exclude", "exclude tables"
         o.string "--config", "config file"
         o.string "--db", "database"
         # TODO much better name for this option
         o.boolean "--to-safe", "accept danger", default: false
         o.boolean "--debug", "debug", default: false
+        o.boolean "--list", "list", default: false
+        o.boolean "--preserve", "preserve", default: false
         o.on "-v", "--version", "print the version" do
           log PgSync::VERSION
           @exit = true
@@ -294,6 +357,27 @@ Options:}
       conn.exec_params(query, [schema, table]).to_a.size > 0
     end
 
+    # http://stackoverflow.com/a/20537829
+    def primary_key(conn, table, schema)
+      query = <<-SQL
+        SELECT
+          pg_attribute.attname,
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
+        FROM
+          pg_index, pg_class, pg_attribute, pg_namespace
+        WHERE
+          pg_class.oid = $2::regclass AND
+          indrelid = pg_class.oid AND
+          nspname = $1 AND
+          pg_class.relnamespace = pg_namespace.oid AND
+          pg_attribute.attrelid = pg_class.oid AND
+          pg_attribute.attnum = any(pg_index.indkey) AND
+          indisprimary
+      SQL
+      row = conn.exec_params(query, [schema, table]).to_a[0]
+      row && row["attname"]
+    end
+
     # TODO better performance
     def rule_match?(table, column, rule)
       regex = Regexp.new('\A' + Regexp.escape(rule).gsub('\*','[^\.]*') + '\z')
@@ -359,8 +443,10 @@ Options:}
 
     def parse_uri(url)
       uri = URI.parse(url)
+      uri.scheme ||= "postgres"
       uri.host ||= "localhost"
       uri.port ||= 5432
+      uri.path = "/#{uri.path}" if uri.path && uri.path[0] != "/"
       uri
     end
 
@@ -419,5 +505,11 @@ Options:}
         Parallel.each(tables, &block)
       end
     end
+
+    def pretty_list(items)
+      items.each do |item|
+        log item
+      end
+    end
   end
 end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-31 00:00:00.000000000 Z
+date: 2016-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop