pgsync 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be53714690e45e12a419c524673bddecbb151fa1
-  data.tar.gz: 6c4a70c4296a2bc96121fbfc7b0512dc9e47ab1b
+  metadata.gz: c8bd6a2e877c863a693c0447ac9dc8a621a4c2c5
+  data.tar.gz: a5322f143b1daaa5cf60b55dbd664b22d31f7b7c
 SHA512:
-  metadata.gz: 13542ba5e8ac5f81be20154f51ba9ea45fd9fc2bfb8d9e3abcbd32eee620f4a3fbb1c71493909ef44664982240ea59e05f0ed829032d8c6ec06f0a4c4b53cf7c
-  data.tar.gz: fe69eac5b5acdfa879664ed5bc88ae99abe853f5cbf4af093b43849be98315acb2695f2fd92ca49b883322a03d541f2da7e79c7914040bbf4ab0943ff68db5c5
+  metadata.gz: 9370577fcad8039401eb4d19dcad7ae5d24af1c0a1dfb0904e940ff8ab802a33218390dbceda4e214ea0f2591c507b14099a09819f1b9df75463ea64b16dc2b7
+  data.tar.gz: cd474765176aea15d99175ae302d47eae50bb3e507ab7cfe7c8d605cf22699fce49c4fbdf8793116e7a7b629b15292ee732c804c67090d06ded178bf73d78933

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.3.4
+
+- Added `--in-batches` mode for production transfers with `--batch-size` and `--sleep`
+
 # 0.3.3
 
 - Added `-d` option as an alias for `--db`

data/lib/pgsync.rb

@@ -176,7 +176,48 @@ module PgSync
                 end
 
                 copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{sql_clause}) TO STDOUT"
-                if !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
+                if opts[:in_batches]
+                  primary_key = self.primary_key(from_connection, table, "public")
+                  abort "No primary key" unless primary_key
+
+                  from_max_id = max_id(from_connection, table, primary_key, sql_clause)
+                  to_max_id = max_id(to_connection, table, primary_key, sql_clause) + 1
+
+                  if to_max_id == 1
+                    from_min_id = min_id(from_connection, table, primary_key, sql_clause)
+                    to_max_id = from_min_id if from_min_id > 0
+                  end
+
+                  starting_id = to_max_id
+                  batch_size = opts[:batch_size]
+
+                  i = 1
+                  batch_count = ((from_max_id - starting_id + 1) / batch_size.to_f).ceil
+
+                  while starting_id <= from_max_id
+                    where = "#{primary_key} >= #{starting_id} AND #{primary_key} < #{starting_id + batch_size}"
+                    log "    #{i}/#{batch_count}: #{where}"
+
+                    # TODO be smarter for advance sql clauses
+                    batch_sql_clause = " #{sql_clause.length > 0 ? "#{sql_clause} AND" : "WHERE"} #{where}"
+
+                    batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{batch_sql_clause}) TO STDOUT"
+                    to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
+                      from_connection.copy_data batch_copy_to_command do
+                        while row = from_connection.get_copy_data
+                          to_connection.put_copy_data(row)
+                        end
+                      end
+                    end
+
+                    starting_id += batch_size
+                    i += 1
+
+                    if opts[:sleep] && starting_id <= from_max_id
+                      sleep(opts[:sleep])
+                    end
+                  end
+                elsif !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
                   primary_key = self.primary_key(from_connection, table, "public")
                   abort "No primary key" unless primary_key
 
@@ -264,6 +305,9 @@ Options:}
         o.boolean "--schema-only", "schema only", default: false
         o.boolean "--no-rules", "do not apply data rules", default: false
         o.boolean "--setup", "setup", default: false
+        o.boolean "--in-batches", "in batches", default: false, help: false
+        o.integer "--batch-size", "batch size", default: 10000, help: false
+        o.float "--sleep", "sleep", default: 0, help: false
         o.on "-v", "--version", "print the version" do
           log PgSync::VERSION
           @exit = true
@@ -505,7 +549,7 @@ Options:}
     end
 
     def in_parallel(tables, &block)
-      if @options[:debug]
+      if @options[:debug] || @options[:in_batches]
         tables.each(&block)
       else
         Parallel.each(tables, &block)
@@ -592,6 +636,14 @@ Options:}
       tables
     end
 
+    def max_id(conn, table, primary_key, sql_clause = nil)
+      conn.exec("SELECT MAX(#{escape_identifier(primary_key)}) FROM #{escape_identifier(table)}#{sql_clause}").to_a[0]["max"].to_i
+    end
+
+    def min_id(conn, table, primary_key, sql_clause = nil)
+      conn.exec("SELECT MIN(#{escape_identifier(primary_key)}) FROM #{escape_identifier(table)}#{sql_clause}").to_a[0]["min"].to_i
+    end
+
     def cast(value)
       value.to_s
     end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-26 00:00:00.000000000 Z
+date: 2016-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop