zdm 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3bbad8595a090ca55222e019298a51b521272442
-  data.tar.gz: a17df5df455399660ae8b4f8ab825ca50339e0cd
+  metadata.gz: 30ce840e815a4a41f68dbf90830f69f253b5c75f
+  data.tar.gz: 453c0299bbfe3bfe72850c1f644bcd7460ed1d79
 SHA512:
-  metadata.gz: 6290405bf6180e28c04e3aa2ed217b7a2bb4b3982a64fa82bb1f31a464a4e493e0e286c0c2f91646cb9b88972e9aa34f5098eba2d2b786b2dba40c1c1ae2f191
-  data.tar.gz: 10cd09f9ad8a6a86266971d30d75baa7d27c6c95d5cdfbc10bb0b89dca4598af25fa48f94e00ccfd0f5ecd44e288854dae479a6104fdb2f4cd82cd0cf4ad1534
+  metadata.gz: a1fb2a19947baf93d0be201af690a51e22bd5ef05d9766b310ffb194e6ba7f4c6bfed360733b7d3851a77c94cf9ac9632a6cf8a5f5846cec52b00b4b6c4ca43f
+  data.tar.gz: 583007414de8ff16f071760707ec701276aa6fab9c8efcd17e2efe7d6de15c7358b90ee6195f2ed7bf113f4eee369c45fd9de96a81e0760703a2e5c5c2a3929e

data/gemfiles/4.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.5)
+    zdm (1.0.6)
       activerecord (>= 4.0)
 
 GEM

data/gemfiles/4.2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.5)
+    zdm (1.0.6)
       activerecord (>= 4.0)
 
 GEM

data/gemfiles/5.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.5)
+    zdm (1.0.6)
       activerecord (>= 4.0)
 
 GEM

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Zdm
-  VERSION = '1.0.5'
+  VERSION = '1.0.6'
 end

data/lib/zdm.rb

@@ -34,6 +34,13 @@ module Zdm
     MIN_BATCH_SIZE = 10_000
     PROGRESS_EVERY = 30  # seconds
     def execute_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
+      find_in_batches(table_name, start: start, finish: finish, batch_size: batch_size, progress_every: progress_every) do |batch_start, batch_end|
+        sql = yield batch_start, batch_end
+        execute(sql) if sql
+      end
+    end
+
+    def find_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
       min = start || connection.select_value('SELECT MIN(`id`) FROM %s' % table_name)
       return unless min
 
@@ -48,8 +55,7 @@ module Zdm
         batch_end = [batch_start + batch_size - 1, max].min
         start_batch_time = Time.now
 
-        sql = yield batch_start, batch_end
-        execute(sql) if sql
+        yield batch_start, batch_end
 
         if $exit
           write(table_name, 'Received SIGTERM, exiting...')

data/spec/zdm_spec.rb

@@ -93,6 +93,31 @@ describe Zdm do
     expect(archive_tables.length).to eq(2)
   end
 
+  context 'find_in_batches' do
+    before(:example) do
+      @conn = ActiveRecord::Base.connection
+      (1..20).each do |idx|
+        @conn.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'person-#{idx}','P#{idx}','2017-03-01 23:59:59')])
+      end
+      Zdm.io = StringIO.new
+    end
+
+    after(:example) do
+      @conn.execute(%[DELETE FROM people WHERE name LIKE 'person%'])
+    end
+
+    it 'find all in batches' do
+      found = []
+      Zdm.find_in_batches('people', batch_size: 4, progress_every: 1) do |batch_start, batch_end|
+        sleep(0.8)
+        found << [batch_start, batch_end]
+      end
+      expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
+      expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
+      expect(found).to eq([[1, 4], [5, 8], [9, 12], [13, 16], [17, 20], [21, 22]])
+    end
+  end
+
   context 'execute_in_batches' do
     before(:example) do
       @conn = ActiveRecord::Base.connection
@@ -112,7 +137,8 @@ describe Zdm do
         sleep(0.8)
         @sql % [batch_start, batch_end]
       end
-      expect(Zdm.io.string).to eq(%[people: 36.36% (8/22)\npeople: 72.73% (16/22)\npeople: Completed (4 secs)\n])
+      expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/22\)/)
+      expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
       expect(@conn.select_value(%[SELECT COUNT(*) FROM people WHERE code LIKE '%U'])).to eq(22)
     end
 
@@ -120,10 +146,12 @@ describe Zdm do
       batches = []
       Zdm.execute_in_batches('people', start: 5, finish: 18, batch_size: 4, progress_every: 1) do |batch_start, batch_end|
         sleep(0.6)
-        batches << @sql % [batch_start, batch_end]
-        @sql % [batch_start, batch_end]
+        sql = @sql % [batch_start, batch_end]
+        batches << sql
+        sql
       end
-      expect(Zdm.io.string).to eq(%[people: 57.14% (8/14)\npeople: Completed (2 secs)\n])
+      expect(Zdm.io.string).to match(/people: \d+\.\d+% \(\d+\/14\)/)
+      expect(Zdm.io.string).to match(/people: Completed \(\d+ secs\)$/)
       expect(batches).to eq([
         %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 5 AND 8],
         %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 9 AND 12],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zdm
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - ITRP Institute, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-21 00:00:00.000000000 Z
+date: 2017-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord