zdm 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e342d256ff439757a08cda9ad15897c90278527
-  data.tar.gz: 72894ff47ca9aaca42c6cd0a3b07bc3d0f492490
+  metadata.gz: 83bf151ca9cb0909f5e1232b0d78f04dbef8cce2
+  data.tar.gz: 60d49fe72d7e6d6c122d101505e3ad29dafe190a
 SHA512:
-  metadata.gz: a57d77a2afd53a03508aac7a253d7521ced90318e52ce0c8583e8a2cc5960804b4631f4e94fb6e6424599bc0b7b2c8c578beea3c2ec6124203f1b0232ae11514
-  data.tar.gz: ff0c63a2eab00afa7fc4ec348fd1088348cdec28ecd23214b42818d2c3c6e9da01b96c452257292248234f30a983ba60e6b8e1cf031a3e778d132ebda75103fc
+  metadata.gz: d3db65316fcd480937bdb39fe7f69b1a725aa7a03de4b677a201abaf35227a4ab43e338618452ebe84c680b236532753d9013fa80eaf631b157e968b4e296ac7
+  data.tar.gz: 6afe1ee9375edc2b7cd61208cf530fe54906dacab4957db70f37d94a5fca8839e05062c4c1e211aae4d1c0d92dec28ae71d4d4e9f7dd99c1054cb61848c63312

data/gemfiles/4.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.3)
+    zdm (1.0.4)
       activerecord (>= 4.0)
 
 GEM

data/gemfiles/4.2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.3)
+    zdm (1.0.4)
       activerecord (>= 4.0)
 
 GEM

data/gemfiles/5.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.3)
+    zdm (1.0.4)
       activerecord (>= 4.0)
 
 GEM

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Zdm
-  VERSION = '1.0.3'
+  VERSION = '1.0.4'
 end

data/lib/zdm.rb

@@ -12,22 +12,89 @@ module Zdm
     end
 
     def cleanup(before: nil)
-      conn = ActiveRecord::Base.connection
-      zdm_tables = conn.send(tables_method).select { |name| name.starts_with?('zdm_') }
+      zdm_tables = connection.send(tables_method).select { |name| name.starts_with?('zdm_') }
       zdm_tables.each { |name| Migrator.new(Table.new(name.sub(/^zdm_/, ''))).cleanup }
 
-      zdm_archive_tables = conn.send(tables_method).select { |name| name.starts_with?('zdma_') }
+      zdm_archive_tables = connection.send(tables_method).select { |name| name.starts_with?('zdma_') }
       if before
         zdm_archive_tables.select! { |table|
           Time.strptime(table, 'zdma_%Y%m%d_%H%M%S%N') <= before
         }
       end
-      zdm_archive_tables.each { |name| conn.execute('DROP TABLE `%s`' % name) }
+      zdm_archive_tables.each { |name| execute('DROP TABLE `%s`' % name) }
     end
 
     def tables_method
       ActiveRecord.version.to_s =~ /^5/ ? :data_sources : :tables
     end
+
+    BATCH_SIZE = 40_000
+    DECREASE_THROTTLER = 4  # seconds
+    DECREASE_SIZE = 5_000
+    MIN_BATCH_SIZE = 10_000
+    PROGRESS_EVERY = 30  # seconds
+    def execute_in_batches(table_name, start: nil, finish: nil, batch_size: BATCH_SIZE, progress_every: PROGRESS_EVERY, &block)
+      min = start || connection.select_value('SELECT MIN(`id`) FROM %s' % table_name)
+      return unless min
+
+      max = finish || connection.select_value('SELECT MAX(`id`) FROM %s' % table_name)
+      todo = max - min + 1
+      return unless todo > 0
+
+      batch_end = min - 1
+      start_time = last_progress = Time.now
+      while true
+        batch_start = batch_end + 1
+        batch_end = [batch_start + batch_size - 1, max].min
+        start_batch_time = Time.now
+
+        sql = yield batch_start, batch_end
+        execute(sql) if sql
+
+        if $exit
+          write(table_name, 'Received SIGTERM, exiting...')
+          cleanup
+          exit 1
+        end
+
+        # The end!
+        break if batch_end >= max
+
+        # Throttle
+        current_time = Time.now
+        if (current_time - start_batch_time) > DECREASE_THROTTLER
+          batch_size = [(batch_size - DECREASE_SIZE).to_i, MIN_BATCH_SIZE].max
+        end
+
+        # Periodically show progress
+        if (current_time - last_progress) >= progress_every
+          last_progress = current_time
+          done = batch_end - min + 1
+          write(table_name, "%.2f%% (#{done}/#{todo})" % (done.to_f / todo * 100.0))
+        end
+      end
+
+      duration = Time.now - start_time
+      duration = (duration < 2*60) ? "#{duration.to_i} secs" : "#{(duration / 60).to_i} mins"
+      write(table_name, "Completed (#{duration})")
+    end
+
+    private
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+
+    def execute(stmt)
+      connection.execute(stmt)
+    end
+
+    def write(table_name, msg)
+      return if Zdm.io == false
+      io = Zdm.io || $stderr
+      io.puts("#{table_name}: #{msg}")
+      io.flush
+    end
   end
 
   class Table
@@ -80,7 +147,7 @@ module Zdm
       drop_copy_indexes
       apply_ddl_statements
       create_triggers
-      batched_copy
+      copy_in_batches
       create_copy_indexes
       atomic_switcharoo!
     ensure
@@ -180,7 +247,7 @@ module Zdm
         "zdmt_#{trigger_type}_#{table.origin}"[0...64]
       end
 
-      # Drop indexes to speed up batched_copy.
+      # Drop indexes to speed up copy_in_batches.
       #
       # "Online DDL support for adding secondary indexes means that you can
       # generally speed the overall process of creating and loading a table
@@ -215,18 +282,7 @@ module Zdm
         execute('ALTER TABLE `%s` %s' % [table.copy, indexes.join(', ')])
       end
 
-      BATCH_SIZE = 40_000
-      DECREASE_THROTTLER = 4  # seconds
-      DECREASE_SIZE = 5_000
-      MIN_BATCH_SIZE = 10_000
-      PROGRESS_EVERY = 30  # seconds
-      def batched_copy
-        min = connection.select_value('SELECT MIN(`id`) FROM %s' % table.origin)
-        return unless min
-
-        max = connection.select_value('SELECT MAX(`id`) FROM %s' % table.origin)
-        todo = max - min + 1
-
+      def copy_in_batches
         insert_columns = common_columns.map {|c| "`#{c}`"}.join(', ')
         select_columns = common_columns.map {|c| "`#{table.origin}`.`#{c}`"}.join(', ')
         sql = <<-SQL.squish
@@ -236,50 +292,11 @@ module Zdm
           WHERE `#{table.origin}`.`id` BETWEEN %s AND %s
         SQL
 
-        batch_size = BATCH_SIZE
-        batch_end = min - 1
-        start_time = last_progress = Time.now
-        while true
-          batch_start = batch_end + 1
-          batch_end = [batch_start + batch_size - 1, max].min
-          start_batch_time = Time.now
-
-          execute(sql % [batch_start, batch_end])
-
-          if $exit
-            write('Received SIGTERM, exiting...')
-            cleanup
-            exit 1
-          end
-
-          # The end!
-          break if batch_end >= max
-
-          # Throttle
-          current_time = Time.now
-          if (current_time - start_batch_time) > DECREASE_THROTTLER
-            batch_size = [(batch_size - DECREASE_SIZE).to_i, MIN_BATCH_SIZE].max
-          end
-
-          # Periodically show progress
-          if (current_time - last_progress) >= PROGRESS_EVERY
-            last_progress = current_time
-            done = batch_end - min + 1
-            write("%.2f%% (#{done}/#{todo})" % (done.to_f / todo * 100.0))
-          end
+        Zdm.execute_in_batches(table.origin) do |batch_start, batch_end|
+          sql % [batch_start, batch_end]
         end
-
-        duration = Time.now - start_time
-        duration = (duration < 2*60) ? "#{duration.to_i} secs" : "#{(duration / 60).to_i} mins"
-        write("Completed (#{duration})")
       end
 
-      def write(msg)
-        return if Zdm.io == false
-        io = Zdm.io || $stderr
-        io.puts("#{table.origin}: #{msg}")
-        io.flush
-      end
   end
 end
 trap('TERM') { $exit = true }

data/spec/spec_helper.rb

@@ -34,8 +34,5 @@ ActiveRecord::Schema.define version: 0 do
   end
 end
 
-ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'foo','bar','2017-03-01 23:59:59')])
-ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (20,'foo2','bar2','2017-03-02 23:59:59')])
-
 # ActiveRecord::Base.logger = Logger.new($stdout)
 

data/spec/zdm_spec.rb

@@ -5,6 +5,11 @@ describe Zdm do
   before(:example) {
     Zdm.io = false
     Zdm.cleanup
+
+    conn = ActiveRecord::Base.connection
+    conn.execute(%[TRUNCATE people])
+    conn.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'foo','bar','2017-03-01 23:59:59')])
+    conn.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (20,'foo2','bar2','2017-03-02 23:59:59')])
   }
 
   it 'requires an autoincrement primary key `id` field' do
@@ -87,4 +92,49 @@ describe Zdm do
     expect(archive_tables.length).to eq(2)
   end
 
+  context 'execute_in_batches' do
+    before(:example) do
+      @conn = ActiveRecord::Base.connection
+      (1..20).each do |idx|
+        @conn.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'person-#{idx}','P#{idx}','2017-03-01 23:59:59')])
+      end
+      Zdm.io = StringIO.new
+      @sql = "UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN %s AND %s"
+    end
+
+    after(:example) do
+      @conn.execute(%[DELETE FROM people WHERE name LIKE 'person%'])
+    end
+
+    it 'updates a table in batches' do
+      Zdm.execute_in_batches('people', batch_size: 4, progress_every: 1) do |batch_start, batch_end|
+        sleep(0.6)
+        @sql % [batch_start, batch_end]
+      end
+      expect(Zdm.io.string).to eq(%[people: 36.36% (8/22)\npeople: 72.73% (16/22)\npeople: Completed (3 secs)\n])
+      expect(@conn.select_value(%[SELECT COUNT(*) FROM people WHERE code LIKE '%U'])).to eq(22)
+    end
+
+    it 'updates part of a table in batches' do
+      batches = []
+      Zdm.execute_in_batches('people', start: 5, finish: 18, batch_size: 4, progress_every: 1) do |batch_start, batch_end|
+        sleep(0.6)
+        batches << @sql % [batch_start, batch_end]
+        @sql % [batch_start, batch_end]
+      end
+      expect(Zdm.io.string).to eq(%[people: 57.14% (8/14)\npeople: Completed (2 secs)\n])
+      expect(batches).to eq([
+        %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 5 AND 8],
+        %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 9 AND 12],
+        %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 13 AND 16],
+        %[UPDATE people SET code = CONCAT(code, 'U') WHERE id BETWEEN 17 AND 18],
+      ])
+      expect(@conn.select_values(%[SELECT code FROM people WHERE code LIKE '%U'])).to eq([
+        'P3U', 'P4U', 'P5U', 'P6U', 'P7U', 'P8U', 'P9U', 'P10U', 'P11U', 'P12U', 'P13U', 'P14U', 'P15U', 'P16U'
+      ])
+    end
+
+  end
+
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zdm
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - ITRP Institute, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-23 00:00:00.000000000 Z
+date: 2017-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord