cleansweep 1.0.6 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8ec1d586978900f85eff3d161db3a7228829272d
-  data.tar.gz: c93186b039b2c9bdf3b08d69d73e237effdde631
+  metadata.gz: 9720deef8bdeca62cd0b483a7d0dc27587667d29
+  data.tar.gz: 6655283cf1ae8df51bb054643ba2f0e78dd1a871
 SHA512:
-  metadata.gz: b8089e971691d784346b1572ac0235bc2aa25866415ac91bada9db10bdf1c1fd2682e724148decbbf72d946ee20f4e4a05f42f90a699da11d037b8a236059cd8
-  data.tar.gz: 67cdf88125ece554602df476975702ef0898e0d47de70ed05bbc56296a53d506d42b505b028ef7ea3340bb217d928b6c3c6bcbd188d4792f0ce5ae9593cfbba2
+  metadata.gz: e4c81408490d26080503edcadf0451f6600e55ad404eea339b4d363a05289958a7f1f42983a5dd0f5974cd240d7b8be1caa7916ba6a8ea9b320a8a6feacb51a6
+  data.tar.gz: f0c5777dd4b362b9b6c790c74c38022e97dbce1faea43897f17d8d9348b4e2852bf3cad658f2fbb8e7a2344b9837ee77eea7202bebd37cd3e2b55f8b16c34bea

data/.travis.yml

@@ -1,10 +1,11 @@
 language: ruby
 rvm:
   - 2.1.4
-  - 1.9.3
 gemfile:
   - gemfiles/Gemfile.rails3
   - gemfiles/Gemfile.rails4
 addons:
   code_climate:
     repo_token: 7ec6fd701b7d2b206cdd233c2202b6e11c8ba6af01f8a93f5e24595008ac20a0
+after_success:
+  - bundle exec codeclimate-test-reporter

data/CHANGES.md

@@ -1,4 +1,9 @@
 See the [documentation](http://bkayser.github.io/cleansweep) for details
+### Version 1.1.0
+
+* Support automatic DB reconnection during a purge run.
+  The max number of reconnections can be controlled with the max_reconnects
+  option to PurgeRunner.
 
 ### Version 1.0.6
 

data/cleansweep.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^spec/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency 'activerecord', '>= 3.0'
+  spec.add_runtime_dependency 'activerecord', '>= 3.0', '< 5.0.0'
   spec.add_runtime_dependency 'newrelic_rpm'
   spec.add_runtime_dependency 'mysql2', '~> 0.3'
 

data/lib/clean_sweep/purge_runner.rb

@@ -74,6 +74,10 @@ require 'stringio'
 # [:max_repl_lag]
 #    The maximum length of the replication lag.  Checked every 5 minutes and if exceeded the purge
 #    pauses until the replication lag is below 90% of this value.
+# [:max_reconnects]
+#    The maximum number of times to automatically attempt to reconnect to the database
+#    in the event of a dropped connection during a query. Set this to zero if you want
+#    to opt-out of the automatic reconnect behavior. Default: 3.
 
 class CleanSweep::PurgeRunner
 
@@ -99,6 +103,8 @@ class CleanSweep::PurgeRunner
 
     @max_history      = options[:max_history]
     @max_repl_lag     = options[:max_repl_lag]
+    @max_reconnects   = options[:max_reconnects] || 3
+    @reconnects_remaining = @max_reconnects
 
     @copy_mode        = @target_model && options[:copy_only]
 
@@ -136,6 +142,27 @@ class CleanSweep::PurgeRunner
     @copy_mode
   end
 
+  def with_reconnect_handling
+    begin
+      yield
+    rescue ActiveRecord::StatementInvalid => e
+      if e.message =~ /Lost connection to MySQL server during query/
+        if @reconnects_remaining > 0
+          @reconnects_remaining -= 1
+          wait_before_reconnect = rand * 10.0
+          log :warn, "Lost connection to DB during query, reconnecting and trying again in #{wait_before_reconnect} seconds. #{@reconnects_remaining} re-connection attempts remaining after this. Original error: #{e}"
+          sleep(wait_before_reconnect)
+          @model.connection.reconnect!
+        else
+          log :error, "Lost connection to MySQL during query, and have already reconnected #{@max_reconnects} times. Giving up. Original error: #{e}"
+          raise
+        end
+      else
+        raise
+      end
+    end
+  end
+
   # Execute the purge in chunks according to the parameters given on instance creation.
   # Will raise <tt>CleanSweep::PurgeStopped</tt> if a <tt>stop_after</tt> option was provided and
   # that limit is hit.
@@ -165,6 +192,7 @@ class CleanSweep::PurgeRunner
     rows = NewRelic::Agent.with_database_metric_name(@model.name, 'SELECT') do
       @model.connection.select_rows @query.to_sql
     end
+
     while rows.any? && (!@stop_after || @total_deleted < @stop_after) do
 #      index_entrypoint_args = Hash[*@source_keys.zip(rows.last).flatten]
       log :debug, "#{verb} #{rows.size} records between #{rows.first.inspect} and #{rows.last.inspect}" if @logger.level == Logger::DEBUG
@@ -180,24 +208,30 @@ class CleanSweep::PurgeRunner
         statement = @table_schema.delete_statement(rows)
       end
       log :debug, statement if @logger.level == Logger::DEBUG
-      chunk_deleted = NewRelic::Agent.with_database_metric_name((@target_model||@model), metric_op_name) do
-        (@target_model||@model).connection.update statement
-      end
 
-      @total_deleted += chunk_deleted
-      raise CleanSweep::PurgeStopped.new("stopped after #{verb} #{@total_deleted} #{@model} records", @total_deleted) if stopped
-      q = @table_schema.scope_to_next_chunk(@query, last_row).to_sql
-      log :debug, "find rows: #{q}" if @logger.level == Logger::DEBUG
+      with_reconnect_handling do
+        chunk_deleted = NewRelic::Agent.with_database_metric_name((@target_model||@model), metric_op_name) do
+          (@target_model||@model).connection.update statement
+        end
+
+        @total_deleted += chunk_deleted
+        raise CleanSweep::PurgeStopped.new("stopped after #{verb} #{@total_deleted} #{@model} records", @total_deleted) if stopped
+        q = @table_schema.scope_to_next_chunk(@query, last_row).to_sql
+        log :debug, "find rows: #{q}" if @logger.level == Logger::DEBUG
 
-      sleep @sleep if @sleep && !copy_mode?
-      @mysql_status.check! if @mysql_status
+        sleep @sleep if @sleep && !copy_mode?
+        @mysql_status.check! if @mysql_status
 
-      rows = NewRelic::Agent.with_database_metric_name(@model, 'SELECT') do
-        @model.connection.select_rows(q)
+        rows = NewRelic::Agent.with_database_metric_name(@model, 'SELECT') do
+          @model.connection.select_rows(q)
+        end
       end
+
       report
     end
+
     report(true)
+
     if copy_mode?
       log :info,  "completed after #{verb} #{@total_deleted} #{@table_schema.name} records to #{@target_model.table_name}"
     else

data/lib/clean_sweep/version.rb

@@ -1,3 +1,3 @@
 module CleanSweep
-  VERSION = "1.0.6"
+  VERSION = "1.1.0"
 end

data/spec/purge_runner_spec.rb

@@ -7,6 +7,7 @@ describe CleanSweep::PurgeRunner do
     before do
       Timecop.freeze Time.parse("2014-12-02 13:47:43.000000 -0800")
     end
+
     after do
       Timecop.return
     end
@@ -135,6 +136,82 @@ EOF
         Book.delete_all
       end
 
+      it 'reconnects after a lost connection' do
+        purger = CleanSweep::PurgeRunner.new model: Book,
+                                             chunk_size: 10
+
+        original_update = Book.connection.method(:update)
+        update_number = 0
+
+        allow(Book.connection).to receive(:update) do |*args|
+          update_number += 1
+
+          if update_number == 2
+            raise ActiveRecord::StatementInvalid.new("Lost connection to MySQL server during query: blah blah")
+          else
+            original_update.call(*args)
+          end
+        end
+
+        expect(purger).to receive(:sleep).once
+        expect(Book.connection).to receive(:reconnect!).once
+
+        purger.execute_in_batches
+
+        expect(Book.count).to eq(0)
+      end
+
+      it 'reconnects after a lost connection during select' do
+        purger = CleanSweep::PurgeRunner.new model: Book,
+                                             chunk_size: 10
+
+        original_select_rows = Book.connection.method(:select_rows)
+        iteration = 0
+
+        allow(Book.connection).to receive(:select_rows) do |*args|
+          iteration += 1
+
+          if iteration == 2
+            raise ActiveRecord::StatementInvalid.new("Lost connection to MySQL server during query: blah blah")
+          else
+            original_select_rows.call(*args)
+          end
+        end
+
+        expect(purger).to receive(:sleep).once
+        expect(Book.connection).to receive(:reconnect!).once
+
+        purger.execute_in_batches
+
+        expect(Book.count).to eq(0)
+      end
+
+      it 'stops trying to reconnect after max_reconnects' do
+        purger = CleanSweep::PurgeRunner.new model: Book,
+                                             chunk_size: 10,
+                                             max_reconnects: 4
+
+        original_update = Book.connection.method(:update)
+        update_number = 0
+
+        allow(Book.connection).to receive(:update) do |*args|
+          update_number += 1
+          if update_number > 1
+            raise ActiveRecord::StatementInvalid.new("Lost connection to MySQL server during query: blah blah")
+          else
+            original_update.call(*args)
+          end
+        end
+
+        expect(purger).to receive(:sleep).exactly(4).times
+        expect(Book.connection).to receive(:reconnect!).exactly(4).times
+
+        expect { purger.execute_in_batches }.to raise_error(ActiveRecord::StatementInvalid)
+
+        # we only got through the first batch of 10, and then gave up
+        expect(Book.count).to eq(40)
+      end
+
       it 'waits for history' do
         purger = CleanSweep::PurgeRunner.new model: Book,
                                              max_history: 100,
@@ -209,10 +286,11 @@ EOF
         count = purger.execute_in_batches
         expect(count).to be(@total_book_size)
         expect(BookTemp.count).to eq(@total_book_size)
-        last_book = BookTemp.last
-        expect(last_book.book_id).to be 200
-        expect(last_book.bin).to be 2000
-        expect(last_book.published_by).to eq 'Random House'
+        last_book = Book.last
+        last_book_copy = BookTemp.last
+        expect(last_book_copy.book_id).to eq(last_book.id)
+        expect(last_book_copy.bin).to eq(last_book.bin)
+        expect(last_book_copy.published_by).to eq(last_book.publisher)
       end
 
     end
@@ -234,12 +312,16 @@ describe CleanSweep::PurgeRunner::MysqlStatus do
     it "fetches innodb status" do
       mysql_status.get_replication_lag
     end
+
     it "checks history and pauses" do
       allow(mysql_status).to receive(:get_history_length).and_return(101, 95, 89)
+      allow(mysql_status).to receive(:get_replication_lag).and_return(50)
       expect(mysql_status).to receive(:pause).twice
       mysql_status.check!
     end
+
     it "checks replication and pauses" do
+      allow(mysql_status).to receive(:get_history_length).and_return(50)
       allow(mysql_status).to receive(:get_replication_lag).and_return(101, 95, 89)
       expect(mysql_status).to receive(:pause).twice
       mysql_status.check!

data/spec/spec_helper.rb

@@ -1,7 +1,8 @@
 ENV['RACK_ENV'] = 'test'
 
-require "codeclimate-test-reporter"
-CodeClimate::TestReporter.start
+require 'simplecov'
+SimpleCov.start
+
 require 'clean_sweep'
 require 'factory_girl'
 require 'fileutils'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cleansweep
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.1.0
 platform: ruby
 authors:
 - Bill Kayser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-18 00:00:00.000000000 Z
+date: 2017-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -17,6 +17,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -24,6 +27,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: newrelic_rpm
   requirement: !ruby/object:Gem::Requirement
@@ -205,7 +211,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Utility to purge or archive rows in mysql tables