perfectqueue 0.8.50 → 0.8.51

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f2a4de693be6fb67cec8c761d949996d4aab365
-  data.tar.gz: 9ae56885f1e52d7e5e685e754ae3ec971d0abc05
+  metadata.gz: a2c5312dd385fbd06f7c00fc52c5661878de5d0e
+  data.tar.gz: c73dfc2576ef192056f9e09810c95bbd5f57640a
 SHA512:
-  metadata.gz: 9cf308fbeaced67ee174de205e12d443ae96fcd695406c217cd6e254fcdba31632850ec734cab875126cb79e474b90642bde7a47b8a821665750f7e72bc4e0f4
-  data.tar.gz: c95228db876ba2427b1b2b288abd76cb245583afabaaadb827bdd29bffa84cfb4a8830a6e6f5a0058b6c068882cfe9b8556102d3c959b3dd60c8b9c5253106a1
+  metadata.gz: eadcea4ea3b019cb9a76b9ec949d3d7eb7eb439479dbe8ced5a746d2c8e8fbb5f4e434307f84ee966e0f216c48a34c9fb927167ef51563fc593e12567fc64806
+  data.tar.gz: 563da9974f2d64efa75c5c515da14f9b45a794fec83d9ef4c95941f90b89ddeb99678464f43fa2683e1fa404a57e056ab22923b40a53ea55f3b735ab5a192c79

data/ChangeLog

@@ -1,3 +1,7 @@
+== 2016-10-24 version 0.8.51
+
+* Retry on temporary DatabaseConnectionError (#59)
+
 == 2016-08-18 version 0.8.50
 
 * Decrease GET_LOCK timeout from 60 to 10 seconds

data/lib/perfectqueue/backend/rdb.rb

@@ -21,6 +21,8 @@ module PerfectQueue::Backend
         host: u.host,
         port: u.port ? u.port.to_i : 3306
       }
+      @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
+      options[:connect_timeout] = config.fetch(:connect_timeout, 3)
       options[:sslca] = config[:sslca] if config[:sslca]
       db_name = u.path.split('/')[1]
       @db = Sequel.mysql2(db_name, options)
@@ -54,23 +56,31 @@ module PerfectQueue::Backend
     end
 
     private
-    def connect(&block)
+    def connect
+      tmax = Process.clock_gettime(Process::CLOCK_REALTIME, :second) + @pq_connect_timeout
       @mutex.synchronize do
         retry_count = 0
         begin
-          block.call
+          yield
+        rescue Sequel::DatabaseConnectionError
+          if (retry_count += 1) < MAX_RETRY && tmax > Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+            STDERR.puts "#{$!}\n  retrying."
+            sleep 2
+            retry
+          end
+          STDERR.puts "#{$!}\n  abort."
+          raise
         rescue
           # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
           if $!.to_s.include?('try restarting transaction')
-            err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
+            err = $!.backtrace.map{|bt| "  #{bt}" }.unshift($!).join("\n")
             retry_count += 1
             if retry_count < MAX_RETRY
-              STDERR.puts err + "\n  retrying."
+              STDERR.puts "#{err}\n  retrying."
               sleep 0.5
               retry
-            else
-              STDERR.puts err + "\n  abort."
             end
+            STDERR.puts "#{err}\n  abort."
           end
           raise
         ensure

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -42,6 +42,7 @@ module PerfectQueue
       def initialize(client, config)
         super
 
+        @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
         url = config[:url]
         @table = config[:table]
         unless @table
@@ -49,7 +50,9 @@ module PerfectQueue
         end
 
         if /\Amysql2:/i =~ url
-          @db = Sequel.connect(url, {max_connections: 1, sslca: config[:sslca]})
+          options = {max_connections: 1, sslca: config[:sslca]}
+          options[:connect_timeout] = config.fetch(:connect_timeout, 3)
+          @db = Sequel.connect(url, options)
           if config.fetch(:use_connection_pooling, nil) != nil
             @use_connection_pooling = !!config[:use_connection_pooling]
           else
@@ -339,7 +342,7 @@ SQL
       end
 
       protected
-      def connect_locked(&block)
+      def connect_locked
         connect {
           locked = false
 
@@ -349,7 +352,7 @@ SQL
               locked = true
             end
 
-            return block.call
+            return yield
           ensure
             if @use_connection_pooling && locked
               @table_unlock.call
@@ -358,16 +361,25 @@ SQL
         }
       end
 
-      def connect(&block)
+      def connect
         now = Time.now.to_i
+        tmax = now + @pq_connect_timeout
         @mutex.synchronize do
           # keepalive_timeout
           @db.disconnect if now - @last_time > KEEPALIVE
 
           count = 0
           begin
-            block.call
+            yield
             @last_time = now
+          rescue Sequel::DatabaseConnectionError
+            if (count += 1) < MAX_RETRY && tmax > Time.now.to_i
+              STDERR.puts "#{$!}\n  retrying."
+              sleep 2
+              retry
+            end
+            STDERR.puts "#{$!}\n  abort."
+            raise
           rescue
             # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
             if $!.to_s.include?('try restarting transaction')

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.50"
+  VERSION = "0.8.51"
 end

data/spec/rdb_backend_spec.rb

@@ -65,5 +65,19 @@ describe Backend::RDBBackend do
         end.to raise_error(RuntimeError)
       end
     end
+    context 'cannot connect' do
+      let (:uri){ 'mysql2://root:@nonexistent/perfectqueue_test' }
+      let (:db) do
+        Backend::RDBBackend.new(uri, table)
+      end
+      it 'raises Sequel::DatabaseConnectionError' do
+        allow(STDERR).to receive(:puts)
+        slept = 0
+        expect(db).to receive(:sleep).exactly(9).times{|n| slept += n }
+        expect(db.db).to receive(:connect).exactly(10).times.and_call_original
+        expect{ db.__send__(:connect){ db.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
+        expect(slept).to be < 30
+      end
+    end
   end
 end

data/spec/rdb_compat_backend_spec.rb

@@ -376,6 +376,18 @@ describe Backend::RDBCompatBackend do
           end
         end.to raise_error(RuntimeError)
       end
+      context 'cannot connect' do
+        let (:config){ {url: 'mysql2://root:@nonexistent/perfectqueue_test', table: table} }
+        it 'raises Sequel::DatabaseConnectionError' do
+          allow(STDERR).to receive(:puts)
+          d = Backend::RDBCompatBackend.new(client, config)
+          slept = 0
+          expect(d).to receive(:sleep).exactly(9).times{|n| slept += n }
+          expect(d.db).to receive(:connect).exactly(10).times.and_call_original
+          expect{ d.__send__(:connect){ d.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
+          expect(slept).to eq(18)
+        end
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.50
+  version: 0.8.51
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-17 00:00:00.000000000 Z
+date: 2016-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel