perfectqueue 0.8.49 → 0.8.54

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61295f3241d66b1d9f27aac68294e32344b09611
-  data.tar.gz: 36371652be7e64c3d5c1b79396c37ff8f5e5344c
+  metadata.gz: ed8791f086ecef1178ae6d001dc4addeef32a14f
+  data.tar.gz: 342eb4a9e95c7a5fdcc29c909f1f7a99c1e236ea
 SHA512:
-  metadata.gz: 2461e220ce8b2f45308db2f377b0816718c4222bc70bcdefec5b6c6fe2df0b3938b7e5d5ff87ad2888b7f20133c91d132a333aa9c0578928a231cb9e8f72b312
-  data.tar.gz: ba6542077acbf7e914c22a90afbcd54833d6e06e42781e8d62300b8d74cda1dfdf29eb5f7bfcb90f3788547828bc79c9cf61e74676127f29b879b2b00f8878d9
+  metadata.gz: 4ca28812f77cfad84f2f4198de80093b41b46ae26f8ebf0fc13fe8054c785e7e24f766baf64839af1a4c0fadc8f38073e5b1abd7c962ad97a32f859a2b5b237e
+  data.tar.gz: 13f80b70c03b3a43538f7f687869db8918eb5a319ce7fc2503963d21a0f4fd5fa3f3a1d0163f6e19424c0001061f2b049e80287261fd230a3628e5cae1fd4f76

data/.circleci/config.yml

@@ -0,0 +1,19 @@
+version: 2.1
+
+executors:
+  executor-circle:
+    docker:
+      - image: buildpack-deps:jessie
+    working_directory: /tmp
+
+jobs:
+  build:
+    executor: executor-circle
+    steps:
+      - run: echo "It has used Travis instead of CircleCI."
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - build

data/.travis.yml

@@ -1,9 +1,13 @@
 rvm:
-  - 2.1.10
-  - 2.2.5
-  - 2.3.1
+  - 2.3.8
+  - 2.4.6
+  - 2.5.5
+  - 2.6.3
   - ruby-head
 
+services:
+  - mysql
+
 script: "bundle exec rake spec"
 
 before_script:

data/ChangeLog

@@ -1,3 +1,24 @@
+== 2020-09-10 version 0.8.54
+
+* Enhance logging on inter-process communication (#68)
+
+== 2019-10-10 version 0.8.53
+
+* Extract max_retry_count as config parameter (#66)
+
+== 2019-07-31 version 0.8.52
+
+* Add sleep lock retrying with exponential-backoff (#64)
+
+== 2016-10-24 version 0.8.51
+
+* Retry on temporary DatabaseConnectionError (#59)
+
+== 2016-08-18 version 0.8.50
+
+* Decrease GET_LOCK timeout from 60 to 10 seconds
+* Add 0.5 to 30 seconds of sleep between GET_LOCK retrying
+
 == 2016-08-02 version 0.8.49
 
 * Revert v0.8.44 migration path (#38)

data/lib/perfectqueue/backend/rdb.rb

@@ -4,7 +4,6 @@ require_relative 'rdb_compat'
 
 module PerfectQueue::Backend
   class RDBBackend
-    MAX_RETRY = ::PerfectQueue::Backend::RDBCompatBackend::MAX_RETRY
     DELETE_OFFSET = ::PerfectQueue::Backend::RDBCompatBackend::DELETE_OFFSET
     class Token < Struct.new(:key)
     end
@@ -21,6 +20,9 @@ module PerfectQueue::Backend
         host: u.host,
         port: u.port ? u.port.to_i : 3306
       }
+      @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
+      @max_retry_count = config.fetch(:max_retry_count, 10)
+      options[:connect_timeout] = config.fetch(:connect_timeout, 3)
       options[:sslca] = config[:sslca] if config[:sslca]
       db_name = u.path.split('/')[1]
       @db = Sequel.mysql2(db_name, options)
@@ -32,6 +34,7 @@ module PerfectQueue::Backend
     end
 
     attr_reader :db
+    attr_reader :max_retry_count
 
     def submit(id, data, time=Process.clock_gettime(Process::CLOCK_REALTIME, :second), resource=nil, max_running=nil)
       connect {
@@ -54,23 +57,31 @@ module PerfectQueue::Backend
     end
 
     private
-    def connect(&block)
+    def connect
+      tmax = Process.clock_gettime(Process::CLOCK_REALTIME, :second) + @pq_connect_timeout
       @mutex.synchronize do
         retry_count = 0
         begin
-          block.call
+          yield
+        rescue Sequel::DatabaseConnectionError
+          if (retry_count += 1) < @max_retry_count && tmax > Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+            STDERR.puts "#{$!}\n  retrying."
+            sleep 2
+            retry
+          end
+          STDERR.puts "#{$!}\n  abort."
+          raise
         rescue
           # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
           if $!.to_s.include?('try restarting transaction')
-            err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
+            err = $!.backtrace.map{|bt| "  #{bt}" }.unshift($!).join("\n")
             retry_count += 1
-            if retry_count < MAX_RETRY
-              STDERR.puts err + "\n  retrying."
+            if retry_count < @max_retry_count
+              STDERR.puts "#{err}\n  retrying."
               sleep 0.5
               retry
-            else
-              STDERR.puts err + "\n  abort."
             end
+            STDERR.puts "#{err}\n  abort."
           end
           raise
         ensure

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -33,12 +33,17 @@ module PerfectQueue
       DELETE_OFFSET = 10_0000_0000
       EVENT_HORIZON = 13_0000_0000 # 2011-03-13 07:06:40 UTC
 
+      LOCK_RETRY_INITIAL_INTERVAL = 0.5
+      LOCK_RETRY_MAX_INTERVAL = 30
+
       class Token < Struct.new(:key)
       end
 
       def initialize(client, config)
         super
 
+        @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
+        @max_retry_count = config.fetch(:max_retry_count, 10)
         url = config[:url]
         @table = config[:table]
         unless @table
@@ -46,7 +51,9 @@ module PerfectQueue
         end
 
         if /\Amysql2:/i =~ url
-          @db = Sequel.connect(url, {max_connections: 1, sslca: config[:sslca]})
+          options = {max_connections: 1, sslca: config[:sslca]}
+          options[:connect_timeout] = config.fetch(:connect_timeout, 3)
+          @db = Sequel.connect(url, options)
           if config.fetch(:use_connection_pooling, nil) != nil
             @use_connection_pooling = !!config[:use_connection_pooling]
           else
@@ -54,11 +61,14 @@ module PerfectQueue
           end
           @table_lock = lambda {
             locked = nil
+            interval = LOCK_RETRY_INITIAL_INTERVAL
             loop do
               @db.fetch("SELECT GET_LOCK('#{@table}', #{LOCK_WAIT_TIMEOUT}) locked") do |row|
                 locked = true if row[:locked] == 1
               end
               break if locked
+              sleep interval
+              interval = [interval * 2, LOCK_RETRY_MAX_INTERVAL].min
             end
           }
           @table_unlock = lambda {
@@ -112,10 +122,10 @@ SQL
       end
 
       attr_reader :db
+      attr_reader :max_retry_count
 
       KEEPALIVE = 10
-      MAX_RETRY = 10
-      LOCK_WAIT_TIMEOUT = 60
+      LOCK_WAIT_TIMEOUT = 10
       DEFAULT_DELETE_INTERVAL = 20
 
       def init_database(options)
@@ -333,7 +343,7 @@ SQL
       end
 
       protected
-      def connect_locked(&block)
+      def connect_locked
         connect {
           locked = false
 
@@ -343,7 +353,7 @@ SQL
               locked = true
             end
 
-            return block.call
+            return yield
           ensure
             if @use_connection_pooling && locked
               @table_unlock.call
@@ -352,22 +362,31 @@ SQL
         }
       end
 
-      def connect(&block)
+      def connect
         now = Time.now.to_i
+        tmax = now + @pq_connect_timeout
         @mutex.synchronize do
           # keepalive_timeout
           @db.disconnect if now - @last_time > KEEPALIVE
 
           count = 0
           begin
-            block.call
+            yield
             @last_time = now
+          rescue Sequel::DatabaseConnectionError
+            if (count += 1) < @max_retry_count && tmax > Time.now.to_i
+              STDERR.puts "#{$!}\n  retrying."
+              sleep 2
+              retry
+            end
+            STDERR.puts "#{$!}\n  abort."
+            raise
           rescue
             # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
             if $!.to_s.include?('try restarting transaction')
               err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
               count += 1
-              if count < MAX_RETRY
+              if count < @max_retry_count
                 STDERR.puts err + "\n  retrying."
                 sleep rand
                 retry

data/lib/perfectqueue/multiprocess/child_process_monitor.rb

@@ -102,8 +102,8 @@ module PerfectQueue
       def send_signal(sig)
         begin
           Process.kill(sig, @pid)
-        rescue Errno::ESRCH, Errno::EPERM
-          # TODO log?
+        rescue Errno::ESRCH, Errno::EPERM => e
+          @log.info "#{e.class}: #{e.message}\n#{e.backtrace}"
         end
       end
 

data/lib/perfectqueue/multiprocess/thread_processor.rb

@@ -121,7 +121,7 @@ module PerfectQueue
       end
 
       def process(task)
-        @log.info "acquired task task=#{task.key} id=#{@processor_id}: #{task.inspect}"
+        @log.info "acquired task task=#{task.key} id=#{@processor_id}"
         begin
           r = @runner.new(task)
           @tm.set_task(task, r)

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.49"
+  VERSION = "0.8.54"
 end

data/lib/perfectqueue/worker.rb

@@ -28,6 +28,7 @@ module PerfectQueue
       config = block.call
 
       @config = config
+      @log = config[:logger] || Logger.new(STDERR)
       @runner = runner
 
       @detach_wait = config[:detach_wait] || config['detach_wait'] || 10.0
@@ -66,9 +67,11 @@ module PerfectQueue
 
         else
           # child process finished unexpectedly
+          @log.info "Child process finished unexpectedly pid=#{pid}"
         end
 
-      rescue Errno::ECHILD
+      rescue Errno::ECHILD => e
+        @log.info "#{e.class}: #{e.message}\n#{e.backtrace}"
       end
     end
 
@@ -130,4 +133,3 @@ module PerfectQueue
   end
 
 end
-

data/spec/rdb_backend_spec.rb

@@ -55,7 +55,7 @@ describe Backend::RDBBackend do
     end
     context 'error' do
       it 'returns block result' do
-        expect(RuntimeError).to receive(:new).exactly(Backend::RDBBackend::MAX_RETRY).and_call_original
+        expect(RuntimeError).to receive(:new).exactly(db.max_retry_count).and_call_original
         allow(STDERR).to receive(:puts)
         allow(db).to receive(:sleep)
         expect do
@@ -65,5 +65,19 @@ describe Backend::RDBBackend do
         end.to raise_error(RuntimeError)
       end
     end
+    context 'cannot connect' do
+      let (:uri){ 'mysql2://root:@nonexistent/perfectqueue_test' }
+      let (:db) do
+        Backend::RDBBackend.new(uri, table)
+      end
+      it 'raises Sequel::DatabaseConnectionError' do
+        allow(STDERR).to receive(:puts)
+        slept = 0
+        expect(db).to receive(:sleep).exactly(9).times{|n| slept += n }
+        expect(db.db).to receive(:connect).exactly(10).times.and_call_original
+        expect{ db.__send__(:connect){ db.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
+        expect(slept).to be < 30
+      end
+    end
   end
 end

data/spec/rdb_compat_backend_spec.rb

@@ -367,7 +367,7 @@ describe Backend::RDBCompatBackend do
     end
     context 'error' do
       it 'returns block result' do
-        expect(RuntimeError).to receive(:new).exactly(Backend::RDBCompatBackend::MAX_RETRY).and_call_original
+        expect(RuntimeError).to receive(:new).exactly(db.max_retry_count).and_call_original
         allow(STDERR).to receive(:puts)
         allow(db).to receive(:sleep)
         expect do
@@ -376,6 +376,18 @@ describe Backend::RDBCompatBackend do
           end
         end.to raise_error(RuntimeError)
       end
+      context 'cannot connect' do
+        let (:config){ {url: 'mysql2://root:@nonexistent/perfectqueue_test', table: table} }
+        it 'raises Sequel::DatabaseConnectionError' do
+          allow(STDERR).to receive(:puts)
+          d = Backend::RDBCompatBackend.new(client, config)
+          slept = 0
+          expect(d).to receive(:sleep).exactly(9).times{|n| slept += n }
+          expect(d.db).to receive(:connect).exactly(10).times.and_call_original
+          expect{ d.__send__(:connect){ d.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
+          expect(slept).to eq(18)
+        end
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.49
+  version: 0.8.54
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-02 00:00:00.000000000 Z
+date: 2020-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -88,6 +88,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".travis.yml"
 - ChangeLog
@@ -98,7 +99,6 @@ files:
 - Rakefile
 - bin/perfectqueue
 - bin/stress
-- circle.yml
 - lib/perfectqueue.rb
 - lib/perfectqueue/application.rb
 - lib/perfectqueue/application/base.rb
@@ -177,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: Highly available distributed cron built on RDBMS
@@ -208,4 +208,3 @@ test_files:
 - spec/task_metadata_spec.rb
 - spec/task_monitor_spec.rb
 - spec/task_spec.rb
-has_rdoc: false

data/circle.yml

@@ -1,7 +0,0 @@
-machine:
-  ruby:
-    version: 2.2.2
-
-database:
-  pre:
-    - mysql -e 'create database perfectqueue_test;'