perfectqueue 0.8.54 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed8791f086ecef1178ae6d001dc4addeef32a14f
-  data.tar.gz: 342eb4a9e95c7a5fdcc29c909f1f7a99c1e236ea
+  metadata.gz: 12beb7e5ec6143447d1fbf9a973fa8cb9305c5a0
+  data.tar.gz: 782ca440e9554af177adaf7607b8306a57b0612c
 SHA512:
-  metadata.gz: 4ca28812f77cfad84f2f4198de80093b41b46ae26f8ebf0fc13fe8054c785e7e24f766baf64839af1a4c0fadc8f38073e5b1abd7c962ad97a32f859a2b5b237e
-  data.tar.gz: 13f80b70c03b3a43538f7f687869db8918eb5a319ce7fc2503963d21a0f4fd5fa3f3a1d0163f6e19424c0001061f2b049e80287261fd230a3628e5cae1fd4f76
+  metadata.gz: b6a1a9955411bf1c8fc34be3ad1c3116550107f7b5bd02c56bcd84553716979e73500ef006651beef88614d04bbf8699dc9b8a5b54aee8bb202488f11a55fac6
+  data.tar.gz: d628b629c1538834cb2b6233ca4bb82410800a31386ebc011913e51b1780e92fd47c7597ce67539038e59303a1867ebf3d8423078001bcc7ffe62c48a397636e

data/.travis.yml

@@ -1,13 +1,9 @@
 rvm:
-  - 2.3.8
-  - 2.4.6
-  - 2.5.5
-  - 2.6.3
+  - 2.1.10
+  - 2.2.5
+  - 2.3.1
   - ruby-head
 
-services:
-  - mysql
-
 script: "bundle exec rake spec"
 
 before_script:

data/ChangeLog

@@ -1,23 +1,10 @@
-== 2020-09-10 version 0.8.54
+== 2016-09-16 version 0.9.0
 
-* Enhance logging on inter-process communication (#68)
-
-== 2019-10-10 version 0.8.53
-
-* Extract max_retry_count as config parameter (#66)
-
-== 2019-07-31 version 0.8.52
-
-* Add sleep lock retrying with exponential-backoff (#64)
-
-== 2016-10-24 version 0.8.51
-
-* Retry on temporary DatabaseConnectionError (#59)
-
-== 2016-08-18 version 0.8.50
-
-* Decrease GET_LOCK timeout from 60 to 10 seconds
-* Add 0.5 to 30 seconds of sleep between GET_LOCK retrying
+* Use UPDATE first strategy (#15)
+  * owner column is added for the table: migration is required
+  * AcquiredTask#timeout is the current value rather than at acquiring
+  * cancel_request is removed
+  * prefetch_break_types is removed
 
 == 2016-08-02 version 0.8.49
 

data/README.md

@@ -46,10 +46,6 @@ Queue#[](key)  #=> #<Task>
 # chack the existance of the task
 Task#exists?
 
-# request to cancel a task
-# (actual behavior depends on the worker program)
-Task#cancel_request!
-
 # force finish a task
 # be aware that worker programs can't detect it
 Task#force_finish!
@@ -64,8 +60,6 @@ TaskError
 # Workers may get these errors:
 #
 
-CancelRequestedError < TaskError
-
 AlreadyFinishedError < TaskError
 
 PreemptedError < TaskError
@@ -206,7 +200,6 @@ Usage: perfectqueue [options] <command>
 commands:
     list                             Show list of tasks
     submit <key> <type> <data>       Submit a new task
-    cancel_request <key>             Cancel request
     force_finish <key>               Force finish a task
     run <class>                      Run a worker process
     init                             Initialize a backend database
@@ -243,10 +236,6 @@ options for run:
                                 k3     system_task                               waiting    2012-05-18 14:04:02 -0700    2012-05-22 15:04:02 -0700   {"task_id"=>32, "type"=>"system_task"}
     3 entries.
 
-### cancel a tasks
-
-    $ perfectqueue cancel_request k1
-
 ### force finish a tasks
 
     $ perfectqueue force_finish k2

data/circle.yml

@@ -0,0 +1,7 @@
+machine:
+  ruby:
+    version: 2.2.2
+
+database:
+  pre:
+    - mysql -e 'create database perfectqueue_test;'

data/lib/perfectqueue/backend/rdb.rb

@@ -4,6 +4,7 @@ require_relative 'rdb_compat'
 
 module PerfectQueue::Backend
   class RDBBackend
+    MAX_RETRY = ::PerfectQueue::Backend::RDBCompatBackend::MAX_RETRY
     DELETE_OFFSET = ::PerfectQueue::Backend::RDBCompatBackend::DELETE_OFFSET
     class Token < Struct.new(:key)
     end
@@ -20,9 +21,6 @@ module PerfectQueue::Backend
         host: u.host,
         port: u.port ? u.port.to_i : 3306
       }
-      @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
-      @max_retry_count = config.fetch(:max_retry_count, 10)
-      options[:connect_timeout] = config.fetch(:connect_timeout, 3)
       options[:sslca] = config[:sslca] if config[:sslca]
       db_name = u.path.split('/')[1]
       @db = Sequel.mysql2(db_name, options)
@@ -34,7 +32,6 @@ module PerfectQueue::Backend
     end
 
     attr_reader :db
-    attr_reader :max_retry_count
 
     def submit(id, data, time=Process.clock_gettime(Process::CLOCK_REALTIME, :second), resource=nil, max_running=nil)
       connect {
@@ -57,31 +54,23 @@ module PerfectQueue::Backend
     end
 
     private
-    def connect
-      tmax = Process.clock_gettime(Process::CLOCK_REALTIME, :second) + @pq_connect_timeout
+    def connect(&block)
       @mutex.synchronize do
         retry_count = 0
         begin
-          yield
-        rescue Sequel::DatabaseConnectionError
-          if (retry_count += 1) < @max_retry_count && tmax > Process.clock_gettime(Process::CLOCK_REALTIME, :second)
-            STDERR.puts "#{$!}\n  retrying."
-            sleep 2
-            retry
-          end
-          STDERR.puts "#{$!}\n  abort."
-          raise
+          block.call
         rescue
           # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
           if $!.to_s.include?('try restarting transaction')
-            err = $!.backtrace.map{|bt| "  #{bt}" }.unshift($!).join("\n")
+            err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
             retry_count += 1
-            if retry_count < @max_retry_count
-              STDERR.puts "#{err}\n  retrying."
+            if retry_count < MAX_RETRY
+              STDERR.puts err + "\n  retrying."
               sleep 0.5
               retry
+            else
+              STDERR.puts err + "\n  abort."
             end
-            STDERR.puts "#{err}\n  abort."
           end
           raise
         ensure

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -33,17 +33,12 @@ module PerfectQueue
       DELETE_OFFSET = 10_0000_0000
       EVENT_HORIZON = 13_0000_0000 # 2011-03-13 07:06:40 UTC
 
-      LOCK_RETRY_INITIAL_INTERVAL = 0.5
-      LOCK_RETRY_MAX_INTERVAL = 30
-
       class Token < Struct.new(:key)
       end
 
       def initialize(client, config)
         super
 
-        @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
-        @max_retry_count = config.fetch(:max_retry_count, 10)
         url = config[:url]
         @table = config[:table]
         unless @table
@@ -51,9 +46,7 @@ module PerfectQueue
         end
 
         if /\Amysql2:/i =~ url
-          options = {max_connections: 1, sslca: config[:sslca]}
-          options[:connect_timeout] = config.fetch(:connect_timeout, 3)
-          @db = Sequel.connect(url, options)
+          @db = Sequel.connect(url, {max_connections: 1, sslca: config[:sslca]})
           if config.fetch(:use_connection_pooling, nil) != nil
             @use_connection_pooling = !!config[:use_connection_pooling]
           else
@@ -61,14 +54,11 @@ module PerfectQueue
           end
           @table_lock = lambda {
             locked = nil
-            interval = LOCK_RETRY_INITIAL_INTERVAL
             loop do
               @db.fetch("SELECT GET_LOCK('#{@table}', #{LOCK_WAIT_TIMEOUT}) locked") do |row|
                 locked = true if row[:locked] == 1
               end
               break if locked
-              sleep interval
-              interval = [interval * 2, LOCK_RETRY_MAX_INTERVAL].min
             end
           }
           @table_unlock = lambda {
@@ -85,35 +75,57 @@ module PerfectQueue
           # connection test
         }
 
+        # MySQL's CONNECTION_ID() is a 64bit unsigned integer from the
+        # server's internal thread ID counter. It is unique while the MySQL
+        # server is running.
+        # https://bugs.mysql.com/bug.php?id=19806
+        #
+        # An acquired task is marked with next_timeout and CONNECTION_ID().
+        # Therefore while alive_time is not changed and we don't restart
+        # the server in 1 second, they won't conflict.
         if config[:disable_resource_limit]
+          @update_sql = <<SQL
+UPDATE `#{@table}`
+  JOIN (
+  SELECT id
+    FROM `#{@table}` FORCE INDEX (`index_#{@table}_on_timeout`)
+   WHERE #{EVENT_HORIZON} < timeout AND timeout <= :now
+   ORDER BY timeout ASC
+      LIMIT :max_acquire FOR UPDATE) AS t1 USING(id)
+   SET timeout=:next_timeout, owner=CONNECTION_ID()
+SQL
           @sql = <<SQL
 SELECT id, timeout, data, created_at, resource
-FROM `#{@table}`
-WHERE #{EVENT_HORIZON} < timeout AND timeout <= ? AND timeout <= ?
-      AND created_at IS NOT NULL
-ORDER BY timeout ASC
-LIMIT ?
+  FROM `#{@table}`
+ WHERE timeout = ? AND owner = CONNECTION_ID()
 SQL
         else
+          @update_sql = <<SQL
+UPDATE `#{@table}`
+  JOIN (
+    SELECT id, IFNULL(max_running, 1) / (IFNULL(running, 0) + 1) AS weight
+    FROM `#{@table}`
+    LEFT JOIN (
+      SELECT resource, COUNT(1) AS running
+      FROM `#{@table}` AS t1
+      WHERE timeout > :now AND resource IS NOT NULL
+      GROUP BY resource
+      FOR UPDATE
+    ) AS t2 USING(resource)
+    WHERE #{EVENT_HORIZON} < timeout AND timeout <= :now AND IFNULL(max_running - running, 1) > 0
+    ORDER BY weight DESC, timeout ASC
+    LIMIT :max_acquire
+    FOR UPDATE
+  ) AS t3 USING (id)
+SET timeout = :next_timeout, owner = CONNECTION_ID()
+SQL
           @sql = <<SQL
-SELECT id, timeout, data, created_at, resource, max_running, IFNULL(max_running, 1) / (IFNULL(running, 0) + 1) AS weight
-FROM `#{@table}`
-LEFT JOIN (
-  SELECT resource AS res, COUNT(1) AS running
-  FROM `#{@table}` AS T
-  WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
-  GROUP BY resource
-) AS R ON resource = res
-WHERE #{EVENT_HORIZON} < timeout AND timeout <= ?
-      AND created_at IS NOT NULL
-      AND (max_running-running IS NULL OR max_running-running > 0)
-ORDER BY weight DESC, timeout ASC
-LIMIT ?
+SELECT id, timeout, data, created_at, resource, max_running
+  FROM `#{@table}`
+ WHERE timeout = ? AND owner = CONNECTION_ID()
 SQL
         end
 
-        @prefetch_break_types = config[:prefetch_break_types] || []
-
         @cleanup_interval = config[:cleanup_interval] || DEFAULT_DELETE_INTERVAL
         # If cleanup_interval > max_request_per_child / max_acquire,
         # some processes won't run DELETE query.
@@ -122,10 +134,10 @@ SQL
       end
 
       attr_reader :db
-      attr_reader :max_retry_count
 
       KEEPALIVE = 10
-      LOCK_WAIT_TIMEOUT = 10
+      MAX_RETRY = 10
+      LOCK_WAIT_TIMEOUT = 60
       DEFAULT_DELETE_INTERVAL = 20
 
       def init_database(options)
@@ -139,6 +151,8 @@ SQL
             created_at INT,
             resource VARCHAR(255),
             max_running INT,
+            /* CONNECTION_ID() can be 64bit: https://bugs.mysql.com/bug.php?id=19806 */
+            owner BIGINT(21) UNSIGNED NOT NULL DEFAULT 0,
             PRIMARY KEY (id)
           )
           SQL
@@ -229,7 +243,7 @@ SQL
         t0 = nil
 
         if @cleanup_interval_count <= 0
-          connect { # TODO: HERE should be still connect_locked ?
+          connect {
             t0=Process.clock_gettime(Process::CLOCK_MONOTONIC)
             @db["DELETE FROM `#{@table}` WHERE timeout <= ?", now-DELETE_OFFSET].delete
             @cleanup_interval_count = @cleanup_interval
@@ -239,34 +253,18 @@ SQL
 
         connect_locked {
           t0=Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          n = @db[@update_sql, next_timeout: next_timeout, now: now, max_acquire: max_acquire].update
+          if n <= 0
+            return nil
+          end
+
           tasks = []
-          @db.fetch(@sql, now, now, max_acquire) {|row|
+          @db.fetch(@sql, next_timeout) {|row|
             attributes = create_attributes(nil, row)
             task_token = Token.new(row[:id])
             task = AcquiredTask.new(@client, row[:id], attributes, task_token)
             tasks.push task
-
-            if @prefetch_break_types.include?(attributes[:type])
-              break
-            end
           }
-
-          if tasks.empty?
-            return nil
-          end
-
-          sql = "UPDATE `#{@table}` FORCE INDEX (PRIMARY) SET timeout=? WHERE timeout <= ? AND id IN ("
-          params = [sql, next_timeout, now]
-          tasks.each {|t| params << t.key }
-          sql << (1..tasks.size).map { '?' }.join(',')
-          sql << ") AND created_at IS NOT NULL"
-
-          n = @db[*params].update
-          if n != tasks.size
-            # NOTE table lock doesn't work. error!
-            return nil
-          end
-
           @cleanup_interval_count -= 1
 
           return tasks
@@ -275,18 +273,6 @@ SQL
         STDERR.puts "PQ:acquire from #{@table}:%6f sec (%d tasks)" % [Process.clock_gettime(Process::CLOCK_MONOTONIC)-t0,tasks.size] if tasks
       end
 
-      # => nil
-      def cancel_request(key, options)
-        # created_at=0 means cancel_requested
-        connect {
-          n = @db["UPDATE `#{@table}` SET created_at=0 WHERE id=? AND created_at IS NOT NULL", key].update
-          if n <= 0
-            raise AlreadyFinishedError, "task key=#{key} does not exist or already finished."
-          end
-        }
-        nil
-      end
-
       def force_finish(key, retention_time, options)
         finish(Token.new(key), retention_time, options)
       end
@@ -343,7 +329,7 @@ SQL
       end
 
       protected
-      def connect_locked
+      def connect_locked(&block)
         connect {
           locked = false
 
@@ -353,7 +339,7 @@ SQL
               locked = true
             end
 
-            return yield
+            return block.call
           ensure
             if @use_connection_pooling && locked
               @table_unlock.call
@@ -362,31 +348,22 @@ SQL
         }
       end
 
-      def connect
+      def connect(&block)
         now = Time.now.to_i
-        tmax = now + @pq_connect_timeout
         @mutex.synchronize do
           # keepalive_timeout
           @db.disconnect if now - @last_time > KEEPALIVE
 
           count = 0
           begin
-            yield
+            block.call
             @last_time = now
-          rescue Sequel::DatabaseConnectionError
-            if (count += 1) < @max_retry_count && tmax > Time.now.to_i
-              STDERR.puts "#{$!}\n  retrying."
-              sleep 2
-              retry
-            end
-            STDERR.puts "#{$!}\n  abort."
-            raise
           rescue
             # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
             if $!.to_s.include?('try restarting transaction')
               err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
               count += 1
-              if count < @max_retry_count
+              if count < MAX_RETRY
                 STDERR.puts err + "\n  retrying."
                 sleep rand
                 retry
@@ -415,8 +392,6 @@ SQL
         if row[:created_at] === nil
           created_at = nil  # unknown creation time
           status = TaskStatus::FINISHED
-        elsif row[:created_at] <= 0
-          status = TaskStatus::CANCEL_REQUESTED
         elsif now && row[:timeout] < now
           created_at = row[:created_at]
           status = TaskStatus::WAITING

data/lib/perfectqueue/client.rb

@@ -69,11 +69,6 @@ module PerfectQueue
       @backend.acquire(alive_time, max_acquire, options)
     end
 
-    # :message => nil
-    def cancel_request(key, options={})
-      @backend.cancel_request(key, options)
-    end
-
     def force_finish(key, options={})
       retention_time = options[:retention_time] || @retention_time