perfectqueue 0.8.17 → 0.8.18

data/ChangeLog

@@ -1,4 +1,12 @@
 
+== 2012-09-04 version 0.8.18
+
+* rdb_compat backend uses table lock to avoid dead locks with InnoDB
+* rdb_compat backend uses bulk cleanup to improve throughput
+* rdb_compat backend uses created_at in WHERE clause to work with partitioning
+* rdb_compat backend supports disable_resource_limit to improve acquire() performance
+
+
 == 2012-09-03 version 0.8.17
 
 * Increased wait time before starting processors to 1 second at average and

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -43,7 +43,16 @@ module PerfectQueue
           # connection test
         }
 
-    @sql = <<SQL
+        if config[:disable_resource_limit]
+          @sql = <<SQL
+SELECT id, timeout, data, created_at, resource
+FROM `#{@table}`
+WHERE timeout <= ? AND timeout <= ? AND created_at IS NOT NULL
+ORDER BY timeout ASC
+LIMIT ?
+SQL
+        else
+          @sql = <<SQL
 SELECT id, timeout, data, created_at, resource, max_running, max_running/running AS weight
 FROM `#{@table}`
 LEFT JOIN (
@@ -52,24 +61,37 @@ LEFT JOIN (
   WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
   GROUP BY resource
 ) AS R ON resource = res
-WHERE timeout <= ? AND (max_running-running IS NULL OR max_running-running > 0)
+WHERE timeout <= ? AND created_at IS NOT NULL AND (max_running-running IS NULL OR max_running-running > 0)
 ORDER BY weight IS NOT NULL, weight DESC, timeout ASC
-LIMIT #{MAX_SELECT_ROW}
+LIMIT ?
 SQL
+        end
 
-        # sqlite doesn't support SELECT ... FOR UPDATE but
-        # sqlite doesn't need it because the db is not shared
-        unless url.split('//',2)[0].to_s.include?('sqlite')
-          @sql << 'FOR UPDATE'
+        case url.split('//',2)[0].to_s
+        when /sqlite/i
+          # sqlite always locks tables on BEGIN
+          @table_lock = nil
+        when /mysql/i
+          if config[:disable_resource_limit]
+            @table_lock = "LOCK TABLES `#{@table}` WRITE"
+          else
+            @table_lock = "LOCK TABLES `#{@table}` WRITE, `#{@table}` AS T WRITE"
+          end
+        else
+          @table_lock = "LOCK TABLE `#{@table}`"
         end
+
+        @prefetch_break_types = config[:prefetch_break_types] || []
+
+        @cleanup_interval = config[:cleanup_interval] || DEFAULT_DELETE_INTERVAL
+        @cleanup_interval_count = 0
       end
 
       attr_reader :db
 
-      MAX_SELECT_ROW = 8
-      MAX_RESOURCE = (ENV['PQ_MAX_RESOURCE'] || 4).to_i
       #KEEPALIVE = 10
       MAX_RETRY = 10
+      DEFAULT_DELETE_INTERVAL = 20
 
       def init_database(options)
         sql = %[
@@ -148,42 +170,49 @@ SQL
         now = (options[:now] || Time.now).to_i
         next_timeout = now + alive_time
 
-        fetched = []
+        tasks = []
 
         connect {
-          while true
-            rows = 0
-            begin
-              @db.transaction do
-                @db.fetch(@sql, now, now) {|row|
-                  unless row[:created_at]
-                    # finished task
-                    @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
-
-                  else
-                    ## optimistic lock is not needed because the row is locked for update
-                    #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
-                    n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
-                    if n > 0
-                      attributes = create_attributes(nil, row)
-                      task_token = Token.new(row[:id])
-                      task = AcquiredTask.new(@client, row[:id], attributes, task_token)
-                      fetched.push task
-                      break if fetched.size >= max_acquire
-                    end
-                  end
-
-                  rows += 1
-                }
+          if @cleanup_interval_count <= 0
+            @db["DELETE FROM `#{@table}` WHERE timeout <= ? AND created_at IS NULL", now].delete
+            @cleanup_interval_count = @cleanup_interval
+          end
+
+          @db.transaction do
+            if @table_lock
+              @db[@table_lock].update
+            end
+
+            tasks = []
+            @db.fetch(@sql, now, now, max_acquire) {|row|
+              attributes = create_attributes(nil, row)
+              task_token = Token.new(row[:id])
+              task = AcquiredTask.new(@client, row[:id], attributes, task_token)
+              tasks.push task
+
+              if @prefetch_break_types.include?(attributes[:type])
+                break
               end
-            rescue
-              raise if fetched.empty?
+            }
+
+            if tasks.empty?
+              return nil
             end
-            unless fetched.empty?
-              return fetched
+
+            sql = "UPDATE `#{@table}` SET timeout = ? WHERE id IN ("
+            params = [sql, next_timeout]
+            tasks.each {|t| params << t.key }
+            sql << (1..tasks.size).map { '?' }.join(',')
+            sql << ") AND created_at IS NOT NULL"
+
+            n = @db[*params].update
+            if n != tasks.size
+              # TODO table lock doesn't work. error?
             end
-            break nil if rows < MAX_SELECT_ROW
+
+            @cleanup_interval_count -= 1
           end
+          return tasks
         }
       end
 

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.17"
+  VERSION = "0.8.18"
 end

data/spec/spec_helper.rb

@@ -25,7 +25,9 @@ module QueueTest
           :type => 'rdb_compat',
           :url => "sqlite://#{database_path}",
           :table => 'test_tasks',
-          :processor_type => 'thread'
+          :processor_type => 'thread',
+          :cleanup_interval => 0,  # for test
+          #:disable_resource_limit => true,  # TODO backend-specific test cases
         }
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.17
+  version: 0.8.18
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-03 00:00:00.000000000 Z
+date: 2012-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel