perfectqueue 0.7.14 → 0.7.15

data/ChangeLog

@@ -1,4 +1,9 @@
 
+== 2012-01-10 version 0.7.15
+
+* RDBBackend: Fixed locking routines to work with possibility of deadlocks
+
+
 == 2012-01-08 version 0.7.14
 
 * Re-enabled order of the tasks

data/lib/perfectqueue/backend/rdb.rb

@@ -7,7 +7,9 @@ class RDBBackend < Backend
     require 'sequel'
     @uri = uri
     @table = table
-    @db = Sequel.connect(@uri)
+    @db = Sequel.connect(@uri, :max_connections=>1)
+    @last_time = Time.now.to_i
+    @mutex = Mutex.new
     #init_db(@uri.split('//',2)[0])
     connect {
       # connection test
@@ -16,15 +18,19 @@ class RDBBackend < Backend
 SELECT id, timeout, data, created_at, resource
 FROM `#{@table}`
 LEFT JOIN (
-  SELECT resource AS res,
-         CASE WHEN resource IS NULL THEN 0 ELSE COUNT(1) END AS running
-  FROM `#{@table}`
-  WHERE timeout > ? AND created_at IS NOT NULL
+  SELECT resource AS res, COUNT(1) AS running
+  FROM `#{@table}` AS T
+  WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
   GROUP BY resource
-) AS T ON resource = res
+) AS R ON resource = res
 WHERE timeout <= ? AND (running IS NULL OR running < #{MAX_RESOURCE})
 ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW}
 SQL
+    # sqlite doesn't support SELECT ... FOR UPDATE but
+    # sqlite doesn't need it because the db is not shared
+    unless @uri.split('//',2)[0].to_s.include?('sqlite')
+      @sql << 'FOR UPDATE'
+    end
   end
 
   def create_tables
@@ -45,10 +51,29 @@ SQL
 
   private
   def connect(&block)
-    begin
-      block.call
-    ensure
-      @db.disconnect
+    now = Time.now.to_i
+    @mutex.synchronize do
+      if now - @last_time > KEEPALIVE
+        @db.disconnect
+      end
+      @last_time = now
+      retry_count = 0
+      begin
+        block.call
+      rescue
+        # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
+        err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
+        if $!.to_s.include?('try restarting transaction')
+          retry_count += 1
+          if retry_count < MAX_RETRY
+            STDERR.puts err + "\nretrying"
+            retry
+          else
+            STDERR.puts err + "\nabort"
+          end
+        end
+        raise
+      end
     end
   end
 
@@ -59,24 +84,25 @@ SQL
     }
   end
 
-  MAX_SELECT_ROW = 4
+  MAX_SELECT_ROW = 8
   MAX_RESOURCE = (ENV['PQ_MAX_RESOURCE'] || 4).to_i
+  KEEPALIVE = 10
+  MAX_RETRY = 10
 
   def acquire(timeout, now=Time.now.to_i)
     connect {
-      @db.run "SET autocommit=0;" rescue nil  # TODO mysql only
-      @db.run "LOCK TABLES `#{@table}` WRITE, T WRITE;" rescue nil  # TODO mysql only
-      begin
+      @db.transaction do
         while true
           rows = 0
           @db.fetch(@sql, now, now) {|row|
-
             unless row[:created_at]
               # finished/canceled task
               @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
 
             else
-              n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?;", timeout, row[:id], row[:timeout]].update
+              ## optimistic lock is not needed because the row is locked for update
+              #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
+              n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", timeout, row[:id]].update
               if n > 0
                 return row[:id], Task.new(row[:id], row[:created_at], row[:data], row[:resource])
               end
@@ -84,12 +110,8 @@ SQL
 
             rows += 1
           }
-          if rows < MAX_SELECT_ROW
-            return nil
-          end
+          break nil if rows < MAX_SELECT_ROW
         end
-      ensure
-        @db.run "UNLOCK TABLES T, `#{@table}`;" rescue nil  # TODO mysql only
       end
     }
   end

data/lib/perfectqueue/version.rb

@@ -1,5 +1,5 @@
 module PerfectQueue
 
-VERSION = '0.7.14'
+VERSION = '0.7.15'
 
 end

data/test/backend_test.rb

@@ -171,13 +171,13 @@ class BackendTest < Test::Unit::TestCase
     assert_not_equal nil, task
 
     assert_nothing_raised do
-      db1.update(token, time+TIMEOUT)
+      db1.update(token, time+TIMEOUT+1)
     end
 
     token_, task_ = db2.acquire(time+TIMEOUT, time)
     assert_equal nil, token_
 
-    token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT)
+    token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT+1)
     assert_not_equal nil, task
     assert_equal @key_prefix+'test1', task.id
     assert_equal time, task.created_at
@@ -199,7 +199,7 @@ class BackendTest < Test::Unit::TestCase
     assert_not_equal nil, task
 
     assert_nothing_raised do
-      db1.update(token, time+TIMEOUT)
+      db1.update(token, time+TIMEOUT+1)
     end
 
     token_, task_ = db2.acquire(time+TIMEOUT, time)
@@ -224,7 +224,7 @@ class BackendTest < Test::Unit::TestCase
     time = Time.now.to_i
 
     5.times do |i|
-      ok = db1.submit(@key_prefix+'test'+i.to_s, 'data1', time, 'user1')
+      ok = db1.submit(@key_prefix+'test'+i.to_s, 'data1', time-i, 'user1')
       assert_equal true, ok
     end
     ok = db1.submit(@key_prefix+'test5', 'data2', time, 'user2')

data/test/stress.rb

@@ -0,0 +1,99 @@
+$LOAD_PATH << File.expand_path(File.dirname(__FILE__)+"/../lib")
+require 'perfectqueue'
+require 'perfectqueue/backend/rdb'
+require 'perfectqueue/backend/simpledb'
+
+class StressTest
+  def initialize(uri, table, npt, thread)
+    @db_proc = Proc.new do
+      PerfectQueue::RDBBackend.new(uri, table)
+    end
+    @db_proc.call.create_tables
+    @npt = npt
+    @thread = thread
+  end
+
+  class ThreadMain < Thread
+    def initialize(key_prefix, db, num, now)
+      @key_prefix = key_prefix
+      @db = db
+      @num = num
+      @now = now
+      super(&method(:run))
+    end
+
+    def run
+      @num.times {|i|
+        @db.submit("#{@key_prefix}-#{i}", "data", @now)
+        token, task = @db.acquire(@now+60)
+        if token == nil
+          puts "acquire failed"
+          next
+        end
+        @db.update(token, @now+70)
+        @db.finish(token, @now+80)
+      }
+    end
+  end
+
+  def run
+    threads = []
+    key_prefix = "stress-#{'%08x'%rand(2**32)}"
+    now = Time.now
+    @thread.times {|i|
+      threads << ThreadMain.new("#{key_prefix}-#{i}", @db_proc.call, @npt, now.to_i)
+    }
+    threads.each {|t|
+      t.join
+    }
+    finish = Time.now
+
+    elapsed = finish - now
+    puts "#{elapsed} sec."
+    puts "#{@npt * @thread / elapsed} req/sec."
+    puts "#{elapsed / (@npt * @thread)} sec/req."
+  end
+end
+
+require 'optparse'
+
+op = OptionParser.new
+op.banner += " <uri> <table>"
+
+num = 100
+thread = 1
+
+op.on('-n', '--num N', Integer) {|n|
+  num = n
+}
+op.on('-t', '--thread N', Integer) {|n|
+  thread = n
+}
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 2
+    puts op.to_s
+    exit 1
+  end
+
+  uri = ARGV[0]
+  table = ARGV[1]
+
+rescue
+  puts op.to_s
+  puts $!
+  exit 1
+end
+
+npt = num / thread
+num = npt * thread
+
+puts "num: #{num}"
+puts "thread: #{thread}"
+puts "num/thread: #{npt}"
+
+t = StressTest.new(uri, table, npt, thread)
+t.run
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.7.14
+  version: 0.7.15
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-08 00:00:00.000000000Z
+date: 2012-01-11 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70276377861000 !ruby/object:Gem::Requirement
+  requirement: &13147240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 3.26.0
   type: :runtime
   prerelease: false
-  version_requirements: *70276377861000
+  version_requirements: *13147240
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &70276377860060 !ruby/object:Gem::Requirement
+  requirement: &13146400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,7 +32,7 @@ dependencies:
         version: 1.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *70276377860060
+  version_requirements: *13146400
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -54,14 +54,15 @@ files:
 - lib/perfectqueue/worker.rb
 - ChangeLog
 - README.rdoc
-- test/backend_test.rb
 - test/exec_test.rb
 - test/test_helper.rb
+- test/backend_test.rb
+- test/stress.rb
+- test/fail.sh
 - test/cat.sh
 - test/echo.sh
-- test/fail.sh
-- test/huge.sh
 - test/success.sh
+- test/huge.sh
 homepage: https://github.com/treasure-data/perfectqueue
 licenses: []
 post_install_message: 
@@ -82,16 +83,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.12
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Highly available distributed queue built on RDBMS or SimpleDB
 test_files:
-- test/backend_test.rb
 - test/exec_test.rb
 - test/test_helper.rb
+- test/backend_test.rb
+- test/stress.rb
+- test/fail.sh
 - test/cat.sh
 - test/echo.sh
-- test/fail.sh
-- test/huge.sh
 - test/success.sh
+- test/huge.sh