perfectqueue 0.8.15 → 0.8.16

data/ChangeLog

@@ -1,4 +1,14 @@
 
+== 2012-09-03 version 0.8.16
+
+* Support task_prefetch option (default = 0, meaning no-prefetch)
+* Wait 0.5 seconds before starting processors to avoid a spike of the
+  number of concurrent connections
+* Increased default child_heartbeat_limit to 60 seconds
+* Fixed a potential deadlock problem in shutdown process
+* Fixed detach time wait
+
+
 == 2012-08-30 version 0.8.15
 
 * Improved detach process routines

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -148,29 +148,39 @@ SQL
         now = (options[:now] || Time.now).to_i
         next_timeout = now + alive_time
 
+        fetched = []
+
         connect {
           while true
             rows = 0
-            @db.transaction do
-              @db.fetch(@sql, now, now) {|row|
-                unless row[:created_at]
-                  # finished task
-                  @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
-
-                else
-                  ## optimistic lock is not needed because the row is locked for update
-                  #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
-                  n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
-                  if n > 0
-                    attributes = create_attributes(nil, row)
-                    task_token = Token.new(row[:id])
-                    task = AcquiredTask.new(@client, row[:id], attributes, task_token)
-                    return [task]
+            begin
+              @db.transaction do
+                @db.fetch(@sql, now, now) {|row|
+                  unless row[:created_at]
+                    # finished task
+                    @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
+
+                  else
+                    ## optimistic lock is not needed because the row is locked for update
+                    #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
+                    n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
+                    if n > 0
+                      attributes = create_attributes(nil, row)
+                      task_token = Token.new(row[:id])
+                      task = AcquiredTask.new(@client, row[:id], attributes, task_token)
+                      fetched.push task
+                      break if fetched.size >= max_acquire
+                    end
                   end
-                end
 
-                rows += 1
-              }
+                  rows += 1
+                }
+              end
+            rescue
+              raise if fetched.empty?
+            end
+            unless fetched.empty?
+              return fetched
             end
             break nil if rows < MAX_SELECT_ROW
           end

data/lib/perfectqueue/client.rb

@@ -24,7 +24,6 @@ module PerfectQueue
 
       @backend = Backend.new_backend(self, @config)
 
-      @max_acquire = @config[:max_acquire] || 1
       @retention_time = @config[:retention_time] || 300
       @alive_time = @config[:alive_time] || 300
       @retry_wait = @config[:retry_wait] || 300  # TODO retry wait algorithm
@@ -65,7 +64,7 @@ module PerfectQueue
     # :alive_time => nil
     def acquire(options={})
       alive_time = options[:alive_time] || @alive_time
-      max_acquire = options[:max_acquire] || @max_acquire
+      max_acquire = options[:max_acquire] || 1
 
       @backend.acquire(alive_time, max_acquire, options)
     end

data/lib/perfectqueue/engine.rb

@@ -74,6 +74,12 @@ module PerfectQueue
     end
 
     def run
+      @processors.each {|c|
+        c.keepalive
+        # add wait time before starting processors to avoid
+        # a spike of the number of concurrent connections.
+        sleep rand  # upto 1 second, average 0.5 seoncd
+      }
       until @finish_flag.set?
         @processors.each {|c| c.keepalive }
         @finish_flag.wait(@child_keepalive_interval)

data/lib/perfectqueue/multiprocess/fork_processor.rb

@@ -33,7 +33,7 @@ module PerfectQueue
       end
 
       def restart(immediate, config)
-        @child_heartbeat_limit = config[:child_heartbeat_limit] || 10.0
+        @child_heartbeat_limit = config[:child_heartbeat_limit] || 60.0
         @child_kill_interval = config[:child_kill_interval] || 2.0
         @child_graceful_kill_limit = config[:child_graceful_kill_limit] || nil
         @child_fork_frequency_limit = config[:child_fork_frequency_limit] || 5.0

data/lib/perfectqueue/multiprocess/thread_processor.rb

@@ -59,6 +59,7 @@ module PerfectQueue
       def restart(immediate, config)
         @poll_interval = config[:poll_interval] || 1.0
         @log = config[:logger]
+        @task_prefetch = config[:task_prefetch] || 0
         @config = config
 
         @tm.stop_task(immediate)
@@ -92,11 +93,25 @@ module PerfectQueue
       def run_loop
         PerfectQueue.open(@config) {|queue|
           until @finish_flag.set?
-            task = queue.poll
-            if task
-              process(task)
-            else
+            tasks = queue.poll_multi(:max_acquire=>1+@task_prefetch)
+            if tasks == nil || tasks.empty?
               @finish_flag.wait(@poll_interval)
+            else
+              begin
+                while task = tasks.shift
+                  process(task)
+                end
+              ensure
+                # TODO do not call release! because rdb_compat backend
+                #      doesn't have a mechanism to detect preemption.
+                #      release! could cause a problem that multiple
+                #      workers run one task concurrently.
+                #tasks.each {|task|
+                #  # ignoring errors doesn't cause serious problems
+                #  # because it's same as failure of this server.
+                #  task.release! rescue nil
+                #}
+              end
             end
           end
         }

data/lib/perfectqueue/signal_queue.rb

@@ -32,19 +32,26 @@ module PerfectQueue
       @queue = []
       @mutex = Mutex.new
       @cond = ConditionVariable.new
+      @finished = false
 
       block.call(self) if block
     end
 
-    def trap(sig, &block)
+    def trap(sig, command=nil, &block)
       sig = sig.to_sym
       old = @handlers[sig]
 
-      Kernel.trap(sig) do
-        enqueue(sig)
+      if block
+        Kernel.trap(sig) do
+          enqueue(sig)
+        end
+        @handlers[sig] = block
+
+      else
+        Kernel.trap(sig, command)
+        @handlers.delete(sig)
       end
 
-      @handlers[sig] = block
       old
     end
 
@@ -57,7 +64,8 @@ module PerfectQueue
     end
 
     def stop
-      enqueue(nil)
+      @finished = true
+      enqueue(nil)  # TODO causes recursive lock?
     end
 
     def shutdown
@@ -66,16 +74,16 @@ module PerfectQueue
     end
 
     def run
-      finished = false
-      until finished
+      @owner_thread = Thread.current
+      until @finished
         h = nil
         @mutex.synchronize do
           while @queue.empty?
-            @cond.wait(@mutex)
+            @cond.wait(@mutex, 0.5)
           end
           sig = @queue.shift
           if sig == nil
-            finished = true
+            @finished = true
           else
             h = @handlers[sig]
           end
@@ -95,7 +103,7 @@ module PerfectQueue
 
     private
     def enqueue(sig)
-      if Thread.current == @thread
+      if Thread.current == @owner_thread
         @queue << sig
         if @mutex.locked?
           @cond.signal

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.15"
+  VERSION = "0.8.16"
 end

data/lib/perfectqueue/worker.rb

@@ -59,7 +59,7 @@ module PerfectQueue
         if @detach
           wait_time = Time.now + @detach_wait
           while (w = wait_time - Time.now) > 0
-            sleep [1, w].max
+            sleep [0.5, w].min
             pid, status = Process.waitpid2(@pid, Process::WNOHANG)
             break if pid
           end

data/spec/queue_spec.rb

@@ -224,5 +224,24 @@ describe Queue do
       queue['task99'].metadata
     }.should raise_error NotFoundError
   end
+
+  it 'prefetch' do
+    now = Time.now.to_i
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task02', 'type2', {"a"=>2}, :now=>now+1)
+    queue.submit('task03', 'type3', {"a"=>3}, :now=>now+2)
+
+    tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
+    tasks.size.should == 2
+    tasks[0].key.should == 'task01'
+    tasks[1].key.should == 'task02'
+
+    tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
+    tasks.size.should == 1
+    tasks[0].key.should == 'task03'
+
+    tasks = queue.poll_multi(:now=>now+10, :alive_time=>10, :max_acquire=>2)
+    tasks.should == nil
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.15
+  version: 0.8.16
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-31 00:00:00.000000000 Z
+date: 2012-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel