perfectqueue 0.7.32 → 0.8.0

data/lib/perfectqueue/task_status.rb

@@ -0,0 +1,27 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module TaskStatus
+    WAITING = :waiting
+    RUNNING = :running
+    FINISHED = :finished
+    CANCEL_REQUESTED = :cancel_requested
+  end
+end
+

data/lib/perfectqueue/version.rb

@@ -1,5 +1,3 @@
 module PerfectQueue
-
-VERSION = '0.7.32'
-
+  VERSION = "0.8.0"
 end

data/lib/perfectqueue/worker.rb

@@ -1,236 +1,154 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
 
 module PerfectQueue
 
+  class Worker
+    def self.run(runner, config=nil, &block)
+      new(runner, config, &block).run
+    end
 
-class MonitorThread
-  def initialize(engine, conf)
-    @engine = engine
-    @log = @engine.log
-    @backend = engine.backend
-    @finished = false
-
-    @timeout = conf[:timeout] || 600
-    @heartbeat_interval = conf[:heartbeat_interval] || @timeout*3/4
-    @kill_timeout = conf[:kill_timeout] || @timeout*10
-    @kill_interval = conf[:kill_interval] || 60
-    @retry_wait = conf[:retry_wait] || nil
-    @delete_wait = conf[:delete_wait] || 3600
-
-    @token = nil
-    @heartbeat_time = nil
-    @kill_time = nil
-    @kill_proc = nil
-    @canceled = false
-    @mutex = Mutex.new
-    @cond = ConditionVariable.new
-  end
-
-  def start
-    @thread = Thread.new(&method(:run))
-  end
+    def initialize(runner, config=nil, &block)
+      # initial logger
+      STDERR.sync = true
+      @log = DaemonsLogger.new(STDERR)
 
-  def run
-    until @finished
-      @mutex.synchronize {
-        while true
-          return if @finished
-          break if @token
-          @cond.wait(@mutex)
-        end
-      }
-      process
+      @runner = runner
+      block = Proc.new { config } if config
+      @config_load_proc = block
+      @finished = false
     end
-  rescue
-    @engine.stop($!)
-  end
 
-  def process
-    while true
-      sleep 1
-      @mutex.synchronize {
-        return if @finished
-        return unless @token
-        now = Time.now.to_i
-        try_extend(now)
-        try_kill(now)
-      }
+    def run
+      @sig = install_signal_handlers
+      begin
+        @engine = Engine.new(@runner, load_config)
+        begin
+          @engine.run
+        ensure
+          @engine.shutdown(true)
+        end
+      ensure
+        @sig.shutdown
+      end
+      return nil
+    rescue
+      @log.error "#{$!.class}: #{$!}"
+      $!.backtrace.each {|x| @log.warn "\t#{x}" }
+      return nil
     end
-  end
 
-  def try_extend(now)
-    if now >= @heartbeat_time && !@canceled
-      @log.debug "extending timeout=#{now+@timeout} id=#{@task_id}"
+    def stop(immediate)
+      @log.info immediate ? "Received immediate stop" : "Received graceful stop"
       begin
-        @backend.update(@token, now+@timeout)
-      rescue CanceledError
-        @log.info "task id=#{@task_id} is canceled."
-        @canceled = true
-        @kill_time = now
+        @engine.stop(immediate) if @engine
       rescue
-        @log.error "unexpected error id=#{@task_id}: #{$!}"
-        @canceled = true
-        @kill_time = now
+        @log.error "failed to stop: #{$!}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+        return false
       end
-      @heartbeat_time = now + @heartbeat_interval
+      return true
     end
-  end
 
-  def try_kill(now)
-    if now >= @kill_time
-      kill!
-      @kill_time = now + @kill_interval
+    def restart(immediate)
+      @log.info immediate ? "Received immediate restart" : "Received graceful restart"
+      begin
+        @engine.restart(immediate, load_config)
+      rescue
+        @log.error "failed to restart: #{$!}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+        return false
+      end
+      return true
     end
-  end
 
-  def kill!
-    if @kill_proc
-      @log.info "killing id=#{@task_id}..."
+    def replace(immediate, command=[$0]+ARGV)
+      @log.info immediate ? "Received immediate binary replace" : "Received graceful binary replace"
       begin
-        @kill_proc.call
+        @engine.replace(immediate, command)
       rescue
-        @log.info "kill failed id=#{@task_id}: #{$!.class}: #{$!}"
-        $!.backtrace.each {|bt|
-          @log.debug "  #{bt}"
-        }
+        @log.error "failed to replace: #{$!}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+        return false
       end
+      return true
     end
-  end
 
-  def stop
-    @mutex.synchronize {
-      @finished = true
-      @cond.broadcast
-    }
-  end
-
-  def shutdown
-    @thread.join
-  end
+    def logrotated
+      @log.info "reopen a log file"
+      @engine.logrotated
+      @log.reopen!
+      return true
+    end
 
-  def set(token, task_id)
-    @mutex.synchronize {
-      now = Time.now.to_i
-      @token = token
-      @task_id = task_id
-      @heartbeat_time = now + @heartbeat_interval
-      @kill_time = now + @kill_timeout
-      @kill_proc = nil
-      @canceled = false
-      @cond.broadcast
-    }
-  end
+    private
+    def load_config
+      raw_config = @config_load_proc.call
+      config = {}
+      raw_config.each_pair {|k,v| config[k.to_sym] = v }
 
-  def set_kill_proc(kill_proc)
-    @kill_proc = kill_proc
-  end
+      old_log = @log
+      log = DaemonsLogger.new(config[:log] || STDERR)
+      old_log.close if old_log
+      @log = log
 
-  def reset(success)
-    @mutex.synchronize {
-      if success
-        @backend.finish(@token, @delete_wait)
-      elsif @retry_wait && !@canceled
-        begin
-          @backend.update(@token, Time.now.to_i+@retry_wait)
-        rescue
-          # ignore CanceledError
-        end
-      end
-      @token = nil
-    }
-  end
-end
+      config[:logger] = log
 
+      return config
+    end
 
-class Worker
-  def initialize(engine, conf)
-    @engine = engine
-    @log = @engine.log
+    def install_signal_handlers
+      SignalQueue.start do |sig|
+        sig.trap :TERM do
+          stop(false)
+        end
+        sig.trap :INT do
+          stop(false)
+        end
 
-    @run_class = conf[:run_class]
-    @monitor = MonitorThread.new(engine, conf)
+        sig.trap :QUIT do
+          stop(true)
+        end
 
-    @token = nil
-    @task = nil
-    @mutex = Mutex.new
-    @cond = ConditionVariable.new
-  end
+        sig.trap :USR1 do
+          restart(false)
+        end
 
-  def start
-    @log.debug "running worker."
-    @thread = Thread.new(&method(:run))
-  end
+        sig.trap :HUP do
+          restart(true)
+        end
 
-  def run
-    @monitor.start
-    begin
-      while true
-        @mutex.synchronize {
-          while true
-            return if @engine.finished?
-            break if @token
-            @cond.wait(@mutex)
-          end
-        }
-        begin
-          process(@token, @task)
-        ensure
-          @token = nil
-          @engine.release_worker(self)
+        sig.trap :CONT do
+          replace(false)
         end
-      end
-    ensure
-      @monitor.stop
-    end
-  rescue
-    @engine.stop($!)
-  end
 
-  def process(token, task)
-    @log.info "processing task id=#{task.id}"
+        sig.trap :WINCH do
+          replace(true)
+        end
 
-    @monitor.set(token, task.id)
-    success = false
-    begin
-      run = @run_class.new(task)
+        sig.trap :USR2 do
+          logrotated
+        end
 
-      if run.respond_to?(:kill)
-        @monitor.set_kill_proc run.method(:kill)
+        trap :CHLD, "SIG_IGN"
       end
-
-      run.run
-
-      @log.info "finished id=#{task.id}"
-      success = true
-
-    rescue
-      @log.info "failed id=#{task.id}: #{$!.class}: #{$!}"
-      $!.backtrace.each {|bt|
-        @log.debug "  #{bt}"
-      }
-
-    ensure
-      @monitor.reset(success)
     end
   end
 
-  def stop
-    submit(nil, nil)
-  end
-
-  def shutdown
-    @monitor.shutdown
-    @thread.join
-  end
-
-  def submit(token, task)
-    @mutex.synchronize {
-      @token = token
-      @task = task
-      @cond.broadcast
-    }
-  end
-end
-
-
 end
 

data/perfectqueue.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+require 'perfectqueue/version'
+
+Gem::Specification.new do |gem|
+  gem.name        = "perfectqueue"
+  gem.description = "Highly available distributed cron built on RDBMS"
+  gem.homepage    = "https://github.com/treasure-data/perfectqueue"
+  gem.summary     = gem.description
+  gem.version     = PerfectQueue::VERSION
+  gem.authors     = ["Sadayuki Furuhashi"]
+  gem.email       = "frsyuki@gmail.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "sequel", "~> 3.26.0"
+  gem.add_development_dependency "rake", "~> 0.9.2"
+  gem.add_development_dependency "rspec", "~> 2.10.0"
+  gem.add_development_dependency "simplecov", "~> 0.5.4"
+  gem.add_development_dependency "sqlite3", "~> 1.3.3"
+end

data/spec/queue_spec.rb

@@ -0,0 +1,234 @@
+require 'spec_helper'
+
+describe Queue do
+  before do
+    @queue = create_test_queue
+  end
+
+  after do
+    @queue.client.close
+  end
+
+  it 'is a Queue' do
+    @queue.class.should == PerfectQueue::Queue
+  end
+
+  it 'succeess submit' do
+    @queue.submit('task01', 'type1', {})
+  end
+
+  it 'fail duplicated submit' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {}, :now=>now)
+
+    lambda {
+      @queue.submit('task01', 'type1', {}, :now=>now+1)
+    }.should raise_error AlreadyExistsError
+
+    @queue['task01'].cancel_request!(:now=>now+2)
+
+    lambda {
+      @queue.submit('task01', 'type1', {}, :now=>now+10)
+    }.should raise_error AlreadyExistsError
+  end
+
+  it 'list' do
+    @queue.submit('task01', 'type1', {"a"=>1})
+    @queue.submit('task02', 'type1', {"a"=>2})
+    @queue.submit('task03', 'type1', {"a"=>3})
+
+    a = []
+    @queue.each {|t| a << t }
+    a.sort_by! {|t| t.key }
+
+    task01 = a.shift
+    task01.finished?.should == false
+    task01.type == 'type1'
+    task01.key.should == 'task01'
+    task01.data["a"].should == 1
+
+    t2 = a.shift
+    t2.finished?.should == false
+    t2.type == 'type1'
+    t2.key.should == 'task02'
+    t2.data["a"].should == 2
+
+    t3 = a.shift
+    t3.finished?.should == false
+    t3.type == 'type1'
+    t3.key.should == 'task03'
+    t3.data["a"].should == 3
+
+    a.empty?.should == true
+  end
+
+  it 'poll' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    @queue.submit('task02', 'type1', {"a"=>2}, :now=>now+1)
+    @queue.submit('task03', 'type1', {"a"=>3}, :now=>now+2)
+
+    task01 = @queue.poll(:now=>now+10)
+    task01.key.should == 'task01'
+
+    t2 = @queue.poll(:now=>now+10)
+    t2.key.should == 'task02'
+
+    t3 = @queue.poll(:now=>now+10)
+    t3.key.should == 'task03'
+
+    t4 = @queue.poll(:now=>now+10)
+    t4.should == nil
+  end
+
+  it 'release' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    task01 = @queue.poll(:now=>now+10)
+    task01.key.should == 'task01'
+
+    t2 = @queue.poll(:now=>now+10)
+    t2.should == nil
+
+    task01.release!(:now=>now+10)
+
+    t3 = @queue.poll(:now=>now+11)
+    t3.key.should == 'task01'
+  end
+
+  it 'timeout' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01.key.should == 'task01'
+
+    t2 = @queue.poll(:now=>now+15)
+    t2.should == nil
+
+    t3 = @queue.poll(:now=>now+20)
+    t3.key.should == 'task01'
+  end
+
+  it 'heartbeat' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01.key.should == 'task01'
+
+    task01.heartbeat!(:alive_time=>15, :now=>now+10)
+
+    t2 = @queue.poll(:now=>now+20)
+    t2.should == nil
+
+    t3 = @queue.poll(:now=>now+30)
+    t3.key.should == 'task01'
+  end
+
+  it 'retry' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01.key.should == 'task01'
+
+    task01.retry!(:retry_wait=>15, :now=>now+10)
+
+    t2 = @queue.poll(:now=>now+20)
+    t2.should == nil
+
+    t3 = @queue.poll(:now=>now+30)
+    t3.key.should == 'task01'
+  end
+
+  it 'froce_finish' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    task01 = @queue.poll(:now=>now+10)
+    task01.key.should == 'task01'
+
+    @queue['task01'].metadata.running?.should == true
+
+    @queue['task01'].force_finish!(:now=>now+11)
+
+    @queue['task01'].metadata.finished?.should == true
+  end
+
+  it 'status' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    # rdb_backend backend can't distinguish running with waiting
+    #@queue['task01'].metadata.finished?.should == false
+    #@queue['task01'].metadata.running?.should == false
+    #@queue['task01'].metadata.waiting?.should == true
+    #@queue['task01'].metadata.cancel_requested?.should == false
+
+    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01.key.should == 'task01'
+
+    @queue['task01'].metadata.finished?.should == false
+    @queue['task01'].metadata.running?.should == true
+    @queue['task01'].metadata.waiting?.should == false
+    @queue['task01'].metadata.cancel_requested?.should == false
+
+    task01.cancel_request!
+
+    # status of cancel_requested running tasks is cancel_requested
+    @queue['task01'].metadata.finished?.should == false
+    @queue['task01'].metadata.running?.should == false
+    @queue['task01'].metadata.waiting?.should == false
+    @queue['task01'].metadata.cancel_requested?.should == true
+
+    task01.finish!
+
+    @queue['task01'].metadata.finished?.should == true
+    @queue['task01'].metadata.running?.should == false
+    @queue['task01'].metadata.waiting?.should == false
+    @queue['task01'].metadata.cancel_requested?.should == false
+  end
+
+  it 'fail canceling finished task' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01.key.should == 'task01'
+
+    task01.finish!
+
+    lambda {
+      @queue['task01'].cancel_request!
+    }.should raise_error AlreadyFinishedError
+  end
+
+  it 'retention_time' do
+    now = Time.now.to_i
+    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+
+    @queue['task01'].metadata.finished?.should == false
+
+    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01.key.should == 'task01'
+
+    task01.finish!(:now=>now+11, :retention_time=>10)
+
+    @queue.poll(:now=>now+12)
+
+    @queue['task01'].exists?.should == true
+
+    @queue.poll(:now=>now+22)
+
+    @queue['task01'].exists?.should == false
+  end
+
+  it 'get_task_metadata failed with NotFoundError' do
+    lambda {
+      @queue['task99'].metadata
+    }.should raise_error NotFoundError
+  end
+end
+