perfectqueue 0.7.32 → 0.8.0

data/lib/perfectqueue/daemons_logger.rb

@@ -0,0 +1,80 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+
+  require 'logger'
+
+  class DaemonsLogger < Logger
+    def initialize(dev, shift_age=0, shift_size=1048576)
+      @stdout_hook = false
+      @stderr_hook = false
+      if dev.is_a?(String)
+        @path = dev
+        @io = File.open(dev, File::WRONLY|File::APPEND)
+      else
+        @io = dev
+      end
+      super(@io, shift_size, shift_size)
+    end
+
+    def hook_stdout!
+      return nil if @io == STDOUT
+      STDOUT.reopen(@io)
+      @stdout_hook = true
+      self
+    end
+
+    def hook_stderr!
+      STDERR.reopen(@io)
+      @stderr_hook = true
+      self
+    end
+
+    def reopen!
+      if @path
+        @io.reopen(@path)
+        if @stdout_hook
+          STDOUT.reopen(@io)
+        end
+        if @stderr_hook
+          STDERR.reopen(@io)
+        end
+      end
+      nil
+    end
+
+    def reopen
+      begin
+        reopen!
+        return true
+      rescue
+        # TODO log?
+        return false
+      end
+    end
+
+    def close
+      if @path
+        @io.close unless @io.closed?
+      end
+      nil
+    end
+  end
+
+end

data/lib/perfectqueue/engine.rb

@@ -1,152 +1,114 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
 
 module PerfectQueue
 
+  class Engine
+    def initialize(runner, config)
+      @runner = runner
 
-class Engine
-  def initialize(backend, log, conf)
-    @backend = backend
-    @log = log
+      @finish_flag = BlockingFlag.new
 
-    @timeout = conf[:timeout]
-    @poll_interval = conf[:poll_interval] || 1
-    @expire = conf[:expire] || 345600
+      processor_type = config[:processor_type] || :process
+      case processor_type.to_sym
+      when :process
+        @processor_class = Multiprocess::ForkProcessor
+      when :thread
+        @processor_class = Multiprocess::ThreadProcessor
+      else
+        raise ConfigError, "Unknown processor_type: #{config[:processor_type].inspect}"
+      end
 
-    num_workers = conf[:workers] || 1
-    @workers = (1..num_workers).map {
-      Worker.new(self, conf)
-    }
-    @available_workers = @workers.dup
+      @processors = []
+      restart(false, config)
+    end
 
-    @finished = false
-    @error = nil
+    def restart(immediate, config)
+      return nil if @finish_flag.set?
 
-    @mutex = Mutex.new
-    @cond = ConditionVariable.new
-  end
+      # TODO connection check
 
-  attr_reader :backend
-  attr_reader :log
-  attr_reader :error
+      @log = config[:logger] || Logger.new(STDERR)
 
-  def finished?
-    @finished
-  end
+      num_processors = config[:processors] || 1
 
-  def run
-    @workers.each {|w|
-      w.start
-    }
-
-    until finished?
-      w = acquire_worker
-      next unless w
-      begin
-
-        until finished?
-          now = Time.now.to_i
-          token, task = @backend.acquire(now+@timeout)
-
-          unless token
-            sleep @poll_interval
-            next
-          end
-          if task.created_at < now-@expire
-            @log.warn "canceling expired task id=#{task.id}"
-            @backend.cancel(token)
-            next
-          end
-
-          @log.info "acquired task id=#{task.id}"
-          w.submit(token, task)
-          w = nil
-          break
+      # scaling
+      extra = num_processors - @processors.length
+      if extra > 0
+        extra.times do
+          @processors << @processor_class.new(@runner, config)
         end
-
-      ensure
-        release_worker(w) if w
+      elsif extra < 0
+        -extra.times do
+          c = @processors.shift
+          c.stop(immediate)
+          c.join
+        end
+        extra = 0
       end
-    end
-  ensure
-    @finished = true
-  end
 
-  def stop(err=nil)
-    @finished = true
-    @error = error
-    @workers.each {|w|
-      w.stop
-    }
-
-    if err
-      @log.error "#{err.class}: #{err}"
-      err.backtrace.each {|x|
-        @log.error "  #{x}"
+      @processors[0..(-extra-1)].each {|c|
+        c.restart(immediate, config)
       }
-    end
-  end
 
-  def shutdown
-    @finished = true
-    @workers.each {|w|
-      w.shutdown
-    }
-  end
+      @child_keepalive_interval = (config[:child_keepalive_interval] || config[:child_heartbeat_interval] || 2).to_i
 
-  def acquire_worker
-    @mutex.synchronize {
-      while @available_workers.empty?
-        return nil if finished?
-        @cond.wait(@mutex)
-      end
-      return @available_workers.pop
-    }
-  end
+      self
+    end
 
-  def release_worker(worker)
-    @mutex.synchronize {
-      @available_workers.push worker
-      if @available_workers.size == 1
-        @cond.broadcast
+    def run
+      until @finish_flag.set?
+        @processors.each {|c| c.keepalive }
+        @finish_flag.wait(@child_keepalive_interval)
       end
-    }
-  end
-end
+      join
+    end
 
+    def stop(immediate)
+      @processors.each {|c| c.stop(immediate) }
+      @finish_flag.set!
+      self
+    end
 
-class ExecRunner
-  def initialize(cmd, task)
-    @cmd = cmd
-    @task = task
-    @iobuf = ''
-    @pid = nil
-    @kill_signal = :TERM
-  end
+    def join
+      @processors.each {|c| c.join }
+      self
+    end
 
-  def run
-    cmdline = "#{@cmd} #{Shellwords.escape(@task.id)}"
-    IO.popen(cmdline, "r+") {|io|
-      @pid = io.pid
-      io.write(@task.data) rescue nil
-      io.close_write
-      begin
-        while true
-          io.sysread(1024, @iobuf)
-          print @iobuf
-        end
-      rescue EOFError
+    def shutdown(immediate)
+      stop(immediate)
+      join
+    end
+
+    def replace(immediate, command=[$0]+ARGV)
+      return if @replaced_pid
+      stop(immediate)
+      @replaced_pid = Process.fork do
+        exec(*command)
+        exit!(127)
       end
-    }
-    if $?.to_i != 0
-      raise "Command failed"
+      self
     end
-  end
 
-  def kill
-    Process.kill(@kill_signal, @pid)
-    @kill_signal = :KILL
+    def logrotated
+      @processors.each {|c| c.logrotated }
+    end
   end
-end
-
 
 end
 

data/lib/perfectqueue/error.rb

@@ -0,0 +1,53 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  class TaskError < StandardError
+  end
+
+  class CancelRequestedError < TaskError
+  end
+
+  class AlreadyFinishedError < TaskError
+  end
+
+  class NotFoundError < TaskError
+  end
+
+  class AlreadyExistsError < TaskError
+  end
+
+  class PreemptedError < TaskError
+  end
+
+  class NotSupportedError < TaskError
+  end
+
+  class ConfigError < RuntimeError
+  end
+
+  class ProcessStopError < RuntimeError
+  end
+
+  class ImmediateProcessStopError < ProcessStopError
+  end
+
+  class GracefulProcessStopError < ProcessStopError
+  end
+end
+

data/lib/perfectqueue/model.rb

@@ -0,0 +1,37 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Model
+    def initialize(client)
+      @client = client
+    end
+
+    attr_reader :client
+
+    def config
+      @client.config
+    end
+
+    ## TODO
+    #def inspect
+    #  "<#{self.class}>"
+    #end
+  end
+end
+

data/lib/perfectqueue/multiprocess.rb

@@ -0,0 +1,31 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Multiprocess
+    {
+      :ChildProcess => 'multiprocess/child_process',
+      :ChildProcessMonitor => 'multiprocess/child_process_monitor',
+      :ForkProcessor => 'multiprocess/fork_processor',
+      :ThreadProcessor => 'multiprocess/thread_processor',
+    }.each_pair {|k,v|
+      autoload k, File.expand_path(v, File.dirname(__FILE__))
+    }
+  end
+end
+

data/lib/perfectqueue/multiprocess/child_process.rb

@@ -0,0 +1,108 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Multiprocess
+
+    class ChildProcess < ThreadProcessor
+      def self.run(runner, config, wpipe)
+        new(runner, config, wpipe).run
+      end
+
+      def initialize(runner, config, wpipe)
+        @wpipe = wpipe
+        @wpipe.sync = true
+        super(runner, config)
+        @sig = install_signal_handlers
+      end
+
+      def run
+        super
+        @sig.shutdown
+      end
+
+      def stop(immediate)
+        @log.info "Exiting worker pid=#{Process.pid}"
+        super
+      end
+
+      # override
+      def join
+        # do nothing
+      end
+
+      # override
+      def keepalive
+        # do nothing
+      end
+
+      def logrotated
+        @log.reopen!
+      end
+
+      def child_heartbeat
+        @wpipe.write HEARTBEAT_PACKET
+      rescue
+        @log.error "Parent process unexpectedly died. Exiting worker pid=#{Process.pid}: #{$!}"
+        stop(true)
+        Process.kill(:KILL, Process.pid)
+        exit! 137
+      end
+
+      HEARTBEAT_PACKET = [0].pack('C')
+
+      private
+      def install_signal_handlers
+        SignalQueue.start do |sig|
+          sig.trap :TERM do
+            stop(false)
+          end
+          sig.trap :INT do
+            stop(false)
+          end
+
+          sig.trap :QUIT do
+            stop(true)
+          end
+
+          sig.trap :USR1 do
+            stop(false)
+          end
+
+          sig.trap :HUP do
+            stop(true)
+          end
+
+          sig.trap :CONT do
+            stop(false)
+          end
+
+          sig.trap :WINCH do
+            stop(true)
+          end
+
+          sig.trap :USR2 do
+            logrotated
+          end
+        end
+      end
+    end
+
+  end
+end
+