perfectqueue 0.7.32 → 0.8.0

data/lib/perfectqueue/multiprocess/child_process_monitor.rb

@@ -0,0 +1,109 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Multiprocess
+
+    class ChildProcessMonitor
+      def initialize(log, pid, rpipe)
+        @log = log
+        @pid = pid
+        @rpipe = rpipe
+        @last_heartbeat = Time.now.to_i
+
+        @kill_start_time = nil
+        @last_kill_time = nil
+        @kill_immediate = false
+
+        @rbuf = ''
+      end
+
+      def check_heartbeat(limit)
+        @rpipe.read_nonblock(1024, @rbuf)
+        @last_heartbeat = Time.now.to_i
+        return true
+      rescue Errno::EINTR, Errno::EAGAIN
+        return Time.now.to_i - @last_heartbeat <= limit
+      end
+
+      def start_killing(immediate)
+        if immediate && !@kill_immediate
+          @kill_immediate = true  # escalation
+        elsif @kill_start_time
+          return
+        end
+
+        now = Time.now.to_i
+        kill_child(now, nil)
+        @kill_start_time = now
+      end
+
+      def try_join(kill_interval, graceful_kill_limit)
+        return nil unless @kill_start_time
+
+        begin
+          if Process.waitpid(@pid, Process::WNOHANG)
+            @log.info "Worker exited pid=#{@pid}"
+            return true
+          end
+        rescue Errno::ECHILD
+          # SIGCHLD is trapped in Worker#install_signal_handlers
+          @log.info "Worker exited pid=#{@pid}"
+          return true
+        end
+
+        # resend signal
+        now = Time.now.to_i
+        if @last_kill_time + kill_interval <= now
+          kill_child(now, graceful_kill_limit)
+        end
+
+        return false
+      end
+
+      def cleanup
+        @rpipe.close unless @rpipe.closed?
+      end
+
+      def send_signal(sig)
+        begin
+          Process.kill(sig, @pid)
+        rescue Errno::ESRCH, Errno::EPERM
+          # TODO log?
+        end
+      end
+
+      private
+      def kill_child(now, graceful_kill_limit)
+        begin
+          if @kill_immediate || (graceful_kill_limit && @kill_start_time + graceful_kill_limit < now)
+            @log.debug "sending SIGKILL to pid=#{@pid} for immediate stop"
+            Process.kill(:KILL, @pid)
+          else
+            @log.debug "sending SIGUSR1 to pid=#{@pid} for graceful stop"
+            Process.kill(:TERM, @pid)
+          end
+        rescue Errno::ESRCH, Errno::EPERM
+          # TODO log?
+        end
+        @last_kill_time = now
+      end
+    end
+
+  end
+end

data/lib/perfectqueue/multiprocess/fork_processor.rb

@@ -0,0 +1,164 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Multiprocess
+
+    class ForkProcessor
+      def initialize(runner, config)
+        @runner = runner
+
+        require 'fcntl'
+        @stop = false
+        @cpm = nil
+
+        restart(false, config)
+      end
+
+      def restart(immediate, config)
+        @child_heartbeat_limit = config[:child_heartbeat_limit] || 10.0
+        @child_kill_interval = config[:child_kill_interval] || 2.0
+        @child_graceful_kill_limit = config[:child_graceful_kill_limit] || nil
+        @log = config[:logger]
+        @config = config  # for child process
+
+        if c = @cpm
+          c.start_killing(immediate)
+        end
+      end
+
+      def stop(immediate)
+        @stop = true
+        if c = @cpm
+          c.start_killing(immediate)
+        end
+        self
+      end
+
+      def keepalive
+        if @stop
+          try_join
+          return
+        end
+
+        if c = @cpm
+          begin
+            # receive heartbeat
+            keptalive = c.check_heartbeat(@child_heartbeat_limit)
+            unless keptalive
+              @log.error "Heartbeat broke out. Restarting child process."
+              c.start_killing(false)
+            end
+          rescue EOFError
+            @log.error "Heartbeat pipe is closed. Restarting child process."
+            c.start_killing(true)
+          rescue
+            @log.error "Unknown error: #{$!.class}: #{$!}. Restarting child process."
+            $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+            c.start_killing(false)
+          end
+
+          try_join
+        end
+
+        unless @cpm
+          begin
+            @cpm = fork_child
+          rescue
+            @log.error "Failed to fork child process: #{$!.class}: #{$!}"
+            $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+          end
+        end
+
+        nil
+      end
+
+      def join
+        while !try_join
+          sleep (@child_kill_interval+1) / 2  # TODO
+        end
+        self
+      end
+
+      def logrotated
+        if c = @cpm
+          c.send_signal(:CONT)
+        end
+      end
+
+      private
+      def try_join
+        unless @cpm
+          return true
+        end
+        if @cpm.try_join(@child_kill_interval, @child_graceful_kill_limit)
+          @cpm.cleanup
+          @cpm = nil
+          return true
+        else
+          return false
+        end
+      end
+
+      def ensure_fork
+        unless @cpm
+          @cpm = fork_child
+        end
+      end
+
+      INTER_FORK_LOCK = Mutex.new
+
+      def fork_child
+        @runner.before_fork if @runner.respond_to?(:before_fork)  # TODO exception handling
+
+        INTER_FORK_LOCK.lock
+        begin
+          rpipe, wpipe = IO.pipe
+          rpipe.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+          wpipe.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+        ensure
+          INTER_FORK_LOCK.unlock
+        end
+
+        pid = fork do
+          #STDIN.close
+          # pass-through STDOUT
+          # pass-through STDERR
+          rpipe.close
+
+          @runner.after_fork if @runner.respond_to?(:after_fork)
+
+          begin
+            ChildProcess.run(@runner, @config, wpipe)
+          ensure
+            @runner.after_child_end if @runner.respond_to?(:after_child_end)  # TODO exception handling
+          end
+
+          exit! 0
+        end
+
+        @log.info "Worker process started. pid=#{pid}"
+
+        wpipe.close
+
+        ChildProcessMonitor.new(@log, pid, rpipe)
+      end
+    end
+
+  end
+end

data/lib/perfectqueue/multiprocess/thread_processor.rb

@@ -0,0 +1,123 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Multiprocess
+
+    class ThreadProcessor
+      def initialize(runner, config)
+        @runner = runner
+
+        @running_flag = BlockingFlag.new
+        @finish_flag = BlockingFlag.new
+
+        @tm = TaskMonitor.new(config, method(:child_heartbeat))
+
+        restart(false, config)
+      end
+
+      def run
+        @tm.start
+        @running_flag.set_region do
+          until @finish_flag.set?
+            run_loop
+          end
+        end
+        @tm.join
+      ensure
+        @thread = nil
+      end
+
+      def join
+        while t = @thread
+          t.join
+        end
+      end
+
+      def keepalive
+        unless @thread
+          @thread = Thread.new(&method(:run))
+        end
+      end
+
+      def restart(immediate, config)
+        @poll_interval = config[:poll_interval] || 1.0
+        @log = config[:logger]
+        @config = config
+
+        @tm.stop_task(immediate)
+
+        @finish_flag.set_region do
+          @running_flag.wait while @running_flag.set?
+        end
+      end
+
+      def stop(immediate)
+        @log.info immediate ? "Stopping worker thread immediately" : "Stopping worker thread gracefully"
+        @tm.stop_task(immediate)
+        @finish_flag.set!
+      end
+
+      def logrotated
+        # do nothing
+      end
+
+      private
+      def child_heartbeat
+        # do nothing
+      end
+
+      def run_loop
+        PerfectQueue.open(@config) {|queue|
+          until @finish_flag.set?
+            task = queue.poll
+            if task
+              process(task)
+            else
+              @finish_flag.wait(@poll_interval)
+            end
+          end
+        }
+      rescue
+        @log.error "Unknown error #{$!.class}: #{$!}. Exiting worker pid=#{Process.pid}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+      ensure
+        @tm.stop
+      end
+
+      def process(task)
+        @log.info "acquired task: #{task.inspect}"
+        begin
+          r = @runner.new(task)
+          @tm.set_task(task, r)
+          begin
+            r.run
+          ensure
+            @tm.task_finished(task)
+          end
+        rescue
+          @log.error "process failed: #{$!.class}: #{$!}"
+          $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+          raise  # force exit
+        end
+      end
+    end
+
+  end
+end
+

data/lib/perfectqueue/queue.rb

@@ -0,0 +1,58 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  class Queue
+    include Model
+
+    def initialize(client)
+      super(client)
+    end
+
+    def [](key)
+      Task.new(@client, key)
+    end
+
+    def each(options={}, &block)
+      @client.list(options, &block)
+    end
+
+    include Enumerable
+
+    def poll(options={})
+      options = options.merge({:max_acquire=>1})
+      if acquired = poll_multi(options)
+        return acquired[0]
+      end
+      return nil
+    end
+
+    def poll_multi(options={})
+      @client.acquire(options)
+    end
+
+    def submit(key, type, data, options={})
+      @client.submit(key, type, data, options)
+    end
+
+    def close
+      client.close
+    end
+  end
+end
+