perfectqueue 0.7.32 → 0.8.0

data/lib/perfectqueue/runner.rb

@@ -0,0 +1,39 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  class Runner
+    def initialize(task)
+      @task = task
+    end
+
+    attr_accessor :task
+
+    def queue
+      Queue.new(task.client)
+    end
+
+    #def run
+    #end
+
+    def kill(reason)
+      # do nothing
+    end
+  end
+end
+

data/lib/perfectqueue/signal_queue.rb

@@ -0,0 +1,112 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+
+  class SignalQueue
+    def self.start(&block)
+      st = new(&block)
+      st.start
+      return st
+    end
+
+    def initialize(&block)
+      require 'thread'
+
+      @handlers = {}
+      @queue = []
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+
+      block.call(self) if block
+    end
+
+    def trap(sig, &block)
+      sig = sig.to_sym
+      old = @handlers[sig]
+
+      Kernel.trap(sig) do
+        enqueue(sig)
+      end
+
+      @handlers[sig] = block
+      old
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def join
+      @thread.join
+    end
+
+    def stop
+      enqueue(nil)
+    end
+
+    def shutdown
+      stop
+      join
+    end
+
+    def run
+      finished = false
+      until finished
+        h = nil
+        @mutex.synchronize do
+          while @queue.empty?
+            @cond.wait(@mutex)
+          end
+          sig = @queue.shift
+          if sig == nil
+            finished = true
+          else
+            h = @handlers[sig]
+          end
+        end
+
+        begin
+          h.call if h
+        rescue
+          STDERR.print "#{$!}\n"
+          $!.backtrace.each {|bt|
+            STDERR.print "\t#{bt}\n"
+            STDERR.flush
+          }
+        end
+      end
+    end
+
+    private
+    def enqueue(sig)
+      if Thread.current == self
+        @queue << sig
+        if @mutex.locked?
+          @cond.signal
+        end
+      else
+        @mutex.synchronize do
+          @queue << sig
+          @cond.signal
+        end
+      end
+    end
+  end
+
+end

data/lib/perfectqueue/task.rb

@@ -0,0 +1,103 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  class Task
+    include Model
+
+    def initialize(client, key)
+      super(client)
+      @key = key
+    end
+
+    attr_reader :key
+
+    def cancel_request!(options={})
+      @client.cancel_request(@key, options)
+    end
+
+    def force_finish!(options={})
+      @client.force_finish(@key, options)
+    end
+
+    def metadata(options={})
+      @client.get_task_metadata(@key, options)
+    end
+
+    def exists?(options={})
+      metadata(options)
+      true
+    rescue NotFoundError
+      false
+    end
+
+    def preempt(options={})
+      @client.preempt(@key, options)
+    end
+
+    def inspect
+      "#<#{self.class} @key=#{@key.inspect}>"
+    end
+  end
+
+  class TaskWithMetadata < Task
+    def initialize(client, key, attributes)
+      super(client, key)
+      @attributes = attributes
+    end
+
+    def inspect
+      "#<#{self.class} @key=#{@key.inspect} @attributes=#{@attributes.inspect}>"
+    end
+
+    include TaskMetadataAccessors
+  end
+
+  class AcquiredTask < TaskWithMetadata
+    def initialize(client, key, attributes, task_token)
+      super(client, key, attributes)
+      @task_token = task_token
+    end
+
+    def heartbeat!(options={})
+      @client.heartbeat(@task_token, options)
+    end
+
+    def finish!(options={})
+      @client.finish(@task_token, options)
+    end
+
+    def release!(options={})
+      @client.release(@task_token, options)
+    end
+
+    def retry!(options={})
+      @client.retry(@task_token, options)
+    end
+
+    #def to_json
+    #  [@key, @task_token, @attributes].to_json
+    #end
+
+    #def self.from_json(data, client)
+    #  key, task_token, attributes = JSON.load(data)
+    #  new(client, key, attributes, task_token)
+    #end
+  end
+end
+

data/lib/perfectqueue/task_metadata.rb

@@ -0,0 +1,98 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module TaskMetadataAccessors
+    attr_reader :attributes
+
+    def type
+      @attributes[:type]
+    end
+
+    def data
+      @attributes[:data]
+    end
+
+    def status
+      @attributes[:status]
+    end
+
+    def message
+      @attributes[:message]
+    end
+
+    def user
+      @attributes[:user]
+    end
+
+    def created_at
+      if t = @attributes[:created_at]
+        return Time.at(t)
+      else
+        return nil
+      end
+    end
+
+    def timeout
+      if t = @attributes[:timeout]
+        return Time.at(t)
+      else
+        return nil
+      end
+    end
+
+    def finished?
+      status == TaskStatus::FINISHED
+    end
+
+    def running?
+      status == TaskStatus::RUNNING
+    end
+
+    def waiting?
+      status == TaskStatus::WAITING
+    end
+
+    def running?
+      status == TaskStatus::RUNNING
+    end
+
+    def cancel_requested?
+      status == TaskStatus::CANCEL_REQUESTED
+    end
+  end
+
+  class TaskMetadata
+    def initialize(client, key, attributes)
+      super(client)
+      @key = key
+      @attributes = attributes
+    end
+
+    def task
+      Task.new(@client, @key)
+    end
+
+    def inspect
+      "#<#{self.class} @key=#{@key.inspect} @attributes=#{@attributes.inspect}>"
+    end
+
+    include TaskMetadataAccessors
+  end
+end
+

data/lib/perfectqueue/task_monitor.rb

@@ -0,0 +1,189 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+
+  class TaskMonitor
+    def initialize(config, child_heartbeat=nil)
+      @config = config
+      @log = config[:logger]
+      @child_heartbeat = child_heartbeat || Proc.new {}
+
+      @child_heartbeat_interval = (@config[:child_heartbeat_interval] || 2).to_i
+      @task_heartbeat_interval = (@config[:task_heartbeat_interval] || 2).to_i
+      @last_child_heartbeat = Time.now.to_i
+      @last_task_heartbeat = Time.now.to_i
+
+      @task = nil
+
+      @mutex = Monitor.new  # support recursive lock
+      @cond = @mutex.new_cond
+      @finished = false
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def stop
+      @finished = true
+      @mutex.synchronize {
+        @cond.broadcast
+      }
+    end
+
+    def join
+      @thread.join
+    end
+
+    def set_task(task, runner)
+      task.extend(TaskMonitorHook)
+      task.task_monitor = self
+      task.runner = runner
+      @mutex.synchronize {
+        @task = task
+        @last_task_heartbeat = Time.now.to_i
+        @heartbeat_message = nil
+      }
+    end
+
+    # callback
+    def set_heartbeat_message(task, message)
+      @mutex.synchronize {
+        if task == @task
+          @heartbeat_message = message
+        end
+      }
+    end
+
+    def stop_task(immediate)
+      if immediate
+        kill_task ImmediateProcessStopError.new('immediate stop requested')
+      else
+        kill_task GracefulProcessStopError.new('graceful stop requested')
+      end
+    end
+
+    def kill_task(reason)
+      @mutex.synchronize {
+        if task = @task
+          begin
+            task.runner.kill(reason)  # may recursive lock
+          rescue
+            @log.error "failed to kill task: #{$!.class}: #{$!}"
+            $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+            raise # force exit
+          end
+        end
+      }
+    end
+
+    # callback
+    def task_finished(task, &block)
+      @mutex.synchronize {
+        ret = block.call if block
+        if task == @task
+          @task = nil
+        end
+        ret
+      }
+    end
+
+    def run
+      @mutex.synchronize {
+        now = Time.now.to_i
+
+        until @finished
+          next_child_heartbeat = @last_child_heartbeat + @child_heartbeat_interval
+
+          if @task
+            next_task_heartbeat = @last_task_heartbeat + @task_heartbeat_interval
+            next_time = [next_child_heartbeat, next_task_heartbeat].min
+          else
+            next_time = next_child_heartbeat
+          end
+
+          next_wait = [1, next_time - now].max
+          @cond.wait(next_wait) if next_wait > 0  # TODO timeout doesn't work?
+
+          now = Time.now.to_i
+          if @task && next_task_heartbeat && now <= next_task_heartbeat
+            task_heartbeat
+            @last_task_heartbeat = now
+          end
+
+          if now <= next_child_heartbeat
+            @child_heartbeat.call  # will recursive lock
+            @last_child_heartbeat = now
+          end
+        end
+      }
+    rescue
+      @log.error "Unknown error #{$!.class}: #{$!}. Exiting worker pid=#{Process.pid}"
+      $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+    end
+
+    private
+    def task_heartbeat
+      @task.heartbeat! :message => @heartbeat_message
+      @heartbeat_message = nil
+    rescue TaskError
+      # finished, cancel_requested, preempted, etc.
+      kill_task($!)
+    end
+  end
+
+  module TaskMonitorHook
+    attr_accessor :task_monitor
+    attr_accessor :runner
+
+    def heartbeat_message=(message)
+      @heartbeat_message = message
+      @task_monitor.set_heartbeat_message(self, message)
+      message
+    end
+
+    attr_reader :heartbeat_message
+
+    def finish!(*args, &block)
+      @task_monitor.task_finished(self) {
+        super(*args, &block)
+      }
+    end
+
+    def release!(*args, &block)
+      @task_monitor.task_finished(self) {
+        super(*args, &block)
+      }
+    end
+
+    def retry!(*args, &block)
+      @task_monitor.task_finished(self) {
+        super(*args, &block)
+      }
+    end
+
+    def cancel_request!(*args, &block)
+      @task_monitor.task_finished(self) {
+        super(*args, &block)
+      }
+    end
+  end
+
+end
+