resque-state 1.0.0

data/lib/resque/plugins/state/hash.rb

@@ -0,0 +1,326 @@
+require 'securerandom'
+
+module Resque
+  module Plugins
+    module State
+      # Resque::Plugins::State::Hash is a Hash object that has helper methods for dealing with
+      # the common status attributes. It also has a number of class methods for
+      # creating/updating/retrieving status objects from Redis
+      class Hash < ::Hash
+        # Create a status, generating a new UUID, passing the message to the status
+        # Returns the UUID of the new status.
+        def self.create(uuid, *messages)
+          set(uuid, *messages)
+          redis.zadd(set_key, Time.now.to_i, uuid)
+          redis.zremrangebyscore(set_key, 0, Time.now.to_i - @expire_in) if @expire_in
+          uuid
+        end
+
+        # Get a status by UUID. Returns a Resque::Plugins::State::Hash
+        def self.get(uuid)
+          val = redis.get(status_key(uuid))
+          val ? Resque::Plugins::State::Hash.new(uuid, decode(val)) : nil
+        end
+
+        # Get multiple statuses by UUID. Returns array of Resque::Plugins::State::Hash
+        def self.mget(uuids)
+          return [] if uuids.empty?
+          status_keys = uuids.map { |u| status_key(u) }
+          vals = redis.mget(*status_keys)
+
+          uuids.zip(vals).map do |uuid, val|
+            val ? Resque::Plugins::State::Hash.new(uuid, decode(val)) : nil
+          end
+        end
+
+        # set a status by UUID. <tt>messages</tt> can be any number of strings or hashes
+        # that are merged in order to create a single status.
+        def self.set(uuid, *messages)
+          val = Resque::Plugins::State::Hash.new(uuid, *messages)
+          redis.set(status_key(uuid), encode(val))
+          redis.expire(status_key(uuid), expire_in) if expire_in
+          val
+        end
+
+        # clear statuses from redis passing an optional range. See `statuses` for info
+        # about ranges
+        def self.clear(range_start = nil, range_end = nil)
+          status_ids(range_start, range_end).each do |id|
+            remove(id)
+          end
+        end
+
+        def self.clear_completed(range_start = nil, range_end = nil)
+          status_ids(range_start, range_end).select do |id|
+            if get(id).completed?
+              remove(id)
+              true
+            else
+              false
+            end
+          end
+        end
+
+        def self.clear_failed(range_start = nil, range_end = nil)
+          status_ids(range_start, range_end).select do |id|
+            if get(id).failed?
+              remove(id)
+              true
+            else
+              false
+            end
+          end
+        end
+
+        def self.remove(uuid)
+          redis.del(status_key(uuid))
+          redis.zrem(set_key, uuid)
+        end
+
+        # Return <tt>num</tt> Resque::Plugins::State::Hash objects in reverse chronological order.
+        # By default returns the entire set.
+        # @param [Numeric] range_start The optional starting range
+        # @param [Numeric] range_end The optional ending range
+        # @example retuning the last 20 statuses
+        #   Resque::Plugins::State::Hash.statuses(0, 20)
+        def self.statuses(range_start = nil, range_end = nil)
+          ids = status_ids(range_start, range_end)
+          mget(ids).compact || []
+        end
+
+        # Return the <tt>num</tt> most recent status/job UUIDs in reverse chronological order.
+        def self.status_ids(range_start = nil, range_end = nil)
+          if range_end && range_start
+            # Because we want a reverse chronological order, we need to get a range starting
+            # by the higest negative number. The ordering is transparent from the API user's
+            # perspective so we need to convert the passed params
+            (redis.zrevrange(set_key, range_start.abs, (range_end || 1).abs) || [])
+          else
+            # Because we want a reverse chronological order, we need to get a range starting
+            # by the higest negative number.
+            redis.zrevrange(set_key, 0, -1) || []
+          end
+        end
+
+        # Kill the job at UUID on its next iteration this works by adding the UUID to a
+        # kill list (a.k.a. a list of jobs to be killed. Each iteration the job checks
+        # if it _should_ be killed by calling <tt>tick</tt> or <tt>at</tt>. If so, it raises
+        # a <tt>Resque::Plugins::State::Killed</tt> error and sets the status to 'killed'.
+        def self.kill(uuid)
+          redis.sadd(kill_key, uuid)
+        end
+
+        # Remove the job at UUID from the kill list
+        def self.killed(uuid)
+          redis.srem(kill_key, uuid)
+        end
+
+        # Return the UUIDs of the jobs on the kill list
+        def self.kill_ids
+          redis.smembers(kill_key)
+        end
+
+        # Kills <tt>num</tt> jobs within range starting with the most recent first.
+        # By default kills all jobs.
+        # Note that the same conditions apply as <tt>kill</tt>, i.e. only jobs that check
+        # on each iteration by calling <tt>tick</tt> or <tt>at</tt> are eligible to killed.
+        # @param [Numeric] range_start The optional starting range
+        # @param [Numeric] range_end The optional ending range
+        # @example killing the last 20 submitted jobs
+        #   Resque::Plugins::State::Hash.killall(0, 20)
+        def self.killall(range_start = nil, range_end = nil)
+          status_ids(range_start, range_end).collect do |id|
+            kill(id)
+          end
+        end
+
+        # Check whether a job with UUID is on the kill list
+        def self.should_kill?(uuid)
+          redis.sismember(kill_key, uuid)
+        end
+
+        # pause the job at UUID on its next iteration this works by adding the UUID to a
+        # pause list (a.k.a. a list of jobs to be pauseed. Each iteration the job checks
+        # if it _should_ be pauseed by calling <tt>tick</tt> or <tt>at</tt>. If so, it sleeps
+        # for 10 seconds before checking again if it should continue sleeping
+        def self.pause(uuid)
+          redis.sadd(pause_key, uuid)
+        end
+
+        # Remove the job at UUID from the pause list
+        def self.unpause(uuid)
+          redis.srem(pause_key, uuid)
+        end
+
+        # Return the UUIDs of the jobs on the pause list
+        def self.pause_ids
+          redis.smembers(pause_key)
+        end
+
+        # pauses <tt>num</tt> jobs within range starting with the most recent first.
+        # By default pauses all jobs.
+        # Note that the same conditions apply as <tt>pause</tt>, i.e. only jobs that check
+        # on each iteration by calling <tt>tick</tt> or <tt>at</tt> are eligible to pauseed.
+        # @param [Numeric] range_start The optional starting range
+        # @param [Numeric] range_end The optional ending range
+        # @example pauseing the last 20 submitted jobs
+        #   Resque::Plugins::State::Hash.pauseall(0, 20)
+        def self.pauseall(range_start = nil, range_end = nil)
+          status_ids(range_start, range_end).collect do |id|
+            pause(id)
+          end
+        end
+
+        # Check whether a job with UUID is on the pause list
+        def self.should_pause?(uuid)
+          redis.sismember(pause_key, uuid)
+        end
+
+        # The time in seconds that jobs and statuses should expire from Redis (after
+        # the last time they are touched/updated)
+        class << self
+          attr_reader :expire_in
+        end
+
+        # Set the <tt>expire_in</tt> time in seconds
+        def self.expire_in=(seconds)
+          @expire_in = seconds.nil? ? nil : seconds.to_i
+        end
+
+        def self.status_key(uuid)
+          "status:#{uuid}"
+        end
+
+        def self.set_key
+          '_statuses'
+        end
+
+        def self.kill_key
+          '_kill'
+        end
+
+        def self.pause_key
+          '_pause'
+        end
+
+        def self.generate_uuid
+          SecureRandom.hex.to_s
+        end
+
+        def self.hash_accessor(name, options = {})
+          options[:default] ||= nil
+          coerce = options[:coerce] ? ".#{options[:coerce]}" : ''
+          module_eval <<-EOT
+          def #{name}
+            value = (self['#{name}'] ? self['#{name}']#{coerce} : #{options[:default].inspect})
+            yield value if block_given?
+            value
+          end
+
+          def #{name}=(value)
+            self['#{name}'] = value
+          end
+
+          def #{name}?
+            !!self['#{name}']
+          end
+          EOT
+        end
+
+        # Proxy deprecated methods directly back to Resque itself.
+        class << self
+          [:redis, :encode, :decode].each do |method|
+            define_method(method) { |*args| Resque.send(method, *args) }
+          end
+        end
+
+        hash_accessor :uuid
+        hash_accessor :name
+        hash_accessor :status
+        hash_accessor :message
+        hash_accessor :time
+        hash_accessor :options
+
+        hash_accessor :num
+        hash_accessor :total
+
+        # Create a new Resque::Plugins::State::Hash object. If multiple arguments are passed
+        # it is assumed the first argument is the UUID and the rest are status objects.
+        # All arguments are subsequentily merged in order. Strings are assumed to
+        # be messages.
+        def initialize(*args)
+          super nil
+          base_status = {
+            'time' => Time.now.to_i,
+            'status' => Resque::Plugins::State::STATUS_QUEUED
+          }
+          base_status['uuid'] = args.shift if args.length > 1
+          status_hash = args.inject(base_status) do |final, m|
+            m = { 'message' => m } if m.is_a?(String)
+            final.merge(m || {})
+          end
+          replace(status_hash)
+        end
+
+        # calculate the % completion of the job based on <tt>status</tt>, <tt>num</tt>
+        # and <tt>total</tt>
+        def pct_complete
+          if completed?
+            100
+          elsif queued?
+            0
+          elsif failed?
+            0
+          else
+            if total.nil?
+              t = 1
+            else t = total
+            end
+            (((num || 0).to_f / t.to_f) * 100).to_i
+          end
+        end
+
+        # Return the time of the status initialization. If set returns a <tt>Time</tt>
+        # object, otherwise returns nil
+        def time
+          time? ? Time.at(self['time']) : nil
+        end
+
+        Resque::Plugins::State::STATUSES.each do |status|
+          define_method("#{status}?") do
+            self['status'] === status
+          end
+        end
+
+        # Can the job be killed? failed, completed, and killed jobs can't be
+        # killed, for obvious reasons
+        def killable?
+          !failed? && !completed? && !killed?
+        end
+
+        # Can the job be paused? failed, completed, paused, and killed jobs can't be
+        # paused, for obvious reasons
+        def pausable?
+          !failed? && !completed? && !killed? && !paused?
+        end
+
+        unless method_defined?(:to_json)
+          def to_json(*_args)
+            json
+          end
+        end
+
+        # Return a JSON representation of the current object.
+        def json
+          h = dup
+          h['pct_complete'] = pct_complete
+          self.class.encode(h)
+        end
+
+        def inspect
+          "#<Resque::Plugins::State::Hash #{super}>"
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/state.rb

@@ -0,0 +1,289 @@
+module Resque
+  module Plugins
+    # Resque::Plugins::State is a module your jobs will include.
+    # It provides helper methods for updating the status/etc from within an
+    # instance as well as class methods for creating and queuing the jobs.
+    #
+    # All you have to do to get this functionality is include
+    # Resque::Plugins::State and then implement a <tt>perform<tt> method.
+    #
+    # For example
+    #
+    #       class ExampleJob
+    #         include Resque::Plugins::State
+    #
+    #         def perform
+    #           num = options['num']
+    #           i = 0
+    #           while i < num
+    #             i += 1
+    #             at(i, num)
+    #           end
+    #           completed("Finished!")
+    #         end
+    #
+    #       end
+    #
+    # This job would iterate num times updating the status as it goes. At the
+    # end we update the status telling anyone listening to this job that its
+    # complete.
+    module State
+      VERSION = '1.0.0'.freeze
+
+      STATUS_QUEUED = 'queued'.freeze
+      STATUS_WORKING = 'working'.freeze
+      STATUS_COMPLETED = 'completed'.freeze
+      STATUS_FAILED = 'failed'.freeze
+      STATUS_KILLED = 'killed'.freeze
+      STATUS_PAUSED = 'paused'.freeze
+      STATUSES = [
+        STATUS_QUEUED,
+        STATUS_WORKING,
+        STATUS_COMPLETED,
+        STATUS_FAILED,
+        STATUS_KILLED,
+        STATUS_PAUSED
+      ].freeze
+
+      autoload :Hash, 'resque/plugins/state/hash'
+
+      # The error class raised when a job is killed
+      class Killed < RuntimeError; end
+      class NotANumber < RuntimeError; end
+
+      attr_reader :uuid, :options
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        # The default queue is :statused, this can be ovveridden in the specific
+        # job class to put the jobs on a specific worker queue
+        def queue
+          :statused
+        end
+
+        # used when displaying the Job in the resque-web UI and identifiyng the
+        # job type by status. By default this is the name of the job class, but
+        # can be overidden in the specific job class to present a more user
+        # friendly job name
+        def name
+          to_s
+        end
+
+        # Create is the primary method for adding jobs to the queue. This would
+        # be called on the job class to create a job of that type. Any options
+        # passed are passed to the Job instance as a hash of options. It returns
+        # the UUID of the job.
+        #
+        # == Example:
+        #
+        #       class ExampleJob
+        #         include Resque::Plugins::State
+        #
+        #         def perform
+        #           job_status "Hey I'm a job num #{options['num']}"
+        #         end
+        #
+        #       end
+        #
+        #       job_id = ExampleJob.create(:num => 100)
+        #
+        def create(options = {})
+          enqueue(self, options)
+        end
+
+        # Adds a job of type <tt>klass<tt> to the queue with <tt>options<tt>.
+        #
+        # Returns the UUID of the job if the job was queued, or nil if the job
+        # was rejected by a before_enqueue hook.
+        def enqueue(klass, options = {})
+          enqueue_to(Resque.queue_from_class(klass) || queue, klass, options)
+        end
+
+        # Adds a job of type <tt>klass<tt> to a specified queue with
+        # <tt>options<tt>.
+        #
+        # Returns the UUID of the job if the job was queued, or nil if the job
+        # was rejected by a before_enqueue hook.
+        def enqueue_to(queue, klass, options = {})
+          uuid = Resque::Plugins::State::Hash.generate_uuid
+          Resque::Plugins::State::Hash.create uuid, options: options
+
+          if Resque.enqueue_to(queue, klass, uuid, options)
+            uuid
+          else
+            Resque::Plugins::State::Hash.remove(uuid)
+            nil
+          end
+        end
+
+        # Removes a job of type <tt>klass<tt> from the queue.
+        #
+        # The initially given options are retrieved from the status hash.
+        # (Resque needs the options to find the correct queue entry)
+        def dequeue(klass, uuid)
+          status = Resque::Plugins::State::Hash.get(uuid)
+          Resque.dequeue(klass, uuid, status.options)
+        end
+
+        # This is the method called by Resque::Worker when processing jobs. It
+        # creates a new instance of the job class and populates it with the uuid
+        # and options.
+        #
+        # You should not override this method, rahter the <tt>perform</tt>
+        # instance method.
+        def perform(uuid = nil, options = {})
+          uuid ||= Resque::Plugins::State::Hash.generate_uuid
+          instance = new(uuid, options)
+          instance.safe_perform!
+          instance
+        end
+
+        # Wrapper API to forward a Resque::Job creation API call into a
+        # Resque::Plugins::State call.
+        # This is needed to be used with resque scheduler
+        # http://github.com/bvandenbos/resque-scheduler
+        def scheduled(queue, _klass, *args)
+          enqueue_to(queue, self, *args)
+        end
+      end
+
+      # Create a new instance with <tt>uuid</tt> and <tt>options</tt>
+      def initialize(uuid, options = {})
+        @uuid    = uuid
+        @options = options
+        @logger = Resque.logger
+      end
+
+      # Run by the Resque::Worker when processing this job. It wraps the
+      # <tt>perform</tt> method ensuring that the final status of the job is set
+      # regardless of error. If an error occurs within the job's work, it will
+      # set the status as failed and re-raise the error.
+      def safe_perform!
+        job_status('status' => STATUS_WORKING)
+        messages = ['Job starting']
+        @logger.info("#{@uuid}: #{messages.join(' ')}")
+        perform
+        if status && status.failed?
+          on_failure(status.message) if respond_to?(:on_failure)
+          return
+        elsif status && !status.completed?
+          completed
+        end
+        on_success if respond_to?(:on_success)
+      rescue Killed
+        Resque::Plugins::State::Hash.killed(uuid)
+        on_killed if respond_to?(:on_killed)
+      rescue => e
+        failed("The task failed because of an error: #{e}")
+        raise e unless respond_to?(:on_failure)
+        on_failure(e)
+      end
+
+      # Set the jobs status. Can take an array of strings or hashes that are
+      # merged (in order) into a final status hash.
+      def status=(new_status)
+        Resque::Plugins::State::Hash.set(uuid, *new_status)
+      end
+
+      # get the Resque::Plugins::State::Hash object for the current uuid
+      def status
+        Resque::Plugins::State::Hash.get(uuid)
+      end
+
+      def name
+        "#{self.class.name}(#{options.inspect unless options.empty?})"
+      end
+
+      # Checks against the kill list if this specific job instance should be
+      # killed on the next iteration
+      def should_kill?
+        Resque::Plugins::State::Hash.should_kill?(uuid)
+      end
+
+      # Checks against the pause list if this specific job instance should be
+      # paused on the next iteration
+      def should_pause?
+        Resque::Plugins::State::Hash.should_pause?(uuid)
+      end
+
+      # set the status of the job for the current itteration. <tt>num</tt> and
+      # <tt>total</tt> are passed to the status as well as any messages.
+      # This will kill the job if it has been added to the kill list with
+      # <tt>Resque::Plugins::State::Hash.kill()</tt>
+      def at(num, total, *messages)
+        if total.to_f <= 0.0
+          raise(NotANumber,
+                "Called at() with total=#{total} which is not a number")
+        end
+        tick({
+               'num' => num,
+               'total' => total
+             }, *messages)
+      end
+
+      # sets the status of the job for the current itteration. You should use
+      # the <tt>at</tt> method if you have actual numbers to track the iteration
+      # count. This will kill or pause the job if it has been added to either
+      # list with <tt>Resque::Plugins::State::Hash.pause()</tt> or
+      # <tt>Resque::Plugins::State::Hash.kill()</tt> respectively
+      def tick(*messages)
+        kill! if should_kill?
+        if should_pause?
+          pause!
+        else
+          job_status({ 'status' => STATUS_WORKING }, *messages)
+          @logger.info("#{@uuid}: #{messages.join(' ')}")
+        end
+      end
+
+      # set the status to 'failed' passing along any additional messages
+      def failed(*messages)
+        job_status({ 'status' => STATUS_FAILED }, *messages)
+        @logger.error("#{@uuid}: #{messages.join(' ')}")
+      end
+
+      # set the status to 'completed' passing along any addional messages
+      def completed(*messages)
+        job_status({
+                     'status' => STATUS_COMPLETED,
+                     'message' => "Completed at #{Time.now}"
+                   }, *messages)
+        @logger.info("#{@uuid}: #{messages.join(' ')}")
+      end
+
+      # kill the current job, setting the status to 'killed' and raising
+      # <tt>Killed</tt>
+      def kill!
+        messages = ["Killed at #{Time.now}"]
+        job_status('status' => STATUS_KILLED,
+                   'message' => messages[0])
+        @logger.error("#{@uuid}: #{messages.join(' ')}")
+        raise Killed
+      end
+
+      # pause the current job, setting the status to 'paused' and sleeping 10
+      # seconds
+      def pause!
+        Resque::Plugins::State::Hash.pause(uuid)
+        messages = ["Paused at #{Time.now}"]
+        job_status('status' => STATUS_PAUSED,
+                   'message' => messages[0])
+        raise Killed if @testing # Don't loop or complete during testing
+        @logger.info("#{@uuid}: #{messages.join(' ')}")
+        while should_pause?
+          kill! if should_kill?
+          sleep 10
+        end
+      end
+
+      private
+
+      def job_status(*args)
+        self.status = [status, { 'name' => name }, args].flatten
+      end
+    end
+  end
+end

data/lib/resque/server/views/state.erb

@@ -0,0 +1,91 @@
+<%= status_view :state_styles, :layout => false %>
+
+<h1 class='wi'>State <%= @status.uuid %></h1>
+<p class='intro'>Viewing a specific job created with Resque::Plugins::State. <a href="<%= u(:statuses) %>">Return to the list of statuses</a></p>
+
+<div class="status-holder" rel="<%= @status.status %>" id="status_<%= @status.uuid %>">
+  <h2>Overview</h2>
+  <div class="status-progress">
+    <div class="status-progress-bar status-<%= @status.status %>" style="width: <%= @status.pct_complete %>%;"></div>
+    <p><%= @status.pct_complete %>%</p>
+  </div>
+  <div class="status-message"><%= @status.message %></div>
+  <div class="status-time"><%= @status.time? ? @status.time : 'Not started' %></div>
+
+  <h2>Details</h2>
+  <div class="status-details">
+    <table class="vertically-top">
+      <thead>
+        <tr>
+          <th>Key</th>
+          <th>Value</th>
+        </tr>
+      </thead>
+      <tbody class="status-details-body">
+      </tbody>
+    </table>
+  </div>
+</div>
+
+<script type="text/javascript" charset="utf-8">
+  jQuery(function($) {
+
+    // itterate over the holders
+    $('.status-holder').each(function() {
+      checkStatus($(this));
+    });
+
+    function checkStatus($status) {
+      var status_id = $status.attr('id').replace('status_', '');
+      $.getJSON('<%= u(:statuses) %>/' + status_id + '.js', function(json) {
+        if (json) {
+          var pct = "0%";
+          if (json.pct_complete) {
+            var pct = json.pct_complete + "%";
+          }
+          $status.find('.status-progress-bar').animate({width: pct});
+          $status.find('.status-progress p').text(pct)
+          if (json.message) {
+            $status.find('.status-message').html(json.message)
+          }
+          if (json.status) {
+            $status
+              .attr('rel', json.status)
+              .find('.status-progress-bar')
+                .attr('class', '')
+                .addClass('status-progress-bar status-' + json.status);
+          }
+          if (json.time) {
+            $status.find('.status-time').text(new Date(json.time * 1000).toString())
+          }
+
+          var $details = $status.find('.status-details-body');
+          $details.empty();
+
+          for (key in json) {
+            var $row = $("<tr>").appendTo($details);
+            $("<td>").text(key).appendTo($row);
+            $("<td>").text(printValue(key, json[key])).appendTo($row);
+          }
+        };
+        var status = $status.attr('rel');
+        if (status == '<%= Resque::Plugins::State::STATUS_WORKING %>' || status == '<%= Resque::Plugins::State::STATUS_QUEUED %>' || status == "") {
+          setTimeout(function() {
+            checkStatus($status)
+          }, 1500);
+        }
+      });
+    };
+
+    function printValue(key, value) {
+      if (/(^|_)time$/.test(key) && typeof value == 'number') {
+        var time = new Date();
+        time.setTime(value * 1000);
+        return time.toUTCString();
+      } else {
+        return JSON.stringify(value, null, " ");
+      }
+    }
+
+  });
+</script>