sidekiq_status 1.0.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.3-p194-perf@sidekiq_status --create

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+#  - jruby-19mode # JRuby in 1.9 mode
+#  - rbx-19mode
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec

data/.yardopts

@@ -0,0 +1 @@
+-m markdown

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq_status.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Artem Ignatyev
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,159 @@
+# SidekiqStatus
+
+[![Build Status](https://secure.travis-ci.org/cryo28/sidekiq_status.png)](http://travis-ci.org/cryo28/sidekiq_status)
+[![Dependency Status](https://gemnasium.com/cryo28/sidekiq_status.png)](https://gemnasium.com/cryo28/sidekiq_status)
+
+Sidekiq extension to track job execution statuses and returning job results back to the client in a convenient manner
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq_status', :git => 'git@github.com:cryo28/sidekiq_status.git'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+### Basic
+
+Create a status-friendly worker by include SidekiqStatus::Worker module having #perform method with Sidekiq worker-compatible signature:
+
+```ruby
+class MyWorker
+   include SidekiqStatus::Worker
+
+   def perform(arg1, arg2)
+      # do something
+   end
+end
+```
+
+Now you can enqueue some jobs for this worker
+
+```ruby
+jid = MyWorker.perform_async('val_for_arg1', 'val_for_arg2')
+```
+
+If a job is rejected by some Client middleware, #perform_async returns false (as it doesn with ordinary Sidekiq worker).
+
+Now, you can easily track the status of the job execution:
+
+```ruby
+status_container = SidekiqStatus::Container.load(jid)
+status_container.status # => 'waiting'
+```
+
+When a jobs is scheduled its status is *waiting*. As soon sidekiq worker begins job execution its status is changed to *working*.
+If the job successfully finishes (i.e. doesn't raise an unhandled exception) its status is *complete*. Otherwise its status is *failed*.
+
+### Communication from Worker to Client
+
+*SidekiqStatus::Container* has some attributes and *SidekiqStatus::Worker* module extends your Worker class with a few methods which allow Worker to leave
+some info for the subsequent fetch by a Client. For example you can notify client of the worker progress via *at* and *total=* methods
+
+```ruby
+class MyWorker
+   include SidekiqStatus::Worker
+
+   def perform(arg1, arg2)
+      objects = Array.new(200) { 'some_object_to_process' }
+      self.total= objects.count
+      objects.each_with_index do |object, index|
+        at(index, "Processing object #{index}")
+        object.process!
+      end
+   end
+end
+```
+
+Lets presume a client refreshes container at the middle of job execution (when it's processing the object number 50):
+
+```ruby
+container = SidekiqStatus::Container.load(jid) # or container.reload
+
+container.status       # => 'working'
+container.at           # => 50
+container.total        # => 200
+container.pct_complete # => 25
+container.message      # => 'Processing object #{50}'
+```
+
+Also, a job can leave for the client any custom payload. The only requirement is json-serializeability
+
+```ruby
+class MyWorker
+   include SidekiqStatus::Worker
+
+   def perform(arg1, arg2)
+      objects = Array.new(5) { |i| i }
+      self.total= objects.count
+      result = objects.inject([]) do |accum, object|
+        accum << "result #{object}"
+        accum
+      end
+
+      self.payload= result
+   end
+end
+```
+
+
+Then a client can fetch the result payload
+
+```ruby
+container = SidekiqStatus::Container.load(jid)
+container.status  # => 'complete'
+container.payload # => ["result 0", "result 1", "result 2", "result 3", "result 4"]
+```
+
+SidekiqStatus stores all container attributes in a separate redis key until it's explicitly deleted via container.delete method
+or until redis key expires (see SidekiqStatus::Container.ttl class_attribute).
+
+### Job kill
+
+Any job which is waiting or working can be killed. A working job is killed at the moment of container access.
+
+```ruby
+container = SidekiqStatus::Container.load(jid)
+container.status # => 'working'
+container.killable? # => true
+container.should_kill # => false
+
+container.request_kill
+
+container.status # => 'working'
+container.killable? # => false
+container.should_kill # => true
+
+sleep(1)
+
+container.reload
+container.status # => 'killed'
+```
+
+### Sidekiq web integration
+
+SidekiqStatus also provides an extension to Sidekiq web interface with /statuses page where you can track and kill jobs
+and clean status containers.
+
+   1. Setup Sidekiq web interface according to Sidekiq documentation
+   2. Add "require 'sidekiq_status/web'" beneath "require 'sidekiq/web'"
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+## Copyright
+
+SidekiqStatus © 2012 by Artem Ignatyev. SidekiqStatus is licensed under the MIT license
+

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+# -*- encoding : utf-8 -*-
+
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+spec = RSpec::Core::RakeTask.new
+
+task :default => :spec

data/lib/sidekiq_status/client_middleware.rb

@@ -0,0 +1,25 @@
+# -*- encoding : utf-8 -*-
+
+module SidekiqStatus
+  class ClientMiddleware
+    def call(worker, item, queue)
+      return yield unless worker < SidekiqStatus::Worker
+
+      jid  = item['jid']
+      args = item['args']
+      item['args'] = [jid]
+
+      SidekiqStatus::Container.create(
+          'jid'    => jid,
+          'worker' => worker.name,
+          'queue'  => queue,
+          'args'   => args
+      )
+
+      yield
+    rescue Exception => exc
+      SidekiqStatus::Container.load(jid).delete rescue nil
+      raise exc
+    end
+  end
+end

data/lib/sidekiq_status/container.rb

@@ -0,0 +1,354 @@
+# -*- encoding : utf-8 -*-
+
+# Sidekiq extension to track job execution statuses and returning job results back to the client in a convenient manner
+module SidekiqStatus
+  # SidekiqStatus job container. Contains all job attributes, redis storage/retrieval logic,
+  # some syntactical sugar, such as status predicates and some attribute writers
+  # Doesn't hook into Sidekiq worker
+  class Container
+    # Exception raised if SidekiqStatus job being loaded is not found in Redis
+    class StatusNotFound < RuntimeError;
+    end
+
+    # Possible SidekiqStatus job statuses
+    STATUS_NAMES          = %w(waiting working complete failed killed).freeze
+
+    # A list of statuses jobs in which are not considered pending
+    FINISHED_STATUS_NAMES = %w(complete failed killed).freeze
+
+    # Redis SortedSet key containing requests to kill {SidekiqStatus} jobs
+    KILL_KEY              = 'sidekiq_status_kill'.freeze
+
+                           # Redis SortedSet key to track existing {SidekiqStatus} jobs
+    STATUSES_KEY          = 'sidekiq_statuses'.freeze
+
+    class_attribute :ttl
+    self.ttl = 60*60*24*30 # 30 days
+
+    # Default attribute values (assigned to a newly created container if not explicitly defined)
+    DEFAULTS = {
+        'args'    => [],
+        'worker'  => 'SidekiqStatus::Worker',
+        'queue'   => '',
+        'status'  => 'waiting',
+        'at'      => 0,
+        'total'   => 100,
+        'message' => nil,
+        'payload' => {}
+    }.freeze
+
+    attr_reader :jid, :args, :worker, :queue
+    attr_reader :status, :at, :total, :message, :last_updated_at
+    attr_accessor :payload
+
+    # @param [#to_s] jid SidekiqStatus job id
+    # @return [String] redis key to store/fetch {SidekiqStatus::Container} for the given job
+    def self.status_key(jid)
+      "sidekiq_status:#{jid}"
+    end
+
+    # @return [String] Redis SortedSet key to track existing {SidekiqStatus} jobs
+    def self.statuses_key
+      STATUSES_KEY
+    end
+
+    # @return [String] Redis SortedSet key containing requests to kill {SidekiqStatus} jobs
+    def self.kill_key
+      KILL_KEY
+    end
+
+    # Delete all {SidekiqStatus} jobs which are in given status
+    #
+    # @param [String,Array<String>,nil] status_names List of status names. If nil - delete jobs in any status
+    def self.delete(status_names = nil)
+      status_names ||= STATUS_NAMES
+      status_names = [status_names] unless status_names.is_a?(Array)
+
+      self.statuses.select { |container| status_names.include?(container.status) }.map(&:delete)
+    end
+
+
+    # Retrieve {SidekiqStatus} job identifiers
+    # It's possible to perform some pagination by specifying range boundaries
+    #
+    # @param [Integer] start
+    # @param [Integer] stop
+    # @return [Array<[String,jid]>] Array of hash-like arrays of job id => last_updated_at (unixtime) pairs
+    # @see *Redis#zrange* for details on return values format
+    def self.status_jids(start = 0, stop = -1)
+      Sidekiq.redis do |conn|
+        conn.zrange(self.statuses_key, start, stop, :with_scores => true)
+      end
+    end
+
+    # Retrieve {SidekiqStatus} jobs
+    # It's possible to perform some pagination by specifying range boundaries
+    #
+    # @param [Integer] start
+    # @param [Integer] stop
+    # @return [Array<SidekiqStatus::Container>]
+    def self.statuses(start = 0, stop = -1)
+      jids = status_jids(start, stop)
+      jids = Hash[jids].keys
+      load_multi(jids)
+    end
+
+    # @return [Integer] Known {SidekiqStatus} jobs amount
+    def self.size
+      Sidekiq.redis do |conn|
+        conn.zcard(self.statuses_key)
+      end
+    end
+
+    # Create (initialize, generate unique jid and save) a new {SidekiqStatus} job with given arguments.
+    #
+    # @overload
+    #   @param [String] jid job identifier
+    # @overload
+    #   @param [Hash] data
+    #   @option data [String] jid (SecureRandom.hex(12)) optional job id to create status container for
+    #   @option data [Array]  args job arguments
+    #   @option data [String] worker job worker class name
+    #   @option data [String] queue job queue
+    #
+    # @return [SidekiqStatus::Container]
+    def self.create(data = {})
+      jid = data.delete('jid') if data.is_a?(Hash)
+      jid ||= SecureRandom.hex(12)
+
+      new(jid, data).tap(&:save)
+    end
+
+    # Load {SidekiqStatus::Container} by job identifier
+    #
+    # @param [String] jid job identifier
+    # @raise [StatusNotFound] if there's no info about {SidekiqStatus} job with given *jid*
+    # @return [SidekiqStatus::Container]
+    def self.load(jid)
+      data = load_data(jid)
+      new(jid, data)
+    end
+
+    # Load a list of {SidekiqStatus::Container SidekiqStatus jobs} from Redis
+    #
+    # @param [Array<String>] jids A list of job identifiers to load
+    # @return [Array<SidekiqStatus::Container>>]
+    def self.load_multi(jids)
+      data = load_data_multi(jids)
+      data.map do |jid, data|
+        new(jid, data)
+      end
+    end
+
+    # Load {SidekiqStatus::Container SidekiqStatus job} {SidekiqStatus::Container#dump serialized data} from Redis
+    #
+    # @param [String] jid job identifier
+    # @raise [StatusNotFound] if there's no info about {SidekiqStatus} job with given *jid*
+    # @return [Hash] Job container data (as parsed json, but container is not yet initialized)
+    def self.load_data(jid)
+      load_data_multi([jid])[jid] or raise StatusNotFound.new(jid.to_s)
+    end
+
+    # Load multiple {SidekiqStatus::Container SidekiqStatus job} {SidekiqStatus::Container#dump serialized data} from Redis
+    #
+    # As this method is the most frequently used one, it also contains expire job clean up logic
+    #
+    # @param [Array<#to_s>] jids a list of job identifiers to load data for
+    # @return [Hash{String => Hash}] A hash of job-id to deserialized data pairs
+    def self.load_data_multi(jids)
+      keys = jids.map { |jid| status_key(jid) }
+
+      return {} if keys.empty?
+
+      threshold = Time.now - self.ttl
+
+      data = Sidekiq.redis do |conn|
+        conn.multi do
+          conn.mget(*keys)
+
+          conn.zremrangebyscore(kill_key, 0, threshold.to_i) # Clean up expired unprocessed kill requests
+          conn.zremrangebyscore(statuses_key, 0, threshold.to_i) # Clean up expired statuses from statuses sorted set
+        end
+      end
+
+      data = data.first.map do |json|
+        json ? Sidekiq.load_json(json) : nil
+      end
+
+      Hash[jids.zip(data)]
+    end
+
+    # Initialize a new {SidekiqStatus::Container} with given unique job identifier and attribute data
+    #
+    # @param [String] jid
+    # @param [Hash] data
+    def initialize(jid, data = {})
+      @jid = jid
+      load(data)
+    end
+
+    # Reload current container data from JSON (in case they've changed)
+    def reload
+      data = self.class.load_data(jid)
+      load(data)
+      self
+    end
+
+    # @return [String] redis key to store current {SidekiqStatus::Container container}
+    #   {SidekiqStatus::Container#dump data}
+    def status_key
+      self.class.status_key(jid)
+    end
+
+    # Save current container attribute values to redis
+    def save
+      data = dump
+      data = Sidekiq.dump_json(data)
+
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.setex(status_key, self.ttl, data)
+          conn.zadd(self.class.statuses_key, Time.now.to_f.to_s, self.jid)
+        end
+      end
+    end
+
+    # Delete current container data from redis
+    def delete
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.del(status_key)
+
+          conn.zrem(self.class.kill_key, self.jid)
+          conn.zrem(self.class.statuses_key, self.jid)
+        end
+      end
+    end
+
+    # Request kill for the {SidekiqStatus::Worker SidekiqStatus job}
+    # which parameters are tracked by the current {SidekiqStatus::Container}
+    def request_kill
+      Sidekiq.redis do |conn|
+        conn.zadd(self.class.kill_key, Time.now.to_f.to_s, self.jid)
+      end
+    end
+
+    # @return [Boolean] if job kill is requested
+    def kill_requested?
+      Sidekiq.redis do |conn|
+        conn.zrank(self.class.kill_key, self.jid)
+      end
+    end
+
+    # Reflect the fact that a job has been killed in redis
+    def kill
+      self.status = 'killed'
+
+      Sidekiq.redis do |conn|
+        conn.multi do
+          save
+          conn.zrem(self.class.kill_key, self.jid)
+        end
+      end
+    end
+
+    # @return [Boolean] can the current job be killed
+    def killable?
+      !kill_requested? && %w(waiting working).include?(self.status)
+    end
+
+    # @return [Integer] Job progress in percents (reported solely by {SidekiqStatus::Worker job})
+    def pct_complete
+      (at.to_f / total * 100).round
+    end
+
+    # @param [Fixnum] at Report the progress of a job which is tracked by the current {SidekiqStatus::Container}
+    def at=(at)
+      raise ArgumentError, "at=#{at.inspect} is not a scalar number" unless at.is_a?(Numeric)
+      @at = at
+      @total = @at if @total < @at
+    end
+
+    # Report the estimated upper limit of {SidekiqStatus::Container#at= job items}
+    #
+    # @param [Fixnum] total
+    def total=(total)
+      raise ArgumentError, "total=#{total.inspect} is not a scalar number" unless total.is_a?(Numeric)
+      @total = total
+    end
+
+    # Report current job execution status
+    #
+    # @param [String] status Current job {SidekiqStatus::STATUS_NAMES status}
+    def status=(status)
+      raise ArgumentError, "invalid status #{status.inspect}" unless STATUS_NAMES.include?(status)
+      @status = status
+    end
+
+    # Report side message for the client code
+    #
+    # @param [String] message
+    def message=(message)
+      @message = message && message.to_s
+    end
+
+    # Assign multiple values to {SidekiqStatus::Container} attributes at once
+    #
+    # @param [Hash{#to_s => #to_json}] attrs Attribute=>value pairs
+    def attributes=(attrs = {})
+      attrs.each do |attr_name, value|
+        setter = "#{attr_name}="
+        send(setter, value)
+      end
+    end
+
+    # Assign multiple values to {SidekiqStatus::Container} attributes at once and save to redis
+    # @param [Hash{#to_s => #to_json}] attrs Attribute=>value pairs
+    def update_attributes(attrs = {})
+      self.attributes = attrs
+      save
+    end
+
+    STATUS_NAMES.each do |status_name|
+      define_method("#{status_name}?") do
+        status == status_name
+      end
+    end
+
+    protected
+
+    # Merge-in given data to the current container
+    #
+    # @private
+    # @param [Hash] data
+    def load(data)
+      data = DEFAULTS.merge(data)
+
+      @args, @worker, @queue         = data.values_at('args', 'worker', 'queue')
+      @status, @at, @total, @message = data.values_at('status', 'at', 'total', 'message')
+      @payload                       = data['payload']
+      @last_updated_at               = data['last_updated_at'] && Time.at(data['last_updated_at'].to_i)
+    end
+
+    # Dump current container attribute values to json-serializable hash
+    #
+    # @private
+    # @return [Hash] Data for subsequent json-serialization
+    def dump
+      {
+          'args'            => self.args,
+          'worker'          => self.worker,
+          'queue'           => self.queue,
+
+          'status'          => self.status,
+          'at'              => self.at,
+          'total'           => self.total,
+          'message'         => self.message,
+
+          'payload'         => self.payload,
+          'last_updated_at' => Time.now.to_i
+      }
+    end
+  end
+end
+

data/lib/sidekiq_status/version.rb

@@ -0,0 +1,5 @@
+# -*- encoding : utf-8 -*-
+module SidekiqStatus
+  # SidekiqStatus version
+  VERSION = "1.0.0"
+end

data/lib/sidekiq_status/web.rb

@@ -0,0 +1,65 @@
+module SidekiqStatus
+  # Hook into *Sidekiq::Web* Sinatra app which adds a new "/statuses" page
+  module Web
+    # A ruby module hook to treat the included code as if it was defined inside Sidekiq::Web.
+    # Thus it extends Sidekiq::Web Sinatra application
+    #
+    # @param [Sidekiq::Web] target
+    def self.extend_object(target)
+      target.class_eval do
+        # Calls the given block for every possible template file in views,
+        # named name.ext, where ext is registered on engine.
+        def find_template(views, name, engine, &block)
+          super(File.expand_path('../../../web/views', __FILE__), name, engine, &block)
+          super
+        end
+
+        tabs << 'Statuses'
+
+        get '/statuses' do
+          @count = (params[:count] || 25).to_i
+
+          @current_page = (params[:page] || 1).to_i
+          @current_page = 1 unless @current_page > 0
+
+          @total_size = SidekiqStatus::Container.size
+
+          pageidx = @current_page - 1
+          @statuses = SidekiqStatus::Container.statuses(pageidx * @count, (pageidx + 1) * @count)
+
+          slim :statuses
+        end
+
+        get '/statuses/:jid' do
+          @status = SidekiqStatus::Container.load(params[:jid])
+          slim :status
+        end
+
+        get '/statuses/:jid/kill' do
+          SidekiqStatus::Container.load(params[:jid]).request_kill
+          redirect to(:statuses)
+        end
+
+        get '/statuses/delete/all' do
+          SidekiqStatus::Container.delete
+          redirect to(:statuses)
+        end
+
+        get '/statuses/delete/complete' do
+          SidekiqStatus::Container.delete('complete')
+          redirect to(:statuses)
+        end
+
+        get '/statuses/delete/finished' do
+          SidekiqStatus::Container.delete(SidekiqStatus::Container::FINISHED_STATUS_NAMES)
+          redirect to(:statuses)
+        end
+
+
+      end
+    end
+  end
+end
+
+require 'sidekiq/web'
+Sidekiq::Web.extend(SidekiqStatus::Web)