sidekiq-status 0.2.0 → 0.3.0

data/.travis.yml

@@ -0,0 +1,2 @@
+language: ruby
+services: redis

data/Gemfile

@@ -0,0 +1,4 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -1,5 +1,8 @@
 # Sidekiq::Status
 
+[![Code Climate](https://codeclimate.com/github/utgarda/sidekiq-status.png)](https://codeclimate.com/github/utgarda/sidekiq-status)
+[![Build Status](https://secure.travis-ci.org/utgarda/sidekiq-status.png)](http://travis-ci.org/utgarda/sidekiq-status)
+
 An extension to [Sidekiq](http://github.com/mperham/sidekiq) message processing to track your jobs. Inspired
 by [resque-status](http://github.com/quirkey/resque-status) and mostly copying its features, using Sidekiq's middleware.
 
@@ -9,6 +12,8 @@ gem install sidekiq-status
 
 ## Usage
 
+### Configuration
+
 Configure your middleware chains, lookup [Middleware usage](https://github.com/mperham/sidekiq/wiki/Middleware)
 on Sidekiq wiki for more info.
 
@@ -24,18 +29,16 @@ end
 
 Sidekiq.configure_server do |config|
   config.server_middleware do |chain|
-    chain.add Sidekiq::Status::ServerMiddleware
+    chain.add Sidekiq::Status::ServerMiddleware, expiration: 30.minutes # default
   end
 end
 ```
 
-When defining those jobs you want to track later, include one more module. Jobs defined without Sidekiq::Status::Worker
-will be processed as usual.
+After that you can use your jobs as usual and only include `Sidekiq::Status::Worker` module if you want additional functionality of tracking progress and passing any data from job to client.
 
 ``` ruby
 class MyJob
   include Sidekiq::Worker
-  include Sidekiq::Status::Worker
 
   def perform(*args)
   # your code goes here
@@ -43,19 +46,61 @@ class MyJob
 end
 ```
 
+### Retrieving status
+
 Query for job status any time later:
 
 ``` ruby
 job_id = MyJob.perform_async(*args)
-# "queued", "working", "complete" or "failed" , nil after expiry (30 minutes)
-status = Sidekiq::Status::get(job_id)
+# :queued, :working, :complete or :failed , nil after expiry (30 minutes)
+status = Sidekiq::Status::status(job_id)
+Sidekiq::Status::queued?   job_id
+Sidekiq::Status::working?  job_id
+Sidekiq::Status::complete? job_id
+Sidekiq::Status::failed?   job_id
+```
+
+### Tracking progress, saving and retrieveing data associated with job
+
+``` ruby
+class MyJob
+  include Sidekiq::Worker
+  include Sidekiq::Status::Worker # Important!
+
+  def perform(*args)
+    # your code goes here
+
+    # the common idiom to track progress of your task
+    at 5, 100, "Almost done"
+
+    # a way to associate data with your job
+    store vino: 'veritas'
+
+    # a way of retrieving said data
+    # remember that retrieved data is always is String|nil
+    vino = retrieve :vino
+  end
+end
+
+job_id = MyJob.perform_async(*args)
+data = Sidekiq::Status::get_all job_id
+data # => {status: 'complete', update_time: 1360006573, vino: 'veritas'}
+Sidekiq::Status::get     job_id, :vino #=> 'veritas'
+Sidekiq::Status::num     job_id #=> 5
+Sidekiq::Status::total   job_id #=> 100
+Sidekiq::Status::message job_id #=> "Almost done"
+Sidekiq::Status::pct_complete job_id #=> 5
 ```
 
 ### Features coming
-* Progress tracking, messages from running jobs
 * Stopping jobs by id
 * Minimal web UI
 
+## Thanks
+Andrew Korzhuev  
+Jon Moses  
+Wayne Hoover
+
 ## License
 MIT License , see LICENSE for more details.
-© 2012 Evgeniy Tsvigun
+© 2012 - 2013 Evgeniy Tsvigun

data/lib/sidekiq-status.rb

@@ -4,24 +4,54 @@ require 'sidekiq-status/worker'
 require 'sidekiq-status/client_middleware'
 require 'sidekiq-status/server_middleware'
 
-module Sidekiq
-  module Status
-    extend Storage
-    DEFAULT_EXPIRY = 60 * 30
-    UUID_REGEXP = /[0-9A-F]{24}/i #RegEx for SecureRandom.hex(12) which is the format Sidekiq uses for its jid
+module Sidekiq::Status
+  extend Storage
+  DEFAULT_EXPIRY = 60 * 30
+  STATUS = %w(queued working complete stopped failed).map(&:to_sym).freeze
 
+  class << self
     # Job status by id
     # @param [String] id job id returned by async_perform
-    # @return [String] job status, possible values: "queued" , "working" , "complete"
-    def self.get(id)
-      read_field_for_id(id, :status)
+    # @return [String] job status, possible values are in STATUS
+    def get(job_id, field)
+      read_field_for_id(job_id, field)
     end
 
     # Get all status fields for a job
     # @params [String] id job id returned by async_perform
     # @return [Hash] hash of all fields stored for the job
-    def self.get_all(id)
+    def get_all(id)
       read_hash_for_id(id)
     end
+
+    def status(job_id)
+      status = get(job_id, :status)
+      status.to_sym  unless status.nil?
+    end
+
+    STATUS.each do |name|
+      class_eval(<<-END, __FILE__, __LINE__)
+        def #{name}?(job_id)
+          status(job_id) == :#{name}
+        end
+      END
+    end
+
+    # Methods for retrieving job completion
+    def num(job_id)
+      get(job_id, :num).to_i
+    end
+
+    def total(job_id)
+      get(job_id, :total).to_i
+    end
+
+    def pct_complete(job_id)
+      (num(job_id).to_f / total(job_id)) * 100
+    end
+
+    def message(job_id)
+      get(job_id, :message)
+    end
   end
 end

data/lib/sidekiq-status/client_middleware.rb

@@ -7,9 +7,7 @@ module Sidekiq::Status
     # @param [Array] msg job arguments
     # @param [String] queue the queue's name
     def call(worker_class, msg, queue)
-      if worker_class.include? Worker
-        store_for_id(msg['jid'], :status => :queued)
-      end
+      store_status msg['jid'], :queued
       yield
     end
   end

data/lib/sidekiq-status/server_middleware.rb

@@ -1,11 +1,13 @@
 module Sidekiq::Status
 # Should be in the server middleware chain
   class ServerMiddleware
+    include Storage
+
     # Parameterized initialization, use it when adding middleware to server chain
     # chain.add Sidekiq::Status::ServerMiddleware, :expiration => 60 * 5
     # @param [Hash] opts middleware initialization options
     # @option opts [Fixnum] :expiration ttl for complete jobs
-    def initialize(opts = {:expiration => 30 * 60})
+    def initialize(opts = {})
       @expiration = opts[:expiration]
     end
 
@@ -20,26 +22,18 @@ module Sidekiq::Status
     # @param [Array] msg job args, should have jid format
     # @param [String] queue queue name
     def call(worker, msg, queue)
-      if worker.is_a? Worker
-        worker.id = msg['jid']
-        unless worker.id.is_a?(String) && UUID_REGEXP.match(worker.id)
-          raise ArgumentError, "First job argument for a #{worker.class.name} should have jid format"
-        end
-        worker.store 'status' => 'working'
-        yield
-        worker.store 'status' => 'complete'
-      else
-        yield
-      end
+      # a way of overriding default expiration time,
+      # so worker wouldn't lose its data
+      worker.expiration = @expiration  if worker.respond_to? :expiration
+
+      store_status worker.jid, :working,  @expiration
+      yield
+      store_status worker.jid, :complete, @expiration
     rescue Worker::Stopped
-      worker.store 'status' => 'stopped'
+      store_status worker.jid, :stopped, @expiration
     rescue
-      if worker.is_a? Worker
-        worker.store 'status' => 'failed'
-      end
+      store_status worker.jid, :failed,  @expiration
       raise
-    ensure
-      Sidekiq.redis { |conn| conn.expire worker.id, @expiration } if worker.is_a? Worker
     end
   end
 end

data/lib/sidekiq-status/storage.rb

@@ -8,24 +8,32 @@ module Sidekiq::Status::Storage
   # @param [String] id job id
   # @param [Hash] status_updates updated values
   # @return [String] Redis operation status code
-  def store_for_id(id, status_updates)
+  def store_for_id(id, status_updates, expiration = nil)
     Sidekiq.redis do |conn|
-      answers = conn.multi do
-        conn.hmset id, 'update_time', Time.now.to_i, *(status_updates.to_a.flatten)
-        conn.expire id, Sidekiq::Status::DEFAULT_EXPIRY
+      conn.multi do
+        conn.hmset  id, 'update_time', Time.now.to_i, *(status_updates.to_a.flatten(1))
+        conn.expire id, (expiration || Sidekiq::Status::DEFAULT_EXPIRY)
         conn.publish "status_updates", id
-      end
-      answers[0]
+      end[0]
     end
   end
 
+  # Stores job status and sets expiration time to it
+  # only in case of :failed or :stopped job
+  # @param [String] id job id
+  # @param [Symbol] job status
+  # @return [String] Redis operation status code
+  def store_status(id, status, expiration = nil)
+    store_for_id id, {status: status}, expiration
+  end
+
   # Gets a single valued from job status hash
   # @param [String] id job id
   # @param [String] Symbol field fetched field name
   # @return [String] Redis operation status code
-  def read_field_for_id(uuid, field)
+  def read_field_for_id(id, field)
     Sidekiq.redis do |conn|
-      conn.hmget(uuid, field)[0]
+      conn.hmget(id, field)[0]
     end
   end
 

data/lib/sidekiq-status/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Status
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

data/lib/sidekiq-status/worker.rb

@@ -4,23 +4,31 @@ module Sidekiq::Status::Worker
   class Stopped < StandardError
   end
 
-  attr_reader :id
-
-  # Worker id initialization
-  # @param [String] id id generated on client-side
-  # @raise [RuntimeError] raised in case of second id initialization attempt
-  # @return [String] id
-  def id=(id)
-    raise RuntimeError("Worker ID is already set : #{@id}") if @id
-    @id=id
-  end
+  attr_accessor :expiration
 
   # Stores multiple values into a job's status hash,
   # sets last update time
   # @param [Hash] status_updates updated values
   # @return [String] Redis operation status code
   def store(hash)
-    store_for_id(@id, hash)
+    store_for_id @jid, hash, @expiration
+  end
+
+  # Read value from job status hash
+  # @param String|Symbol hask key
+  # @return [String]
+  def retrieve(name)
+    read_field_for_id @jid, name
+  end
+
+  # Sets current task progress
+  # (inspired by resque-status)
+  # @param Fixnum number of tasks done
+  # @param Fixnum total number of tasks
+  # @param String optional message
+  # @return [String]
+  def at(num, total, message=nil)
+    store({num: num, total: total, message: message})
   end
 
-end
+end

data/sidekiq-status.gemspec

@@ -15,5 +15,6 @@ Gem::Specification.new do |gem|
   gem.version       = Sidekiq::Status::VERSION
 
   gem.add_dependency                  'sidekiq', '~> 2.7'
+  gem.add_development_dependency      'rake'
   gem.add_development_dependency      'rspec'
 end

data/spec/lib/sidekiq-status/client_middleware_spec.rb

@@ -13,6 +13,7 @@ describe Sidekiq::Status::ClientMiddleware do
       SecureRandom.should_receive(:hex).once.and_return(job_id)
       StubJob.perform_async(:arg1 => 'val1').should == job_id
       redis.hget(job_id, :status).should == 'queued'
+      Sidekiq::Status::queued?(job_id).should be_true
     end
 
     it "sets status hash ttl" do

data/spec/lib/sidekiq-status/server_middleware_spec.rb

@@ -21,6 +21,7 @@ describe Sidekiq::Status::ServerMiddleware do
                                 job_id]
       end
       redis.hget(job_id, :status).should == 'complete'
+      Sidekiq::Status::complete?(job_id).should be_true
     end
 
     it "sets failed status" do
@@ -31,6 +32,7 @@ describe Sidekiq::Status::ServerMiddleware do
         }.should == [job_id]*3
       end
       redis.hget(job_id, :status).should == 'failed'
+      Sidekiq::Status::failed?(job_id).should be_true
     end
 
     it "sets status hash ttl" do

data/spec/lib/sidekiq-status_spec.rb

@@ -11,17 +11,53 @@ describe Sidekiq::Status do
   # so we should wait till they expire
   before { redis.flushall; sleep 0.1 }
 
-  describe ".get" do
-    it "gets job status by id" do
+  describe ".status, .working?, .complete?" do
+    it "gets job status by id as symbol" do
       SecureRandom.should_receive(:hex).once.and_return(job_id)
 
       start_server do
         capture_status_updates(2) {
           LongJob.perform_async(1).should == job_id
         }.should == [job_id]*2
-        Sidekiq::Status.get(job_id).should == "working"
+        Sidekiq::Status.status(job_id).should == :working
+        Sidekiq::Status.working?(job_id).should be_true
+        Sidekiq::Status::queued?(job_id).should be_false
+        Sidekiq::Status::failed?(job_id ).should be_false
+        Sidekiq::Status::complete?(job_id).should be_false
+        Sidekiq::Status::stopped?(job_id).should be_false
+      end
+      Sidekiq::Status.status(job_id).should == :complete
+      Sidekiq::Status.complete?(job_id).should be_true
+    end
+  end
+
+  describe ".get" do
+    it "gets a single value from data hash as string" do
+      SecureRandom.should_receive(:hex).once.and_return(job_id)
+
+      start_server do
+        capture_status_updates(3) {
+          DataJob.perform_async.should == job_id
+        }.should == [job_id]*3
+        Sidekiq::Status.get(job_id, :status).should == 'working'
+      end
+      Sidekiq::Status.get(job_id, :data).should == 'meow'
+    end
+  end
+
+  describe ".num, .total, .pct_complete, .message" do
+    it "should return job progress with correct type to it" do
+      SecureRandom.should_receive(:hex).once.and_return(job_id)
+
+      start_server do
+        capture_status_updates(3) {
+          ProgressJob.perform_async.should == job_id
+        }.should == [job_id]*3
       end
-      Sidekiq::Status.get(job_id).should == 'complete'
+      Sidekiq::Status.num(job_id).should == 100
+      Sidekiq::Status.total(job_id).should == 500
+      Sidekiq::Status.pct_complete(job_id).should == 20
+      Sidekiq::Status.message(job_id).should == 'howdy, partner?'
     end
   end
 
@@ -42,19 +78,19 @@ describe Sidekiq::Status do
   end
 
   context "keeps normal Sidekiq functionality" do
-    it "does jobs with and without status processing" do
-      SecureRandom.should_receive(:hex).exactly(4).times.and_return(job_id, job_id_1)
+    it "does jobs with and without included worker module" do
+      SecureRandom.should_receive(:hex).exactly(4).times.and_return(job_id, job_id, job_id_1, job_id_1)
       start_server do
-        capture_status_updates(6) {
+        capture_status_updates(12) {
           StubJob.perform_async.should == job_id
           NoStatusConfirmationJob.perform_async(1)
           StubJob.perform_async.should == job_id_1
           NoStatusConfirmationJob.perform_async(2)
-        }.should =~ [job_id, job_id_1] * 3
+        }.should =~ [job_id, job_id_1] * 6
       end
       redis.mget('NoStatusConfirmationJob_1', 'NoStatusConfirmationJob_2').should == %w(done)*2
-      Sidekiq::Status.get(job_id).should == 'complete'
-      Sidekiq::Status.get(job_id_1).should == 'complete'
+      Sidekiq::Status.status(job_id).should == :complete
+      Sidekiq::Status.status(job_id_1).should == :complete
     end
 
     it "retries failed jobs" do
@@ -64,7 +100,7 @@ describe Sidekiq::Status do
           RetriedJob.perform_async().should == job_id
         }.should == [job_id] * 5
       end
-      Sidekiq::Status.get(job_id).should == 'complete'
+      Sidekiq::Status.status(job_id).should == :complete
     end
   end
 

data/spec/spec_helper.rb

@@ -65,4 +65,4 @@ def start_server()
   Timeout::timeout(10) { Process.wait pid } rescue Timeout::Error
 ensure
   Process.kill 'KILL', pid rescue "OK" # it's OK if the process is gone already
-end
+end

data/spec/support/test_jobs.rb

@@ -14,10 +14,26 @@ class LongJob < StubJob
   end
 end
 
+class DataJob < StubJob
+  def perform
+    sleep 0.1
+    store({data: 'meow'})
+    retrieve(:data).should == 'meow'
+    sleep 0.1
+  end
+end
+
+class ProgressJob < StubJob
+  def perform
+    at 100, 500, 'howdy, partner?'
+    sleep 0.1
+  end
+end
+
 class ConfirmationJob < StubJob
   def perform(*args)
     Sidekiq.redis do |conn|
-      conn.publish "job_messages_#{id}", "while in #perform, status = #{conn.hget id, :status}"
+      conn.publish "job_messages_#{jid}", "while in #perform, status = #{conn.hget jid, :status}"
     end
   end
 end
@@ -28,7 +44,6 @@ class NoStatusConfirmationJob
     Sidekiq.redis do |conn|
       conn.set "NoStatusConfirmationJob_#{id}", "done"
     end
-
   end
 end
 
@@ -42,7 +57,7 @@ class RetriedJob < StubJob
   sidekiq_options 'retry' => 'true'
   def perform()
     Sidekiq.redis do |conn|
-      key = "RetriedJob_#{id}"
+      key = "RetriedJob_#{jid}"
       unless conn.exists key
         conn.set key, 'tried'
         raise StandardError

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-status
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-04 00:00:00.000000000 Z
+date: 2013-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -27,6 +27,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +68,8 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rspec
+- .travis.yml
+- Gemfile
 - LICENSE
 - README.md
 - Rakefile