sidekiq-status 0.7.0 → 0.8.0

data/lib/sidekiq-status/server_middleware.rb

@@ -1,6 +1,17 @@
+if Sidekiq.major_version < 5
+  require 'sidekiq/middleware/server/retry_jobs'
+else
+  require 'sidekiq/job_retry'
+end
+
 module Sidekiq::Status
-# Should be in the server middleware chain
+  # Should be in the server middleware chain
   class ServerMiddleware
+
+    DEFAULT_MAX_RETRY_ATTEMPTS = Sidekiq.major_version < 5 ?
+      Sidekiq::Middleware::Server::RetryJobs::DEFAULT_MAX_RETRY_ATTEMPTS :
+      Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS
+
     include Storage
 
     # Parameterized initialization, use it when adding middleware to server chain
@@ -22,28 +33,64 @@ module Sidekiq::Status
     # @param [Array] msg job args, should have jid format
     # @param [String] queue queue name
     def call(worker, msg, queue)
-      # a way of overriding default expiration time,
-      # so worker wouldn't lose its data
-      # and it allows also to overwrite global expiration time on worker basis
-      if worker.respond_to? :expiration
-        if !worker.expiration && worker.respond_to?(:expiration=)
-          worker.expiration = @expiration
-        else
-          @expiration = worker.expiration
-        end
+
+      # Initial assignment to prevent SystemExit & co. from excepting
+      expiry = @expiration
+
+      # Determine the actual job class
+      klass = msg["args"][0]["job_class"] || msg["class"] rescue msg["class"]
+      job_class = klass.is_a?(Class) ? klass : Module.const_get(klass)
+
+      # Bypass unless this is a Sidekiq::Status::Worker job
+      unless job_class.ancestors.include?(Sidekiq::Status::Worker)
+        yield
+        return
       end
 
-      store_status worker.jid, :working,  @expiration
+      # Determine job expiration
+      expiry = job_class.new.expiration || @expiration rescue @expiration
+
+      store_status worker.jid, :working,  expiry
       yield
-      store_status worker.jid, :complete, @expiration
+      store_status worker.jid, :complete, expiry
     rescue Worker::Stopped
-      store_status worker.jid, :stopped, @expiration
+      store_status worker.jid, :stopped, expiry
     rescue SystemExit, Interrupt
-      store_status worker.jid, :interrupted, @expiration
+      store_status worker.jid, :interrupted, expiry
       raise
-    rescue
-      store_status worker.jid, :failed,  @expiration
+    rescue Exception
+      status = :failed
+      if msg['retry']
+        retry_count = msg['retry_count'] || 0
+        if retry_count < retry_attempts_from(msg['retry'], DEFAULT_MAX_RETRY_ATTEMPTS)
+          status = :retrying
+        end
+      end
+      store_status worker.jid, status, expiry
       raise
     end
+
+    private
+
+    def retry_attempts_from(msg_retry, default)
+      msg_retry.is_a?(Integer) ? msg_retry : default
+    end
+  end
+
+  # Helper method to easily configure sidekiq-status server middleware
+  # whatever the Sidekiq version is.
+  # @param [Sidekiq] sidekiq_config the Sidekiq config
+  # @param [Hash] server_middleware_options server middleware initialization options
+  # @option server_middleware_options [Fixnum] :expiration ttl for complete jobs
+  def self.configure_server_middleware(sidekiq_config, server_middleware_options = {})
+    sidekiq_config.server_middleware do |chain|
+      if Sidekiq.major_version < 5
+        chain.insert_after Sidekiq::Middleware::Server::Logging,
+          Sidekiq::Status::ServerMiddleware, server_middleware_options
+      else
+        chain.add Sidekiq::Status::ServerMiddleware, server_middleware_options
+      end
+    end
+
   end
 end

data/lib/sidekiq-status/sidekiq_extensions.rb

@@ -0,0 +1,7 @@
+require 'sidekiq/version'
+
+module Sidekiq
+  def self.major_version
+    VERSION.split('.').first.to_i
+  end
+end

data/lib/sidekiq-status/testing/inline.rb

@@ -11,6 +11,10 @@ module Sidekiq
     def store_status(id, status, expiration = nil, redis_pool=nil)
       'ok'
     end
+    
+    def store_for_id(id, status_updates, expiration = nil, redis_pool=nil)
+      'ok'
+    end
   end
 end
 

data/lib/sidekiq-status/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Status
-    VERSION = '0.7.0'
+    VERSION = '0.8.0'
   end
 end

data/lib/sidekiq-status/web.rb

@@ -6,9 +6,39 @@ module Sidekiq::Status
     # Location of Sidekiq::Status::Web view templates
     VIEW_PATH = File.expand_path('../../../web/views', __FILE__)
 
+    DEFAULT_PER_PAGE_OPTS = [25, 50, 100].freeze
+    DEFAULT_PER_PAGE = 25
+
+    class << self
+      def per_page_opts= arr
+        @per_page_opts = arr
+      end
+      def per_page_opts
+        @per_page_opts || DEFAULT_PER_PAGE_OPTS
+      end
+      def default_per_page= val
+        @default_per_page = val
+      end
+      def default_per_page
+        @default_per_page || DEFAULT_PER_PAGE
+      end
+    end
+
     # @param [Sidekiq::Web] app
     def self.registered(app)
+
+      # Allow method overrides to support RESTful deletes
+      app.set :method_override, true
+
       app.helpers do
+        def csrf_tag
+          "<input type='hidden' name='authenticity_token' value='#{session[:csrf]}'/>"
+        end
+
+        def poll_path
+          "?#{request.query_string}" if params[:poll]
+        end
+
         def sidekiq_status_template(name)
           path = File.join(VIEW_PATH, name.to_s) + ".erb"
           File.open(path).read
@@ -16,7 +46,7 @@ module Sidekiq::Status
 
         def add_details_to_status(status)
           status['label'] = status_label(status['status'])
-          status["pct_complete"] = pct_complete(status)
+          status["pct_complete"] ||= pct_complete(status)
           return status
         end
 
@@ -29,7 +59,7 @@ module Sidekiq::Status
           case status
           when 'complete'
             'success'
-          when 'working'
+          when 'working', 'retrying'
             'warning'
           when 'queued'
             'primary'
@@ -39,53 +69,50 @@ module Sidekiq::Status
         end
 
         def has_sort_by?(value)
-          ["worker", "status", "update_time", "pct_complete", "message"].include?(value)
+          ["worker", "status", "update_time", "pct_complete", "message", "args"].include?(value)
         end
       end
 
       app.get '/statuses' do
+
         namespace_jids = Sidekiq.redis{ |conn| conn.keys('sidekiq:status:*') }
-        jids = namespace_jids.map{|id_namespace| id_namespace.split(':').last }
+        jids = namespace_jids.map{ |id_namespace| id_namespace.split(':').last }
         @statuses = []
 
         jids.each do |jid|
           status = Sidekiq::Status::get_all jid
           next if !status || status.count < 2
           status = add_details_to_status(status)
-          @statuses << OpenStruct.new(status)
+          @statuses << status
         end
 
         sort_by = has_sort_by?(params[:sort_by]) ? params[:sort_by] : "update_time"
         sort_dir = "asc"
 
         if params[:sort_dir] == "asc"
-          @statuses = @statuses.sort { |x,y| x.send(sort_by) <=> y.send(sort_by) }
+          @statuses = @statuses.sort { |x,y| (x[sort_by] <=> y[sort_by]) || -1 }
         else
           sort_dir = "desc"
-          @statuses = @statuses.sort { |y,x| x.send(sort_by) <=> y.send(sort_by) }
-        end
-
-        working_jobs = @statuses.select{|job| job.status == "working"}
-        size = params[:size] ? params[:size].to_i : 25
-        if working_jobs.size >= size
-         @statuses = working_jobs
-        else
-         @statuses = (@statuses.size >= size) ? @statuses.take(size) : @statuses
+          @statuses = @statuses.sort { |y,x| (x[sort_by] <=> y[sort_by]) || 1 }
         end
 
+        # Sidekiq pagination
+        @total_size = @statuses.count
+        @count = params[:per_page] ? params[:per_page].to_i : Sidekiq::Status::Web.default_per_page
+        @count = @total_size if params[:per_page] == 'all'
+        @current_page = params[:page].to_i < 1 ? 1 : params[:page].to_i
+        @statuses = @statuses.slice((@current_page - 1) * @count, @count)
 
         @headers = [
-          { id: "worker", name: "Worker / JID", class: nil, url: nil},
-          { id: "args", name: "Arguments", class: nil, url: nil},
-          { id: "status", name: "Status", class: nil, url: nil},
-          { id: "update_time", name: "Last Updated", class: nil, url: nil},
-          { id: "pct_complete", name: "Progress", class: nil, url: nil},
+          {id: "worker", name: "Worker / JID", class: nil, url: nil},
+          {id: "args", name: "Arguments", class: nil, url: nil},
+          {id: "status", name: "Status", class: nil, url: nil},
+          {id: "update_time", name: "Last Updated", class: nil, url: nil},
+          {id: "pct_complete", name: "Progress", class: nil, url: nil},
         ]
 
         @headers.each do |h|
-          params["sort_by"] = h[:id]
-          params["sort_dir"] = (sort_by == h[:id] && sort_dir == "asc") ? "desc" : "asc"
-          h[:url] = "statuses?" + params.map {|k,v| "#{k}=#{v}" }.join("&")
+          h[:url] = "statuses?" + params.merge("sort_by" => h[:id], "sort_dir" => (sort_by == h[:id] && sort_dir == "asc") ? "desc" : "asc").map{|k, v| "#{k}=#{CGI.escape v.to_s}"}.join("&")
           h[:class] = "sorted_#{sort_dir}" if sort_by == h[:id]
         end
 
@@ -98,16 +125,33 @@ module Sidekiq::Status
         if job.empty?
           halt [404, {"Content-Type" => "text/html"}, [erb(sidekiq_status_template(:status_not_found))]]
         else
-          @status = OpenStruct.new(add_details_to_status(job))
+          @status = add_details_to_status(job)
           erb(sidekiq_status_template(:status))
         end
       end
+
+      # Retries a failed job from the status list
+      app.put '/statuses' do
+        job = Sidekiq::RetrySet.new.find_job(params[:jid])
+        job ||= Sidekiq::DeadSet.new.find_job(params[:jid])
+        job.retry if job
+        halt [302, { "Location" => request.referer }, []]
+      end
+
+      # Removes a completed job from the status list
+      app.delete '/statuses' do
+        Sidekiq::Status.delete(params[:jid])
+        halt [302, { "Location" => request.referer }, []]
+      end
     end
   end
 end
 
 require 'sidekiq/web' unless defined?(Sidekiq::Web)
 Sidekiq::Web.register(Sidekiq::Status::Web)
+["per_page", "sort_by", "sort_dir"].each do |key|
+  Sidekiq::WebHelpers::SAFE_QPARAMS.push(key)
+end
 if Sidekiq::Web.tabs.is_a?(Array)
   # For sidekiq < 2.5
   Sidekiq::Web.tabs << "statuses"

data/lib/sidekiq-status/worker.rb

@@ -27,14 +27,16 @@ module Sidekiq::Status::Worker
   # @param String optional message
   # @return [String]
   def at(num, message = nil)
-    total(100) if retrieve(:total).nil?
-    store(at: num, message: message)
+    @_status_total = 100 if @_status_total.nil?
+    pct_complete = ((num / @_status_total.to_f) * 100).to_i rescue 0
+    store(at: num, total: @_status_total, pct_complete: pct_complete, message: message)
   end
 
   # Sets total number of tasks
   # @param Fixnum total number of tasks
   # @return [String]
   def total(num)
+    @_status_total = num
     store(total: num)
   end
 

data/sidekiq-status.gemspec

@@ -2,8 +2,8 @@
 require File.expand_path('../lib/sidekiq-status/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ['Evgeniy Tsvigun']
-  gem.email         = ['utgarda@gmail.com']
+  gem.authors       = ['Evgeniy Tsvigun', 'Kenaniah Cerny']
+  gem.email         = ['utgarda@gmail.com', 'kenaniah@gmail.com']
   gem.summary       = 'An extension to the sidekiq message processing to track your jobs'
   gem.homepage      = 'http://github.com/utgarda/sidekiq-status'
   gem.license       = 'MIT'
@@ -14,8 +14,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
   gem.version       = Sidekiq::Status::VERSION
 
-  gem.add_dependency                  'sidekiq', '>= 2.7'
+  gem.add_dependency                  'sidekiq', '>= 3.0'
   gem.add_dependency                  'chronic_duration'
+  gem.add_development_dependency      'appraisal'
+  gem.add_development_dependency      'colorize'
   gem.add_development_dependency      'rack-test'
   gem.add_development_dependency      'rake'
   gem.add_development_dependency      'rspec'

data/spec/lib/sidekiq-status/client_middleware_spec.rb

@@ -5,18 +5,20 @@ describe Sidekiq::Status::ClientMiddleware do
   let!(:redis) { Sidekiq.redis { |conn| conn } }
   let!(:job_id) { SecureRandom.hex(12) }
 
-  describe "#call" do
-    before { client_middleware }
+  before do
+    allow(SecureRandom).to receive(:hex).once.and_return(job_id)
+  end
+
+  describe "without :expiration parameter" do
+
     it "sets queued status" do
-      allow(SecureRandom).to receive(:hex).once.and_return(job_id)
-      expect(StubJob.perform_async(:arg1 => 'val1')).to eq(job_id)
+      expect(StubJob.perform_async arg1: 'val1').to eq(job_id)
       expect(redis.hget("sidekiq:status:#{job_id}", :status)).to eq('queued')
       expect(Sidekiq::Status::queued?(job_id)).to be_truthy
     end
 
     it "sets status hash ttl" do
-      allow(SecureRandom).to receive(:hex).once.and_return(job_id)
-      expect(StubJob.perform_async(:arg1 => 'val1')).to eq(job_id)
+      expect(StubJob.perform_async arg1: 'val1').to eq(job_id)
       expect(1..Sidekiq::Status::DEFAULT_EXPIRY).to cover redis.ttl("sidekiq:status:#{job_id}")
     end
 
@@ -35,18 +37,22 @@ describe Sidekiq::Status::ClientMiddleware do
         Sidekiq::Status::ClientMiddleware.new.call(StubJob, {'jid' => SecureRandom.hex}, :queued) do end
       end
     end
+
   end
 
-  describe ":expiration parameter" do
+  describe "with :expiration parameter" do
+
     let(:huge_expiration) { Sidekiq::Status::DEFAULT_EXPIRY * 100 }
+
+    # Ensure client middleware is loaded with an expiration parameter set
     before do
-      allow(SecureRandom).to receive(:hex).once.and_return(job_id)
+      client_middleware expiration: huge_expiration
     end
 
     it "overwrites default expiry value" do
-      client_middleware(expiration: huge_expiration)
-      StubJob.perform_async(:arg1 => 'val1')
+      StubJob.perform_async arg1: 'val1'
       expect((Sidekiq::Status::DEFAULT_EXPIRY+1)..huge_expiration).to cover redis.ttl("sidekiq:status:#{job_id}")
     end
+
   end
 end

data/spec/lib/sidekiq-status/server_middleware_spec.rb

@@ -5,15 +5,18 @@ describe Sidekiq::Status::ServerMiddleware do
   let!(:redis) { Sidekiq.redis { |conn| conn } }
   let!(:job_id) { SecureRandom.hex(12) }
 
-  describe "#call" do
+  describe "without :expiration parameter" do
     it "sets working/complete status" do
-      thread = confirmations_thread 4, "status_updates", "job_messages_#{job_id}"
       allow(SecureRandom).to receive(:hex).once.and_return(job_id)
       start_server do
-        expect(ConfirmationJob.perform_async(:arg1 => 'val1')).to eq(job_id)
-        expect(thread.value).to eq([job_id, job_id,
-                                "while in #perform, status = working",
-                                job_id])
+        thread = redis_thread 4, "status_updates", "job_messages_#{job_id}"
+        expect(ConfirmationJob.perform_async arg1: 'val1').to eq(job_id)
+        expect(thread.value).to eq([
+          job_id,
+          job_id,
+          "while in #perform, status = working",
+          job_id
+        ])
       end
       expect(redis.hget("sidekiq:status:#{job_id}", :status)).to eq('complete')
       expect(Sidekiq::Status::complete?(job_id)).to be_truthy
@@ -30,6 +33,17 @@ describe Sidekiq::Status::ServerMiddleware do
       expect(Sidekiq::Status::failed?(job_id)).to be_truthy
     end
 
+    it "sets failed status when Exception raised" do
+      allow(SecureRandom).to receive(:hex).once.and_return(job_id)
+      start_server do
+        expect(capture_status_updates(3) {
+          expect(FailingHardJob.perform_async).to eq(job_id)
+        }).to eq([job_id]*3)
+      end
+      expect(redis.hget("sidekiq:status:#{job_id}", :status)).to eq('failed')
+      expect(Sidekiq::Status::failed?(job_id)).to be_truthy
+    end
+
     context "sets interrupted status" do
       it "on system exit signal" do
         allow(SecureRandom).to receive(:hex).once.and_return(job_id)
@@ -58,13 +72,13 @@ describe Sidekiq::Status::ServerMiddleware do
     it "sets status hash ttl" do
       allow(SecureRandom).to receive(:hex).once.and_return(job_id)
       start_server do
-        expect(StubJob.perform_async(:arg1 => 'val1')).to eq(job_id)
+        expect(StubJob.perform_async arg1: 'val1').to eq(job_id)
       end
       expect(1..Sidekiq::Status::DEFAULT_EXPIRY).to cover redis.ttl("sidekiq:status:#{job_id}")
     end
   end
 
-  describe ":expiration parameter" do
+  describe "with :expiration parameter" do
     let(:huge_expiration) { Sidekiq::Status::DEFAULT_EXPIRY * 100 }
     before do
       allow(SecureRandom).to receive(:hex).once.and_return(job_id)
@@ -72,7 +86,7 @@ describe Sidekiq::Status::ServerMiddleware do
 
     it "overwrites default expiry value" do
       start_server(:expiration => huge_expiration) do
-        StubJob.perform_async(:arg1 => 'val1')
+        StubJob.perform_async arg1: 'val1'
       end
       expect((Sidekiq::Status::DEFAULT_EXPIRY-1)..huge_expiration).to cover redis.ttl("sidekiq:status:#{job_id}")
     end
@@ -81,7 +95,7 @@ describe Sidekiq::Status::ServerMiddleware do
       overwritten_expiration = huge_expiration * 100
       allow_any_instance_of(StubJob).to receive(:expiration).and_return(overwritten_expiration)
       start_server(:expiration => huge_expiration) do
-        StubJob.perform_async(:arg1 => 'val1')
+        StubJob.perform_async arg1: 'val1'
       end
       expect((huge_expiration+1)..overwritten_expiration).to cover redis.ttl("sidekiq:status:#{job_id}")
     end