sidekiq-status 0.6.0 → 3.0.3

data/lib/sidekiq-status/server_middleware.rb

@@ -1,6 +1,13 @@
+if Sidekiq.major_version >= 5
+  require 'sidekiq/job_retry'
+end
+
 module Sidekiq::Status
-# Should be in the server middleware chain
+  # Should be in the server middleware chain
   class ServerMiddleware
+
+    DEFAULT_MAX_RETRY_ATTEMPTS = Sidekiq.major_version >= 5 ? Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS : 25
+
     include Storage
 
     # Parameterized initialization, use it when adding middleware to server chain
@@ -22,28 +29,77 @@ module Sidekiq::Status
     # @param [Array] msg job args, should have jid format
     # @param [String] queue queue name
     def call(worker, msg, queue)
-      # a way of overriding default expiration time,
-      # so worker wouldn't lose its data
-      # and it allows also to overwrite global expiration time on worker basis
-      if worker.respond_to? :expiration
-        if !worker.expiration && worker.respond_to?(:expiration=)
-          worker.expiration = @expiration
-        else
-          @expiration = worker.expiration
+
+      # Initial assignment to prevent SystemExit & co. from excepting
+      expiry = @expiration
+
+      # Determine the actual job class
+      klass = msg["args"][0]["job_class"] || msg["class"] rescue msg["class"]
+      job_class = klass.is_a?(Class) ? klass : Module.const_get(klass)
+
+      # Bypass unless this is a Sidekiq::Status::Worker job
+      unless job_class.ancestors.include?(Sidekiq::Status::Worker)
+        yield
+        return
+      end
+
+      begin
+        # Determine job expiration
+        expiry = job_class.new.expiration || @expiration rescue @expiration
+
+        store_status worker.jid, :working,  expiry
+        yield
+        store_status worker.jid, :complete, expiry
+      rescue Worker::Stopped
+        store_status worker.jid, :stopped, expiry
+      rescue SystemExit, Interrupt
+        store_status worker.jid, :interrupted, expiry
+        raise
+      rescue Exception
+        status = :failed
+        if msg['retry']
+          if retry_attempt_number(msg) < retry_attempts_from(msg['retry'], DEFAULT_MAX_RETRY_ATTEMPTS)
+            status = :retrying
+          end
         end
+        store_status(worker.jid, status, expiry) if job_class && job_class.ancestors.include?(Sidekiq::Status::Worker)
+        raise
+      end
+    end
+
+    private
+
+    def retry_attempt_number(msg)
+      if msg['retry_count']
+        msg['retry_count'] + sidekiq_version_dependent_retry_offset
+      else
+        0
       end
+    end
+
+    def retry_attempts_from(msg_retry, default)
+      msg_retry.is_a?(Integer) ? msg_retry : default
+    end
 
-      store_status worker.jid, :working,  @expiration
-      yield
-      store_status worker.jid, :complete, @expiration
-    rescue Worker::Stopped
-      store_status worker.jid, :stopped, @expiration
-    rescue SystemExit, Interrupt
-      store_status worker.jid, :interrupted, @expiration
-      raise
-    rescue
-      store_status worker.jid, :failed,  @expiration
-      raise
+    def sidekiq_version_dependent_retry_offset
+      Sidekiq.major_version >= 4 ? 1 : 0
     end
   end
+
+  # Helper method to easily configure sidekiq-status server middleware
+  # whatever the Sidekiq version is.
+  # @param [Sidekiq] sidekiq_config the Sidekiq config
+  # @param [Hash] server_middleware_options server middleware initialization options
+  # @option server_middleware_options [Fixnum] :expiration ttl for complete jobs
+  def self.configure_server_middleware(sidekiq_config, server_middleware_options = {})
+    sidekiq_config.server_middleware do |chain|
+      if Sidekiq.major_version < 5
+        chain.insert_after Sidekiq::Middleware::Server::Logging,
+          Sidekiq::Status::ServerMiddleware, server_middleware_options
+      else
+        chain.add Sidekiq::Status::ServerMiddleware, server_middleware_options
+      end
+    end
+
+  end
 end

data/lib/sidekiq-status/sidekiq_extensions.rb

@@ -0,0 +1,7 @@
+require 'sidekiq/version'
+
+module Sidekiq
+  def self.major_version
+    VERSION.split('.').first.to_i
+  end
+end

data/lib/sidekiq-status/storage.rb

@@ -12,11 +12,12 @@ module Sidekiq::Status::Storage
   # @param [ConnectionPool] redis_pool optional redis connection pool
   # @return [String] Redis operation status code
   def store_for_id(id, status_updates, expiration = nil, redis_pool=nil)
+    status_updates.transform_values!(&:to_s)
     redis_connection(redis_pool) do |conn|
-      conn.multi do
-        conn.hmset  key(id), 'update_time', Time.now.to_i, *(status_updates.to_a.flatten(1))
-        conn.expire key(id), (expiration || Sidekiq::Status::DEFAULT_EXPIRY)
-        conn.publish "status_updates", id
+      conn.multi do |pipeline|
+        pipeline.hset  key(id), 'update_time', Time.now.to_i, *(status_updates.to_a.flatten(1))
+        pipeline.expire key(id), (expiration || Sidekiq::Status::DEFAULT_EXPIRY)
+        pipeline.publish "status_updates", id
       end[0]
     end
   end
@@ -36,7 +37,7 @@ module Sidekiq::Status::Storage
   # @param [String] id job id
   # @param [Num] job_unix_time, unix timestamp for the scheduled job
   def delete_and_unschedule(job_id, job_unix_time = nil)
-    Sidekiq.redis do |conn|
+    Sidekiq::Status.redis_adapter do |conn|
       scan_options = {offset: 0, conn: conn, start: (job_unix_time || '-inf'), end: (job_unix_time || '+inf')}
 
       while not (jobs = schedule_batch(scan_options)).empty?
@@ -52,12 +53,21 @@ module Sidekiq::Status::Storage
     false
   end
 
+  # Deletes status hash info for given job id
+  # @param[String] job id
+  # @retrun [Integer] number of keys that were removed
+  def delete_status(id)
+    redis_connection do |conn|
+      conn.del(key(id))
+    end
+  end
+
   # Gets a single valued from job status hash
   # @param [String] id job id
   # @param [String] Symbol field fetched field name
   # @return [String] Redis operation status code
   def read_field_for_id(id, field)
-    Sidekiq.redis do |conn|
+    Sidekiq::Status.redis_adapter do |conn|
       conn.hget(key(id), field)
     end
   end
@@ -66,8 +76,8 @@ module Sidekiq::Status::Storage
   # @param [String] id job id
   # @return [Hash] Hash stored in redis
   def read_hash_for_id(id)
-    Sidekiq.redis do |conn|
-      conn.hgetall key(id)
+    Sidekiq::Status.redis_adapter do |conn|
+      conn.hgetall(key(id))
     end
   end
 
@@ -81,7 +91,7 @@ module Sidekiq::Status::Storage
   #  -  end: end score (i.e. +inf or a unix timestamp)
   #  -  offset: current progress through (all) jobs (e.g.: 100 if you want jobs from 100 to BATCH_LIMIT)
   def schedule_batch(options)
-    options[:conn].zrangebyscore "schedule", options[:start], options[:end], {limit: [options[:offset], BATCH_LIMIT]}
+    Sidekiq::Status.wrap_redis_connection(options[:conn]).schedule_batch("schedule", options.merge(limit: BATCH_LIMIT))
   end
 
   # Searches the jobs Array for the job_id

data/lib/sidekiq-status/testing/inline.rb

@@ -5,6 +5,16 @@ module Sidekiq
         :complete
       end
     end
+
+    module Storage
+      def store_status(id, status, expiration = nil, redis_pool=nil)
+        'ok'
+      end
+
+      def store_for_id(id, status_updates, expiration = nil, redis_pool=nil)
+        'ok'
+      end
+    end
   end
 end
 

data/lib/sidekiq-status/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Status
-    VERSION = '0.6.0'
+    VERSION = '3.0.3'
   end
 end

data/lib/sidekiq-status/web.rb

@@ -6,9 +6,40 @@ module Sidekiq::Status
     # Location of Sidekiq::Status::Web view templates
     VIEW_PATH = File.expand_path('../../../web/views', __FILE__)
 
+    DEFAULT_PER_PAGE_OPTS = [25, 50, 100].freeze
+    DEFAULT_PER_PAGE = 25
+    COMMON_STATUS_HASH_KEYS = %w(update_time jid status worker args label pct_complete total at message working_at elapsed eta)
+
+    class << self
+      def per_page_opts= arr
+        @per_page_opts = arr
+      end
+      def per_page_opts
+        @per_page_opts || DEFAULT_PER_PAGE_OPTS
+      end
+      def default_per_page= val
+        @default_per_page = val
+      end
+      def default_per_page
+        @default_per_page || DEFAULT_PER_PAGE
+      end
+    end
+
     # @param [Sidekiq::Web] app
     def self.registered(app)
+
+      # Allow method overrides to support RESTful deletes
+      app.set :method_override, true
+
       app.helpers do
+        def csrf_tag
+          "<input type='hidden' name='authenticity_token' value='#{session[:csrf]}'/>"
+        end
+
+        def poll_path
+          "?#{request.query_string}" if params[:poll]
+        end
+
         def sidekiq_status_template(name)
           path = File.join(VIEW_PATH, name.to_s) + ".erb"
           File.open(path).read
@@ -16,20 +47,40 @@ module Sidekiq::Status
 
         def add_details_to_status(status)
           status['label'] = status_label(status['status'])
-          status["pct_complete"] = pct_complete(status)
+          status["pct_complete"] ||= pct_complete(status)
+          status["elapsed"] ||= elapsed(status).to_s
+          status["eta"] ||= eta(status).to_s
+          status["custom"] = process_custom_data(status)
           return status
         end
 
+        def process_custom_data(hash)
+          hash.reject { |key, _| COMMON_STATUS_HASH_KEYS.include?(key) }
+        end
+
         def pct_complete(status)
           return 100 if status['status'] == 'complete'
-          Sidekiq::Status::pct_complete(status['jid'])
+          Sidekiq::Status::pct_complete(status['jid']) || 0
+        end
+
+        def elapsed(status)
+          case status['status']
+          when 'complete'
+            Sidekiq::Status.update_time(status['jid']) - Sidekiq::Status.working_at(status['jid'])
+          when 'working', 'retrying'
+            Time.now.to_i - Sidekiq::Status.working_at(status['jid'])
+          end
+        end
+
+        def eta(status)
+          Sidekiq::Status.eta(status['jid']) if status['status'] == 'working'
         end
 
         def status_label(status)
           case status
           when 'complete'
             'success'
-          when 'working'
+          when 'working', 'retrying'
             'warning'
           when 'queued'
             'primary'
@@ -39,62 +90,102 @@ module Sidekiq::Status
         end
 
         def has_sort_by?(value)
-          ["worker", "status", "update_time", "pct_complete", "message"].include?(value)
+          ["worker", "status", "update_time", "pct_complete", "message", "args"].include?(value)
         end
       end
 
       app.get '/statuses' do
-        namespace_jids = Sidekiq.redis{ |conn| conn.keys('sidekiq:status:*') }
-        jids = namespace_jids.map{|id_namespace| id_namespace.split(':').last }
+
+        jids = Sidekiq::Status.redis_adapter do |conn|
+          conn.scan(match: 'sidekiq:status:*', count: 100).map do |key|
+            key.split(':').last
+          end.uniq
+        end
         @statuses = []
 
         jids.each do |jid|
           status = Sidekiq::Status::get_all jid
           next if !status || status.count < 2
           status = add_details_to_status(status)
-          @statuses << OpenStruct.new(status)
+          @statuses << status
         end
 
         sort_by = has_sort_by?(params[:sort_by]) ? params[:sort_by] : "update_time"
         sort_dir = "asc"
 
         if params[:sort_dir] == "asc"
-          @statuses = @statuses.sort { |x,y| x.send(sort_by) <=> y.send(sort_by) }
+          @statuses = @statuses.sort { |x,y| (x[sort_by] <=> y[sort_by]) || -1 }
         else
           sort_dir = "desc"
-          @statuses = @statuses.sort { |y,x| x.send(sort_by) <=> y.send(sort_by) }
+          @statuses = @statuses.sort { |y,x| (x[sort_by] <=> y[sort_by]) || 1 }
         end
-        
-        working_jobs = @statuses.select{|job| job.status == "working"}
-        if working_jobs.size >= 25
-          @statuses = working_jobs
-        else
-          @statuses = (@statuses.size >= 25) ? @statuses.take(25) : @statuses 
+
+        if params[:status] && params[:status] != "all"
+          @statuses = @statuses.select {|job_status| job_status["status"] == params[:status] }
         end
-        
+
+        # Sidekiq pagination
+        @total_size = @statuses.count
+        @count = params[:per_page] ? params[:per_page].to_i : Sidekiq::Status::Web.default_per_page
+        @count = @total_size if params[:per_page] == 'all'
+        @current_page = params[:page].to_i < 1 ? 1 : params[:page].to_i
+        @statuses = @statuses.slice((@current_page - 1) * @count, @count)
+
         @headers = [
-          { id: "worker", name: "Worker/jid", class: nil, url: nil},
-          { id: "status", name: "Status", class: nil, url: nil},
-          { id: "update_time", name: "Last Updated", class: nil, url: nil},
-          { id: "pct_complete", name: "Progress", class: nil, url: nil},
-          { id: "message", name: "Message", class: nil, url: nil}
+          {id: "worker", name: "Worker / JID", class: nil, url: nil},
+          {id: "args", name: "Arguments", class: nil, url: nil},
+          {id: "status", name: "Status", class: nil, url: nil},
+          {id: "update_time", name: "Last Updated", class: nil, url: nil},
+          {id: "pct_complete", name: "Progress", class: nil, url: nil},
+          {id: "elapsed", name: "Time Elapsed", class: nil, url: nil},
+          {id: "eta", name: "ETA", class: nil, url: nil},
         ]
 
         @headers.each do |h|
-          params["sort_by"] = h[:id]
-          params["sort_dir"] = (sort_by == h[:id] && sort_dir == "asc") ? "desc" : "asc"
-          h[:url] = "statuses?" + params.map {|k,v| "#{k}=#{v}" }.join("&")
+          h[:url] = "statuses?" + params.merge("sort_by" => h[:id], "sort_dir" => (sort_by == h[:id] && sort_dir == "asc") ? "desc" : "asc").map{|k, v| "#{k}=#{CGI.escape v.to_s}"}.join("&")
           h[:class] = "sorted_#{sort_dir}" if sort_by == h[:id]
         end
 
         erb(sidekiq_status_template(:statuses))
       end
+
+      app.get '/statuses/:jid' do
+        job = Sidekiq::Status::get_all params['jid']
+
+        if job.empty?
+          throw :halt, [404, {"Content-Type" => "text/html"}, [erb(sidekiq_status_template(:status_not_found))]]
+        else
+          @status = add_details_to_status(job)
+          erb(sidekiq_status_template(:status))
+        end
+      end
+
+      # Retries a failed job from the status list
+      app.put '/statuses' do
+        job = Sidekiq::RetrySet.new.find_job(params[:jid])
+        job ||= Sidekiq::DeadSet.new.find_job(params[:jid])
+        job.retry if job
+        throw :halt, [302, { "Location" => request.referer }, []]
+      end
+
+      # Removes a completed job from the status list
+      app.delete '/statuses' do
+        Sidekiq::Status.delete(params[:jid])
+        throw :halt, [302, { "Location" => request.referer }, []]
+      end
     end
   end
 end
 
-require 'sidekiq/web' unless defined?(Sidekiq::Web)
+unless defined?(Sidekiq::Web)
+  require 'delegate' # Needed for sidekiq 5.x
+  require 'sidekiq/web'
+end
+
 Sidekiq::Web.register(Sidekiq::Status::Web)
+["per_page", "sort_by", "sort_dir", "status"].each do |key|
+  Sidekiq::WebHelpers::SAFE_QPARAMS.push(key)
+end
 if Sidekiq::Web.tabs.is_a?(Array)
   # For sidekiq < 2.5
   Sidekiq::Web.tabs << "statuses"

data/lib/sidekiq-status/worker.rb

@@ -11,14 +11,14 @@ module Sidekiq::Status::Worker
   # @param [Hash] status_updates updated values
   # @return [String] Redis operation status code
   def store(hash)
-    store_for_id @jid, hash, @expiration
+    store_for_id @provider_job_id || @job_id || @jid || "", hash, @expiration
   end
 
   # Read value from job status hash
   # @param String|Symbol hask key
   # @return [String]
   def retrieve(name)
-    read_field_for_id @jid, name
+    read_field_for_id @provider_job_id || @job_id || @jid || "", name
   end
 
   # Sets current task progress
@@ -27,15 +27,22 @@ module Sidekiq::Status::Worker
   # @param String optional message
   # @return [String]
   def at(num, message = nil)
-    total(100) if retrieve(:total).nil?
-    store(at: num, message: message)
+    @_status_total = 100 if @_status_total.nil?
+    pct_complete = ((num / @_status_total.to_f) * 100).to_i rescue 0
+    store(at: num, total: @_status_total, pct_complete: pct_complete, message: message, working_at: working_at)
   end
 
   # Sets total number of tasks
   # @param Fixnum total number of tasks
   # @return [String]
   def total(num)
-    store(total: num)
+    @_status_total = num
+    store(total: num, working_at: working_at)
   end
 
+  private
+
+  def working_at
+    @working_at ||= Time.now.to_i
+  end
 end

data/lib/sidekiq-status.rb

@@ -1,14 +1,18 @@
-require "sidekiq-status/version"
+require 'sidekiq-status/version'
+require 'sidekiq-status/sidekiq_extensions'
 require 'sidekiq-status/storage'
 require 'sidekiq-status/worker'
+require 'sidekiq-status/redis_client_adapter'
+require 'sidekiq-status/redis_adapter'
 require 'sidekiq-status/client_middleware'
 require 'sidekiq-status/server_middleware'
 require 'sidekiq-status/web' if defined?(Sidekiq::Web)
+require 'chronic_duration'
 
 module Sidekiq::Status
   extend Storage
   DEFAULT_EXPIRY = 60 * 30
-  STATUS = [ :queued, :working, :complete, :stopped, :failed, :interrupted ].freeze
+  STATUS = [ :queued, :working, :retrying, :complete, :stopped, :failed, :interrupted ].freeze
 
   class << self
     # Job status by id
@@ -21,8 +25,8 @@ module Sidekiq::Status
     # Get all status fields for a job
     # @params [String] id job id returned by async_perform
     # @return [Hash] hash of all fields stored for the job
-    def get_all(id)
-      read_hash_for_id(id)
+    def get_all(job_id)
+      read_hash_for_id(job_id)
     end
 
     def status(job_id)
@@ -34,6 +38,10 @@ module Sidekiq::Status
       delete_and_unschedule(job_id, job_unix_time)
     end
 
+    def delete(job_id)
+      delete_status(job_id)
+    end
+
     alias_method :unschedule, :cancel
 
     STATUS.each do |name|
@@ -52,11 +60,38 @@ module Sidekiq::Status
     end
 
     def pct_complete(job_id)
-      ((at(job_id) / total(job_id).to_f) * 100 ).to_i if total(job_id).to_f > 0
+      get(job_id, :pct_complete).to_i
+    end
+
+    def working_at(job_id)
+      (get(job_id, :working_at) || Time.now).to_i
+    end
+
+    def update_time(job_id)
+      (get(job_id, :update_time) || Time.now).to_i
+    end
+
+    def eta(job_id)
+      at = at(job_id)
+      return nil if at.zero?
+
+      (Time.now.to_i - working_at(job_id)).to_f / at * (total(job_id) - at)
     end
 
     def message(job_id)
       get(job_id, :message)
     end
+
+    def wrap_redis_connection(conn)
+      if Sidekiq.major_version >= 7
+        conn.is_a?(RedisClientAdapter) ? conn : RedisClientAdapter.new(conn)
+      else
+        conn.is_a?(RedisAdapter) ? conn : RedisAdapter.new(conn)
+      end
+    end
+
+    def redis_adapter
+      Sidekiq.redis { |conn| yield wrap_redis_connection(conn) }
+    end
   end
 end

data/sidekiq-status.gemspec

@@ -2,10 +2,10 @@
 require File.expand_path('../lib/sidekiq-status/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ['Evgeniy Tsvigun']
-  gem.email         = ['utgarda@gmail.com']
+  gem.authors       = ['Evgeniy Tsvigun', 'Kenaniah Cerny']
+  gem.email         = ['utgarda@gmail.com', 'kenaniah@gmail.com']
   gem.summary       = 'An extension to the sidekiq message processing to track your jobs'
-  gem.homepage      = 'http://github.com/utgarda/sidekiq-status'
+  gem.homepage      = 'https://github.com/kenaniah/sidekiq-status'
   gem.license       = 'MIT'
 
   gem.files         = `git ls-files`.split($\)
@@ -14,7 +14,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
   gem.version       = Sidekiq::Status::VERSION
 
-  gem.add_dependency                  'sidekiq', '>= 2.7'
+  gem.add_dependency                  'sidekiq', '>= 6.0', '< 8'
+  gem.add_dependency                  'chronic_duration'
+  gem.add_development_dependency      'appraisal'
+  gem.add_development_dependency      'colorize'
   gem.add_development_dependency      'rack-test'
   gem.add_development_dependency      'rake'
   gem.add_development_dependency      'rspec'

data/spec/environment.rb

@@ -0,0 +1 @@
+# This file has been intentionally left blank

data/spec/lib/sidekiq-status/client_middleware_spec.rb

@@ -5,21 +5,20 @@ describe Sidekiq::Status::ClientMiddleware do
   let!(:redis) { Sidekiq.redis { |conn| conn } }
   let!(:job_id) { SecureRandom.hex(12) }
 
-  # Clean Redis before each test
-  before { redis.flushall }
+  before do
+    allow(SecureRandom).to receive(:hex).once.and_return(job_id)
+  end
+
+  describe "without :expiration parameter" do
 
-  describe "#call" do
-    before { client_middleware }
     it "sets queued status" do
-      allow(SecureRandom).to receive(:hex).once.and_return(job_id)
-      expect(StubJob.perform_async(:arg1 => 'val1')).to eq(job_id)
+      expect(StubJob.perform_async 'arg1' => 'val1').to eq(job_id)
       expect(redis.hget("sidekiq:status:#{job_id}", :status)).to eq('queued')
       expect(Sidekiq::Status::queued?(job_id)).to be_truthy
     end
 
     it "sets status hash ttl" do
-      allow(SecureRandom).to receive(:hex).once.and_return(job_id)
-      expect(StubJob.perform_async(:arg1 => 'val1')).to eq(job_id)
+      expect(StubJob.perform_async 'arg1' => 'val1').to eq(job_id)
       expect(1..Sidekiq::Status::DEFAULT_EXPIRY).to cover redis.ttl("sidekiq:status:#{job_id}")
     end
 
@@ -38,18 +37,22 @@ describe Sidekiq::Status::ClientMiddleware do
         Sidekiq::Status::ClientMiddleware.new.call(StubJob, {'jid' => SecureRandom.hex}, :queued) do end
       end
     end
+
   end
 
-  describe ":expiration parameter" do
+  describe "with :expiration parameter" do
+
     let(:huge_expiration) { Sidekiq::Status::DEFAULT_EXPIRY * 100 }
+
+    # Ensure client middleware is loaded with an expiration parameter set
     before do
-      allow(SecureRandom).to receive(:hex).once.and_return(job_id)
+      client_middleware expiration: huge_expiration
     end
 
     it "overwrites default expiry value" do
-      client_middleware(expiration: huge_expiration)
-      StubJob.perform_async(:arg1 => 'val1')
+      StubJob.perform_async 'arg1' => 'val1'
       expect((Sidekiq::Status::DEFAULT_EXPIRY+1)..huge_expiration).to cover redis.ttl("sidekiq:status:#{job_id}")
     end
+
   end
 end