sidekiq 2.4.0 → 2.5.0

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 dump.rdb
 .rbx
 coverage/
+.sass-cache/

data/Changes.md

@@ -1,3 +1,14 @@
+2.5.0
+-----------
+
+- REDESIGNED WEB UI! [unity, cavneb]
+- Support Honeybadger for error delivery
+- Inline testing runs the client middleware before executing jobs [#465]
+- Web UI can now remove jobs from queue. [#466, dleung]
+- Web UI can now show the full message, not just 100 chars [#464, dleung]
+- Add APIs for manipulating the retry and job queues.  See sidekiq/api. [#457]
+
+
 2.4.0
 -----------
 

data/Gemfile

@@ -10,3 +10,9 @@ gem 'sqlite3'
 group :test do
   gem 'simplecov', :require => false
 end
+
+group :development do
+  gem 'sprockets-sass'
+  gem 'compass'
+  gem 'shotgun'
+end

data/lib/sidekiq.rb

@@ -5,6 +5,7 @@ require 'sidekiq/worker'
 require 'sidekiq/redis_connection'
 require 'sidekiq/util'
 require 'sidekiq/stats'
+require 'sidekiq/api'
 
 require 'sidekiq/extensions/class_methods'
 require 'sidekiq/extensions/action_mailer'

data/lib/sidekiq/api.rb

@@ -0,0 +1,184 @@
+require 'sidekiq'
+
+module Sidekiq
+
+  ##
+  # Encapsulates a queue within Sidekiq.
+  # Allows enumeration of all jobs within the queue
+  # and deletion of jobs.
+  #
+  #   queue = Sidekiq::Queue.new("mailer")
+  #   queue.each do |job|
+  #     job.klass # => 'MyWorker'
+  #     job.args # => [1, 2, 3]
+  #     job.delete if job.jid == 'abcdef1234567890'
+  #   end
+  #
+  class Queue
+    include Enumerable
+
+    attr_reader :name
+
+    def initialize(name="default")
+      @name = name
+      @rname = "queue:#{name}"
+    end
+
+    def size
+      Sidekiq.redis { |con| con.llen(@rname) }
+    end
+
+    def each(&block)
+      page = 0
+      page_size = 50
+
+      loop do
+        entries = Sidekiq.redis do |conn|
+          conn.lrange @rname, page * page_size, (page * page_size) + page_size - 1
+        end
+        break if entries.empty?
+        page += 1
+        entries.each do |entry|
+          block.call Job.new(entry, @name)
+        end
+      end
+    end
+  end
+
+  ##
+  # Encapsulates a pending job within a Sidekiq queue or
+  # sorted set.
+  #
+  # The job should be considered immutable but may be
+  # removed from the queue via Job#delete.
+  #
+  class Job
+    attr_reader :item
+
+    def initialize(item, queue_name=nil)
+      @value = item
+      @item = Sidekiq.load_json(item)
+      @queue = queue_name || @item['queue']
+    end
+
+    def klass
+      @item['class']
+    end
+
+    def args
+      @item['args']
+    end
+
+    def jid
+      @item['jid']
+    end
+
+    def queue
+      @queue
+    end
+
+    ##
+    # Remove this job from the queue.
+    def delete
+      count = Sidekiq.redis do |conn|
+        conn.lrem("queue:#{@queue}", 0, @value)
+      end
+      count != 0
+    end
+
+    def [](name)
+      @item.send(:[], name)
+    end
+  end
+
+  class SortedEntry < Job
+    attr_reader :score
+
+    def initialize(parent, score, item)
+      super(item)
+      @score = score
+      @parent = parent
+    end
+
+    def at
+      Time.at(@score)
+    end
+
+    def delete
+      @parent.delete(@score)
+    end
+  end
+
+  class SortedSet
+    include Enumerable
+
+    def initialize(name)
+      @zset = name
+    end
+
+    def size
+      Sidekiq.redis {|c| c.zcard(@zset) }
+    end
+
+    def each(&block)
+      # page thru the sorted set backwards so deleting entries doesn't screw up indexing
+      page = -1
+      page_size = 50
+
+      loop do
+        elements = Sidekiq.redis do |conn|
+          conn.zrange @zset, page * page_size, (page * page_size) + (page_size - 1), :with_scores => true
+        end
+        break if elements.empty?
+        page -= 1
+        elements.each do |element, score|
+          block.call SortedEntry.new(self, score, element)
+        end
+      end
+    end
+
+    def delete(score)
+      count = Sidekiq.redis do |conn|
+        conn.zremrangebyscore(@zset, score, score)
+      end
+      count != 0
+    end
+  end
+
+  ##
+  # Allows enumeration of scheduled jobs within Sidekiq.
+  # Based on this, you can search/filter for jobs.  Here's an
+  # example where I'm selecting all jobs of a certain type
+  # and deleting them from the retry queue.
+  #
+  #   r = Sidekiq::ScheduledSet.new
+  #   r.select do |retri|
+  #     retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
+  #     retri.args[0] == 'User' &&
+  #     retri.args[1] == 'setup_new_subscriber'
+  #   end.map(&:delete)
+  class ScheduledSet < SortedSet
+    def initialize
+      super 'schedule'
+    end
+  end
+
+  ##
+  # Allows enumeration of retries within Sidekiq.
+  # Based on this, you can search/filter for jobs.  Here's an
+  # example where I'm selecting all jobs of a certain type
+  # and deleting them from the retry queue.
+  #
+  #   r = Sidekiq::RetrySet.new
+  #   r.select do |retri|
+  #     retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
+  #     retri.args[0] == 'User' &&
+  #     retri.args[1] == 'setup_new_subscriber'
+  #   end.map(&:delete)
+  class RetrySet < SortedSet
+    def initialize
+      super 'retry'
+    end
+  end
+
+end

data/lib/sidekiq/cli.rb

@@ -32,6 +32,7 @@ require 'yaml'
 require 'singleton'
 require 'optparse'
 require 'celluloid'
+require 'erb'
 
 require 'sidekiq'
 require 'sidekiq/util'
@@ -149,7 +150,7 @@ module Sidekiq
 
       @parser = OptionParser.new do |o|
         o.on "-q", "--queue QUEUE[,WEIGHT]...", "Queues to process with optional weights" do |arg|
-          queues_and_weights = arg.scan(/([\w-]+),?(\d*)/)
+          queues_and_weights = arg.scan(/([\w\.-]+),?(\d*)/)
           queues_and_weights.each {|queue_and_weight| parse_queues(opts, *queue_and_weight)}
           opts[:strict] = queues_and_weights.collect(&:last).none? {|weight| weight != ''}
         end
@@ -208,7 +209,7 @@ module Sidekiq
     def parse_config(cli)
       opts = {}
       if cli[:config_file] && File.exist?(cli[:config_file])
-        opts = YAML.load_file cli[:config_file]
+        opts = YAML.load(ERB.new(IO.read(cli[:config_file])).result)
         queues = opts.delete(:queues) || []
         queues.each { |name, weight| parse_queues(opts, name, weight) }
       end

data/lib/sidekiq/client.rb

@@ -40,16 +40,7 @@ module Sidekiq
       normed, payload = process_single(item['class'], normed)
 
       pushed = false
-      Sidekiq.redis do |conn|
-        if normed['at']
-          pushed = conn.zadd('schedule', normed['at'].to_s, payload)
-        else
-          _, pushed = conn.multi do
-            conn.sadd('queues', normed['queue'])
-            conn.rpush("queue:#{normed['queue']}", payload)
-          end
-        end
-      end if normed
+      pushed = raw_push(normed, payload) if normed
       pushed ? normed['jid'] : nil
     end
 
@@ -104,6 +95,21 @@ module Sidekiq
 
     private
 
+    def self.raw_push(normed, payload) # :nodoc:
+      pushed = false
+      Sidekiq.redis do |conn|
+        if normed['at']
+          pushed = conn.zadd('schedule', normed['at'].to_s, payload)
+        else
+          _, pushed = conn.multi do
+            conn.sadd('queues', normed['queue'])
+            conn.rpush("queue:#{normed['queue']}", payload)
+          end
+        end
+      end
+      pushed
+    end
+
     def self.process_single(worker_class, item)
       queue = item['queue']
 

data/lib/sidekiq/core_ext.rb

@@ -51,4 +51,34 @@ rescue LoadError
   end
 end
 
+begin
+  require 'active_support/core_ext/hash/keys'
+rescue LoadError
+  class Hash
+    def stringify_keys(hash)
+      hash.keys.each do |key|
+        hash[key.to_s] = hash.delete(key)
+      end
+      hash
+    end
+  end if !{}.responds_to(:stringify_keys)
+end
+
+begin
+  require 'active_support/core_ext/string/inflections'
+rescue LoadError
+  class String
+    def constantize
+      names = self.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    end
+  end if !"".responds_to(:constantize)
+end
+
 

data/lib/sidekiq/exception_handler.rb

@@ -5,7 +5,12 @@ module Sidekiq
       Sidekiq.logger.warn msg
       Sidekiq.logger.warn ex
       Sidekiq.logger.warn ex.backtrace.join("\n")
+      # This list of services is getting a bit ridiculous.
+      # For future error services, please add your own
+      # middleware like BugSnag does:
+      # https://github.com/bugsnag/bugsnag-ruby/blob/master/lib/bugsnag/sidekiq.rb
       send_to_airbrake(msg, ex) if defined?(::Airbrake)
+      send_to_honeybadger(msg, ex) if defined?(::Honeybadger)
       send_to_exceptional(msg, ex) if defined?(::Exceptional)
       send_to_exception_notifier(msg, ex) if defined?(::ExceptionNotifier)
     end
@@ -16,6 +21,10 @@ module Sidekiq
       ::Airbrake.notify_or_ignore(ex, :parameters => msg)
     end
 
+    def send_to_honeybadger(msg, ex)
+      ::Honeybadger.notify_or_ignore(ex, :parameters => msg)
+    end
+
     def send_to_exceptional(msg, ex)
       if ::Exceptional::Config.should_send_to_api?
         ::Exceptional.context(msg)

data/lib/sidekiq/processor.rb

@@ -34,7 +34,7 @@ module Sidekiq
       defer do
         begin
           msg = Sidekiq.load_json(msgstr)
-          klass  = constantize(msg['class'])
+          klass  = msg['class'].constantize
           worker = klass.new
 
           stats(worker, msg, queue) do

data/lib/sidekiq/testing/inline.rb

@@ -27,9 +27,9 @@ module Sidekiq
     #   assert_equal 1, $external_variable
     #
     singleton_class.class_eval do
-      alias_method :push_old, :push
-      def push(hash)
-        hash['class'].new.perform(*Sidekiq.load_json(Sidekiq.dump_json(hash['args'])))
+      alias_method :raw_push_old, :raw_push
+      def raw_push(hash, _)
+        hash['class'].constantize.new.perform(*Sidekiq.load_json(Sidekiq.dump_json(hash['args'])))
         true
       end
     end

data/lib/sidekiq/util.rb

@@ -1,5 +1,6 @@
 require 'socket'
 require 'sidekiq/exception_handler'
+require 'sidekiq/core_ext'
 
 module Sidekiq
   ##
@@ -10,17 +11,6 @@ module Sidekiq
 
     EXPIRY = 60 * 60 * 24
 
-    def constantize(camel_cased_word)
-      names = camel_cased_word.split('::')
-      names.shift if names.empty? || names.first.empty?
-
-      constant = Object
-      names.each do |name|
-        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-      end
-      constant
-    end
-
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.4.0"
+  VERSION = "2.5.0"
 end

data/lib/sidekiq/web.rb

@@ -1,6 +1,9 @@
 require 'sinatra/base'
 require 'slim'
 require 'sprockets'
+require 'sprockets-sass'
+require 'sass'
+require 'compass'
 require 'sidekiq/paginator'
 
 module Sidekiq
@@ -16,6 +19,33 @@ module Sidekiq
       @environment.append_path 'assets/stylesheets'
       @environment.append_path 'assets/stylesheets/vendor'
       @environment.append_path 'assets/images'
+
+      Compass.configuration do |config|
+
+        config.project_path = "#{@root}/assets"
+
+        config.images_dir           = 'images'
+        config.sass_dir             = 'stylesheets'
+        config.css_dir              = 'stylesheets'
+        config.javascripts_dir      = 'javascripts'
+        config.fonts_dir            = 'stylesheets/fonts'
+
+        config.http_images_path     = '/assets'
+        config.http_generated_images_path     = '/assets'
+        config.http_javascripts_path     = '/assets'
+        config.http_stylesheets_path     = '/assets'
+
+        # You can select your preferred output style here (can be overridden via the command line):
+        output_style = :compressed
+
+        # To enable relative paths to assets via compass helper functions. Uncomment:
+        relative_assets = true
+
+        # To disable debugging comments that display the original location of your selectors. Uncomment:
+        line_comments = false
+      end
+
+
     end
 
     def call(env)
@@ -36,8 +66,10 @@ module Sidekiq
     set :views,  "#{dir}/views"
     set :root, "#{dir}/public"
     set :slim, :pretty => true
+
     use SprocketsMiddleware, :root => dir
 
+
     helpers do
 
       def reset_worker_list
@@ -99,6 +131,10 @@ module Sidekiq
         "#{env['SCRIPT_NAME']}/"
       end
 
+      def current_path
+        @current_path ||= request.path_info.gsub(/^\//,'')
+      end
+
       def current_status
         return 'idle' if workers.size == 0
         return 'active'
@@ -165,6 +201,13 @@ module Sidekiq
       redirect "#{root_path}queues"
     end
 
+    post "/queues/:name/delete" do
+      Sidekiq.redis do |conn|
+        conn.lrem("queue:#{params[:name]}", 0, params[:key_val])
+      end
+      redirect "#{root_path}queues/#{params[:name]}"
+    end
+
     get "/retries/:score" do
       halt 404 unless params[:score]
       @score = params[:score].to_f
@@ -241,7 +284,12 @@ module Sidekiq
     end
 
     def self.tabs
-      @tabs ||= ["Queues", "Retries", "Scheduled"]
+      @tabs ||= {
+        "Workers"   =>'',
+        "Queues"    =>'queues',
+        "Retries"   =>'retries',
+        "Scheduled" =>'scheduled'
+      }
     end
 
   end