sidekiq 2.15.1 → 2.15.2

data/Changes.md

@@ -1,3 +1,10 @@
+2.15.2
+-----------
+
+- Iterating over Sidekiq::Queue and Sidekiq::SortedSet will now work as
+  intended when jobs are deleted [#866, aackerman]
+- A few more minor Web UI fixes [#1247]
+
 2.15.1
 -----------
 

data/lib/sidekiq/api.rb

@@ -117,18 +117,23 @@ module Sidekiq
     end
 
     def each(&block)
+      initial_size = size
+      deleted_size = 0
       page = 0
       page_size = 50
 
       loop do
+        range_start = page * page_size - deleted_size
+        range_end   = page * page_size - deleted_size + (page_size - 1)
         entries = Sidekiq.redis do |conn|
-          conn.lrange @rname, page * page_size, (page * page_size) + page_size - 1
+          conn.lrange @rname, range_start, range_end
         end
         break if entries.empty?
         page += 1
         entries.each do |entry|
           block.call Job.new(entry, @name)
         end
+        deleted_size = initial_size - size
       end
     end
 
@@ -265,19 +270,23 @@ module Sidekiq
     end
 
     def each(&block)
-      # page thru the sorted set backwards so deleting entries doesn't screw up indexing
+      initial_size = size
+      deleted_size = 0
       page = -1
       page_size = 50
 
       loop do
+        range_start = page * page_size + deleted_size
+        range_end   = page * page_size + deleted_size + (page_size - 1)
         elements = Sidekiq.redis do |conn|
-          conn.zrange @zset, page * page_size, (page * page_size) + (page_size - 1), :with_scores => true
+          conn.zrange @zset, range_start, range_end, :with_scores => true
         end
         break if elements.empty?
         page -= 1
         elements.each do |element, score|
           block.call SortedEntry.new(self, score, element)
         end
+        deleted_size = initial_size - size
       end
     end
 

data/lib/sidekiq/exception_handler.rb

@@ -4,7 +4,7 @@ module Sidekiq
     def handle_exception(ex, ctxHash={})
       Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
       Sidekiq.logger.warn ex
-      Sidekiq.logger.warn ex.backtrace.join("\n")
+      Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
       # This list of services is getting a bit ridiculous.
       # For future error services, please add your own
       # middleware like BugSnag does:

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.15.1"
+  VERSION = "2.15.2"
 end

data/lib/sidekiq/web.rb

@@ -5,6 +5,7 @@ require 'sinatra/base'
 require 'sidekiq'
 require 'sidekiq/api'
 require 'sidekiq/paginator'
+require 'sidekiq/web_helpers'
 
 module Sidekiq
   class Web < Sinatra::Base
@@ -15,166 +16,25 @@ module Sidekiq
     set :views, Proc.new { "#{root}/views" }
     set :locales, Proc.new { "#{root}/locales" }
 
-    helpers do
-      def strings
-        @strings ||= begin
-          Dir["#{settings.locales}/*.yml"].inject({}) do |memo, file|
-            memo.merge(YAML.load(File.open(file)))
-          end
-        end
-      end
-
-      def locale
-        lang = (request.env["HTTP_ACCEPT_LANGUAGE"] || 'en')[0,2]
-        strings[lang] ? lang : 'en'
-      end
-
-      def get_locale
-        strings[locale]
-      end
-
-      def t(msg, options={})
-        string = get_locale[msg] || msg
-        string % options
-      end
-
-      def reset_worker_list
-        Sidekiq.redis do |conn|
-          workers = conn.smembers('workers')
-          conn.srem('workers', workers) if !workers.empty?
-        end
-      end
-
-      def workers_size
-        @workers_size ||= Sidekiq.redis do |conn|
-          conn.scard('workers')
-        end
-      end
-
-      def workers
-        @workers ||= begin
-          to_rem = []
-          workers = Sidekiq.redis do |conn|
-            conn.smembers('workers').map do |w|
-              msg = conn.get("worker:#{w}")
-              msg ? [w, Sidekiq.load_json(msg)] : (to_rem << w; nil)
-            end.compact.sort { |x| x[1] ? -1 : 1 }
-          end
-
-          # Detect and clear out any orphaned worker records.
-          # These can be left in Redis if Sidekiq crashes hard
-          # while processing jobs.
-          if to_rem.size > 0
-            Sidekiq.redis { |conn| conn.srem('workers', to_rem) }
-          end
-          workers
-        end
-      end
-
-      def stats
-        @stats ||= Sidekiq::Stats.new
-      end
-
-      def retries_with_score(score)
-        Sidekiq.redis do |conn|
-          results = conn.zrangebyscore('retry', score, score)
-          results.map { |msg| Sidekiq.load_json(msg) }
-        end
-      end
-
-      def location
-        Sidekiq.redis { |conn| conn.client.location }
-      end
-
-      def namespace
-        @@ns ||= Sidekiq.redis {|conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
-      end
-
-      def root_path
-        "#{env['SCRIPT_NAME']}/"
-      end
+    helpers WebHelpers
 
-      def current_path
-        @current_path ||= request.path_info.gsub(/^\//,'')
-      end
-
-      def current_status
-        return 'idle' if workers_size == 0
-        return 'active'
-      end
-
-      def relative_time(time)
-        %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
-      end
-
-      def job_params(job, score)
-        "#{score}-#{job['jid']}"
-      end
-
-      def parse_params(params)
-        score, jid = params.split("-")
-        [score.to_f, jid]
-      end
-
-      def truncate(text, truncate_after_chars = 2000)
-        truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
-      end
-
-      def display_args(args, truncate_after_chars = 2000)
-        args.map do |arg|
-          a = arg.inspect
-          truncate(a)
-        end.join(", ")
-      end
-
-      RETRY_JOB_KEYS = Set.new(%w(
-        queue class args retry_count retried_at failed_at
-        jid error_message error_class backtrace
-        error_backtrace enqueued_at retry
-      ))
-
-      def retry_extra_items(retry_job)
-        @retry_extra_items ||= {}.tap do |extra|
-          retry_job.item.each do |key, value|
-            extra[key] = value unless RETRY_JOB_KEYS.include?(key)
-          end
-        end
-      end
+    DEFAULT_TABS = {
+      "Dashboard" => '',
+      "Workers"   => 'workers',
+      "Queues"    => 'queues',
+      "Retries"   => 'retries',
+      "Scheduled" => 'scheduled',
+    }
 
-      def tabs
-        @tabs ||= {
-          "Dashboard" => '',
-          "Workers"   => 'workers',
-          "Queues"    => 'queues',
-          "Retries"   => 'retries',
-          "Scheduled" => 'scheduled',
-        }
+    class << self
+      def default_tabs
+        DEFAULT_TABS
       end
 
       def custom_tabs
-        self.class.tabs
-      end
-
-      def number_with_delimiter(number)
-        begin
-          Float(number)
-        rescue ArgumentError, TypeError
-          return number
-        end
-
-        options = {:delimiter => ',', :separator => '.'}
-        parts = number.to_s.to_str.split('.')
-        parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-        parts.join(options[:separator])
-      end
-
-      def redis_keys
-        ["redis_stats", "uptime_in_days", "connected_clients", "used_memory_human", "used_memory_peak_human"]
-      end
-
-      def h(text)
-        ERB::Util.h(text)
+        @custom_tabs ||= {}
       end
+      alias_method :tabs, :custom_tabs
     end
 
     get "/workers" do
@@ -207,7 +67,7 @@ module Sidekiq
 
     post "/queues/:name/delete" do
       Sidekiq::Job.new(params[:key_val], params[:name]).delete
-      redirect "#{root_path}queues/#{params[:name]}"
+      redirect_with_query("#{root_path}queues/#{params[:name]}")
     end
 
     get '/retries' do
@@ -236,7 +96,7 @@ module Sidekiq
           job.delete
         end
       end
-      redirect "#{root_path}retries"
+      redirect_with_query("#{root_path}retries")
     end
 
     post "/retries/all/delete" do
@@ -259,7 +119,7 @@ module Sidekiq
           job.delete
         end
       end
-      redirect "#{root_path}retries"
+      redirect_with_query("#{root_path}retries")
     end
 
     get '/scheduled' do
@@ -289,7 +149,7 @@ module Sidekiq
           end
         end
       end
-      redirect "#{root_path}scheduled"
+      redirect_with_query("#{root_path}scheduled")
     end
 
     post "/scheduled/:key" do
@@ -302,20 +162,23 @@ module Sidekiq
           job.delete
         end
       end
-      redirect "#{root_path}scheduled"
+      redirect_with_query("#{root_path}scheduled")
     end
 
     get '/' do
-      @redis_info = Sidekiq.redis { |conn| conn.info }.select{ |k, v| redis_keys.include? k }
+      @redis_info = Sidekiq.redis { |conn| conn.info }.select{ |k, v| REDIS_KEYS.include? k }
       stats_history = Sidekiq::Stats::History.new((params[:days] || 30).to_i)
       @processed_history = stats_history.processed
       @failed_history = stats_history.failed
       erb :dashboard
     end
 
+    REDIS_KEYS = %w(redis_stats uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+
     get '/dashboard/stats' do
       sidekiq_stats = Sidekiq::Stats.new
-      redis_stats   = Sidekiq.redis { |conn| conn.info }.select{ |k, v| redis_keys.include? k }
+      queue         = Sidekiq::Queue.new
+      redis_stats   = Sidekiq.redis { |conn| conn.info }.select{ |k, v| REDIS_KEYS.include? k }
 
       content_type :json
       Sidekiq.dump_json({
@@ -326,15 +189,11 @@ module Sidekiq
           enqueued:   sidekiq_stats.enqueued,
           scheduled:  sidekiq_stats.scheduled_size,
           retries:    sidekiq_stats.retry_size,
+          default_latency: queue.latency,
         },
         redis: redis_stats
       })
     end
 
-    def self.tabs
-      @custom_tabs ||= {}
-    end
-
   end
-
 end

data/lib/sidekiq/web_helpers.rb

@@ -0,0 +1,158 @@
+require 'uri'
+
+module Sidekiq
+  # This is not a public API
+  module WebHelpers
+    def strings
+      @@strings ||= begin
+        Dir["#{settings.locales}/*.yml"].inject({}) do |memo, file|
+          memo.merge(YAML.load(File.open(file)))
+        end
+      end
+    end
+
+    def locale
+      lang = (request.env["HTTP_ACCEPT_LANGUAGE"] || 'en')[0,2]
+      strings[lang] ? lang : 'en'
+    end
+
+    def get_locale
+      strings[locale]
+    end
+
+    def t(msg, options={})
+      string = get_locale[msg] || msg
+      string % options
+    end
+
+    def reset_worker_list
+      Sidekiq.redis do |conn|
+        workers = conn.smembers('workers')
+        conn.srem('workers', workers) if !workers.empty?
+      end
+    end
+
+    def workers_size
+      @workers_size ||= Sidekiq.redis do |conn|
+        conn.scard('workers')
+      end
+    end
+
+    def workers
+      @workers ||= begin
+        to_rem = []
+        workers = Sidekiq.redis do |conn|
+          conn.smembers('workers').map do |w|
+            msg = conn.get("worker:#{w}")
+            msg ? [w, Sidekiq.load_json(msg)] : (to_rem << w; nil)
+          end.compact.sort { |x| x[1] ? -1 : 1 }
+        end
+
+        # Detect and clear out any orphaned worker records.
+        # These can be left in Redis if Sidekiq crashes hard
+        # while processing jobs.
+        if to_rem.size > 0
+          Sidekiq.redis { |conn| conn.srem('workers', to_rem) }
+        end
+        workers
+      end
+    end
+
+    def stats
+      @stats ||= Sidekiq::Stats.new
+    end
+
+    def retries_with_score(score)
+      Sidekiq.redis do |conn|
+        conn.zrangebyscore('retry', score, score)
+      end.map { |msg| Sidekiq.load_json(msg) }
+    end
+
+    def location
+      Sidekiq.redis { |conn| conn.client.location }
+    end
+
+    def namespace
+      @@ns ||= Sidekiq.redis {|conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
+    end
+
+    def root_path
+      "#{env['SCRIPT_NAME']}/"
+    end
+
+    def current_path
+      @current_path ||= request.path_info.gsub(/^\//,'')
+    end
+
+    def current_status
+      workers_size == 0 ? 'idle' : 'active'
+    end
+
+    def relative_time(time)
+      %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
+    end
+
+    def job_params(job, score)
+      "#{score}-#{job['jid']}"
+    end
+
+    def parse_params(params)
+      score, jid = params.split("-")
+      [score.to_f, jid]
+    end
+
+    def truncate(text, truncate_after_chars = 2000)
+      truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
+    end
+
+    def display_args(args, truncate_after_chars = 2000)
+      args.map do |arg|
+        a = arg.inspect
+        truncate(a)
+      end.join(", ")
+    end
+
+    RETRY_JOB_KEYS = Set.new(%w(
+      queue class args retry_count retried_at failed_at
+      jid error_message error_class backtrace
+      error_backtrace enqueued_at retry
+    ))
+
+    def retry_extra_items(retry_job)
+      @retry_extra_items ||= {}.tap do |extra|
+        retry_job.item.each do |key, value|
+          extra[key] = value unless RETRY_JOB_KEYS.include?(key)
+        end
+      end
+    end
+
+    def number_with_delimiter(number)
+      begin
+        Float(number)
+      rescue ArgumentError, TypeError
+        return number
+      end
+
+      options = {:delimiter => ',', :separator => '.'}
+      parts = number.to_s.to_str.split('.')
+      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
+      parts.join(options[:separator])
+    end
+
+    def h(text)
+      ERB::Util.h(text)
+    end
+
+    # Any paginated list that performs an action needs to redirect
+    # back to the proper page after performing that action.
+    def redirect_with_query(url)
+      r = request.referer
+      if r && r =~ /\?/
+        ref = URI(r)
+        redirect("#{url}?#{ref.query}")
+      else
+        redirect url
+      end
+    end
+  end
+end

data/sidekiq.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency                  'redis', '>= 3.0.4'
   gem.add_dependency                  'redis-namespace', '>= 1.3.1'
   gem.add_dependency                  'connection_pool', '>= 1.0.0'
-  gem.add_dependency                  'celluloid', '>= 0.15.1'
+  gem.add_dependency                  'celluloid', '>= 0.15.2'
   gem.add_dependency                  'json'
   gem.add_development_dependency      'sinatra'
   gem.add_development_dependency      'minitest', '~> 4.2'

data/test/config.yml

@@ -1,6 +1,5 @@
 ---
 :verbose:      false
-:environment:  xzibit
 :require:      ./test/fake_env.rb
 :pidfile:      /tmp/sidekiq-config-test.pid
 :logfile:      /tmp/sidekiq.log

data/test/helper.rb

@@ -20,7 +20,8 @@ end
 require 'minitest/autorun'
 require 'minitest/pride'
 
-require 'celluloid/autostart'
+require 'celluloid/test'
+Celluloid.boot
 require 'sidekiq'
 require 'sidekiq/util'
 Sidekiq.logger.level = Logger::ERROR
@@ -34,9 +35,3 @@ REDIS = Sidekiq::RedisConnection.create(:url => redis_url, :namespace => 'testy'
 Sidekiq.configure_client do |config|
   config.redis = { :url => redis_url, :namespace => 'testy' }
 end
-
-Celluloid.class_eval do
-  def self.shutdown
-    $stderr.puts "Celluloid shutdown disabled"
-  end
-end

data/test/test_api.rb

@@ -192,6 +192,26 @@ class TestApi < Sidekiq::Test
       assert_in_delta job.latency, 0.0, 0.01
     end
 
+    it 'can remove jobs when iterating over a sorted set' do
+      # scheduled jobs must be greater than SortedSet#each underlying page size
+      51.times do
+        ApiWorker.perform_in(100, 'aaron')
+      end
+      set = Sidekiq::ScheduledSet.new
+      set.map(&:delete)
+      assert_equal set.size, 0
+    end
+
+    it 'can remove jobs when iterating over a queue' do
+      # initial queue size must be greater than Queue#each underlying page size
+      51.times do
+        ApiWorker.perform_async(1, 'aaron')
+      end
+      q = Sidekiq::Queue.new
+      q.map(&:delete)
+      assert_equal q.size, 0
+    end
+
     it 'can find job by id in queues' do
       q = Sidekiq::Queue.new
       job_id = ApiWorker.perform_async(1, 'jason')

data/test/test_cli.rb

@@ -174,8 +174,8 @@ class TestCli < Sidekiq::Test
         assert_equal './test/fake_env.rb', Sidekiq.options[:require]
       end
 
-      it 'sets environment' do
-        assert_equal 'xzibit', Sidekiq.options[:environment]
+      it 'does not set environment' do
+        assert_equal nil, Sidekiq.options[:environment]
       end
 
       it 'sets concurrency' do

data/test/test_exception_handler.rb

@@ -37,6 +37,20 @@ class TestExceptionHandler < Sidekiq::Test
       assert_match /Something didn't work!/, log[1], "didn't include the exception message"
       assert_match /test\/test_exception_handler.rb/, log[2], "didn't include the backtrace"
     end
+
+    describe "when the exception does not have a backtrace" do
+      it "does not fail" do
+        exception = ExceptionHandlerTestException.new
+        assert_nil exception.backtrace
+
+        begin
+          Component.new.handle_exception exception
+          pass
+        rescue => e
+          flunk "failed handling a nil backtrace"
+        end
+      end
+    end
   end
 
   describe "with fake Airbrake" do

data/test/test_web.rb

@@ -324,6 +324,10 @@ class TestWeb < Sidekiq::Test
         it 'reports scheduled' do
           assert_equal 3, @response["sidekiq"]["scheduled"]
         end
+
+        it 'reports latency' do
+          assert_equal 0, @response["sidekiq"]["default_latency"]
+        end
       end
 
       describe "for redis" do

data/web/views/_nav.erb

@@ -5,7 +5,7 @@
       <%= erb :_status %>
     </a>
     <ul class="nav navbar-nav">
-      <% tabs.each do |title, url| %>
+      <% Sidekiq::Web.default_tabs.each do |title, url| %>
         <% if url == '' %>
           <li class="<%= current_path == url ? 'active' : '' %>">
             <a href="<%= root_path %><%= url %>"><%= t(title) %></a>
@@ -16,14 +16,11 @@
           </li>
         <% end %>
       <% end %>
-      <% custom_tabs.each do |title, url| %>
+      <% Sidekiq::Web.custom_tabs.each do |title, url| %>
         <li class="<%= current_path.start_with?(url) ? 'active' : '' %>">
           <a href="<%= root_path %><%= url %>"><%= t(title) %></a>
         </li>
       <% end %>
-
-
-
     </ul>
 
     <div class="col-sm-2 pull-right poll-wrapper">

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 2.15.1
+  version: 2.15.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-04 00:00:00.000000000 Z
+date: 2013-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.15.1
+        version: 0.15.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.15.1
+        version: 0.15.2
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -240,6 +240,7 @@ files:
 - lib/sidekiq/util.rb
 - lib/sidekiq/version.rb
 - lib/sidekiq/web.rb
+- lib/sidekiq/web_helpers.rb
 - lib/sidekiq/worker.rb
 - lib/sidekiq/yaml_patch.rb
 - sidekiq.gemspec