sidekiq 5.0.0.beta1 → 5.0.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95da469a298b66ee3541eebedff02fa7e78bafa5
-  data.tar.gz: 527583034d9da07b968edaf9e5494d86fd953a3c
+  metadata.gz: 30b76f139772bc64e9e195dea6f3902457a3c57f
+  data.tar.gz: 494f165c4db4cc2a157eddc96edc6f4482e1e24e
 SHA512:
-  metadata.gz: 69243c242c3a520711d6afd75a986287b0a064751565d11ee136de74e74261c42646f2f2d0dd6c590213f58bb2ad40bd5c0aca7478a614cc6811b4aa27dfaa7a
-  data.tar.gz: 21159d28edee7f271deec190f7ce013d91cdac0d91e69c995643c26d0c4df97ce7425131f3e749da0e5880a374569fb1361cdd1c5ec598150b2d29a49f8fbbcf
+  metadata.gz: 372324120d0b6790f2dce79e46f75cfacb1889138ac355365480fb3bde66d315739d63308e3ed51dd95a1cf0a9f0a89d663374d22cbcade15b2cfc0155d23077
+  data.tar.gz: '0905217ddba496c76b7afbfa68765eb6e256924264ec5c5f3d1c76b29d2ff98d87300f6b266486effd00b1373b2fad1f2e1824b5b7242f938cd2cd8cd78c6656'

data/Changes.md

@@ -17,6 +17,12 @@ Sidekiq::Middleware::Server::Logging -> Sidekiq::JobLogging
 - Rails 3.2 and Ruby 2.0 and 2.1 are no longer supported.
 - Please see the [5.0 Upgrade notes](5.0-Upgrade.md) for more detail.
 
+master
+-----------
+
+- Fix error class name display on retry page [#3348]
+- More robust latency calculation [#3340]
+
 4.2.9
 -----------
 

data/Pro-Changes.md

@@ -3,10 +3,11 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
-HEAD
+3.4.3
 ---------
 
-- Update reliable scheduler to enqueue up to 100 jobs per call, minimizing Redis latency [#3332]
+- Limit reliable scheduler to enqueue up to 100 jobs per call, minimizing Redis latency [#3332]
+- Fix bug in super\_fetch logic for queues with `_` in the name [#3339]
 
 3.4.2
 ---------

data/bin/sidekiqload

@@ -14,7 +14,7 @@ include Sidekiq::Util
 
 Sidekiq.configure_server do |config|
   #config.options[:concurrency] = 1
-  config.redis = { driver: :hiredis, db: 13, port: 6380 }
+  config.redis = { db: 13 }
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
@@ -36,14 +36,14 @@ end
 # brew tap shopify/shopify
 # brew install toxiproxy
 # gem install toxiproxy
-require 'toxiproxy'
+#require 'toxiproxy'
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
-Toxiproxy.populate([{
-    "name": "redis",
-    "listen": "127.0.0.1:6380",
-    "upstream": "127.0.0.1:6379"
-}])
+#Toxiproxy.populate([{
+    #"name": "redis",
+    #"listen": "127.0.0.1:6380",
+    #"upstream": "127.0.0.1:6379"
+#}])
 
 self_read, self_write = IO.pipe
 %w(INT TERM TSTP TTIN).each do |sig|
@@ -103,7 +103,7 @@ Sidekiq.logger.error "Created #{count*iter} jobs"
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 2
+      sleep 1
       qsize, retries = Sidekiq.redis do |conn|
         conn.pipelined do
           conn.llen "queue:default"
@@ -125,8 +125,8 @@ begin
   #RubyProf::exclude_threads = [ Monitoring ]
   #RubyProf.start
   fire_event(:startup)
-  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 
@@ -134,7 +134,7 @@ begin
       signal = readable_io.first[0].gets.strip
       handle_signal(launcher, signal)
     end
-  end
+  #end
 rescue SystemExit => e
   #Sidekiq.logger.error("Profiling...")
   #result = RubyProf.stop

data/lib/sidekiq/api.rb

@@ -75,7 +75,10 @@ module Sidekiq
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
-                                Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at'.freeze]
+                                job = Sidekiq.load_json(entry)
+                                now = Time.now.to_f
+                                thence = job['enqueued_at'.freeze] || now
+                                now - thence
                               else
                                 0
                               end
@@ -225,7 +228,10 @@ module Sidekiq
         conn.lrange(@rname, -1, -1)
       end.first
       return 0 unless entry
-      Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at']
+      job = Sidekiq.load_json(entry)
+      now = Time.now.to_f
+      thence = job['enqueued_at'] || now
+      now - thence
     end
 
     def each
@@ -355,7 +361,8 @@ module Sidekiq
     end
 
     def latency
-      Time.now.to_f - (self['enqueued_at'] || self['created_at'] || 0)
+      now = Time.now.to_f
+      now - (@item['enqueued_at'] || @item['created_at'] || now)
     end
 
     ##
@@ -592,13 +599,13 @@ module Sidekiq
   # Allows enumeration of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
+  # and deleting them from the schedule queue.
   #
   #   r = Sidekiq::ScheduledSet.new
-  #   r.select do |retri|
-  #     retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
-  #     retri.args[0] == 'User' &&
-  #     retri.args[1] == 'setup_new_subscriber'
+  #   r.select do |scheduled|
+  #     scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
+  #     scheduled.args[0] == 'User' &&
+  #     scheduled.args[1] == 'setup_new_subscriber'
   #   end.map(&:delete)
   class ScheduledSet < JobSet
     def initialize

data/lib/sidekiq/job_retry.rb

@@ -70,30 +70,36 @@ module Sidekiq
     # require the worker to be instantiated.
     def global(msg, queue)
       yield
-    rescue Skip
-      raise
-    rescue Sidekiq::Shutdown
+    rescue Skip => ex
+      raise ex
+    rescue Sidekiq::Shutdown => ey
       # ignore, will be pushed back onto queue during hard_shutdown
-      raise
+      raise ey
     rescue Exception => e
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
       raise e unless msg['retry']
       attempt_retry(nil, msg, queue, e)
+      raise e
     end
 
 
     # The local retry support means that any errors that occur within
     # this block can be associated with the given worker instance.
     # This is required to support the `sidekiq_retries_exhausted` block.
+    #
+    # Note that any exception from the block is wrapped in the Skip
+    # exception so the global block does not reprocess the error.  The
+    # Skip exception is unwrapped within Sidekiq::Processor#process before
+    # calling the handle_exception handlers.
     def local(worker, msg, queue)
       yield
-    rescue Skip
-      raise
-    rescue Sidekiq::Shutdown
+    rescue Skip => ex
+      raise ex
+    rescue Sidekiq::Shutdown => ey
       # ignore, will be pushed back onto queue during hard_shutdown
-      raise
+      raise ey
     rescue Exception => e
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
@@ -161,8 +167,6 @@ module Sidekiq
         # Goodbye dear message, you (re)tried your best I'm sure.
         retries_exhausted(worker, msg, exception)
       end
-
-      raise exception
     end
 
     def retries_exhausted(worker, msg, exception)
@@ -217,8 +221,7 @@ module Sidekiq
     end
 
     def exception_caused_by_shutdown?(e, checked_causes = [])
-      # In Ruby 2.1.0 only, check if exception is a result of shutdown.
-      return false unless defined?(e.cause)
+      return false unless e.cause
 
       # Handle circular causes
       checked_causes << e.object_id

data/lib/sidekiq/processor.rb

@@ -179,8 +179,9 @@ module Sidekiq
         # we didn't properly finish it.
         ack = false
       rescue Exception => ex
-        handle_exception(ex, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
-        raise
+        e = ex.is_a?(::Sidekiq::JobRetry::Skip) && ex.cause ? ex.cause : ex
+        handle_exception(e, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
+        raise e
       ensure
         work.acknowledge if ack
       end

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "5.0.0.beta1"
+  VERSION = "5.0.0.beta2"
 end

data/lib/sidekiq/web/application.rb

@@ -281,7 +281,7 @@ module Sidekiq
         when :json
           { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }
         when String
-          { "Content-Type" => action.type, "Cache-Control" => "no-cache" }
+          { "Content-Type" => (action.type || "text/html"), "Cache-Control" => "no-cache" }
         else
           { "Content-Type" => "text/html", "Cache-Control" => "no-cache" }
         end

data/lib/sidekiq/web/helpers.rb

@@ -2,6 +2,7 @@
 require 'uri'
 require 'set'
 require 'yaml'
+require 'cgi'
 
 module Sidekiq
   # This is not a public API
@@ -161,7 +162,7 @@ module Sidekiq
     def qparams(options)
       options = options.stringify_keys
       params.merge(options).map do |key, value|
-        SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
+        SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       end.compact.join("&")
     end
 

data/sidekiq.gemspec

@@ -10,8 +10,8 @@ Gem::Specification.new do |gem|
   gem.license       = "LGPL-3.0"
 
   gem.executables   = ['sidekiq', 'sidekiqctl']
-  gem.files         = `git ls-files | grep -Ev '^(myapp|examples)'`.split("\n")
-  gem.test_files    = `git ls-files -- test/*`.split("\n")
+  gem.files         = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
+  gem.test_files    = []
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION

data/web/views/retry.erb

@@ -7,7 +7,7 @@
       <tr>
         <th><%= t('ErrorClass') %></th>
         <td>
-          <code><%= h @retry.display_class %></code>
+          <code><%= h @retry['error_class'] %></code>
         </td>
       </tr>
       <tr>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 5.0.0.beta1
+  version: 5.0.0.beta2
 platform: ruby
 authors:
 - Mike Perham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-31 00:00:00.000000000 Z
+date: 2017-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -286,38 +286,6 @@ files:
 - lib/sidekiq/web/router.rb
 - lib/sidekiq/worker.rb
 - sidekiq.gemspec
-- test/config.yml
-- test/env_based_config.yml
-- test/fake_env.rb
-- test/fixtures/en.yml
-- test/helper.rb
-- test/test_actors.rb
-- test/test_api.rb
-- test/test_cli.rb
-- test/test_client.rb
-- test/test_exception_handler.rb
-- test/test_extensions.rb
-- test/test_fetch.rb
-- test/test_launcher.rb
-- test/test_logging.rb
-- test/test_manager.rb
-- test/test_middleware.rb
-- test/test_processor.rb
-- test/test_rails.rb
-- test/test_redis_connection.rb
-- test/test_retry.rb
-- test/test_retry_exhausted.rb
-- test/test_scheduled.rb
-- test/test_scheduling.rb
-- test/test_sidekiq.rb
-- test/test_testing.rb
-- test/test_testing_fake.rb
-- test/test_testing_inline.rb
-- test/test_util.rb
-- test/test_web.rb
-- test/test_web_auth.rb
-- test/test_web_helpers.rb
-- test/test_web_sessions.rb
 - web/assets/images/favicon.ico
 - web/assets/images/logo.png
 - web/assets/images/status.png
@@ -389,36 +357,4 @@ rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Simple, efficient background processing for Ruby
-test_files:
-- test/config.yml
-- test/env_based_config.yml
-- test/fake_env.rb
-- test/fixtures/en.yml
-- test/helper.rb
-- test/test_actors.rb
-- test/test_api.rb
-- test/test_cli.rb
-- test/test_client.rb
-- test/test_exception_handler.rb
-- test/test_extensions.rb
-- test/test_fetch.rb
-- test/test_launcher.rb
-- test/test_logging.rb
-- test/test_manager.rb
-- test/test_middleware.rb
-- test/test_processor.rb
-- test/test_rails.rb
-- test/test_redis_connection.rb
-- test/test_retry.rb
-- test/test_retry_exhausted.rb
-- test/test_scheduled.rb
-- test/test_scheduling.rb
-- test/test_sidekiq.rb
-- test/test_testing.rb
-- test/test_testing_fake.rb
-- test/test_testing_inline.rb
-- test/test_util.rb
-- test/test_web.rb
-- test/test_web_auth.rb
-- test/test_web_helpers.rb
-- test/test_web_sessions.rb
+test_files: []

data/test/config.yml

@@ -1,9 +0,0 @@
----
-:verbose:      false
-:require:      ./test/fake_env.rb
-:pidfile:      /tmp/sidekiq-config-test.pid
-:logfile:      /tmp/sidekiq.log
-:concurrency:  50
-:queues:
-  - [<%="very_"%>often, 2]
-  - [seldom, 1]

data/test/env_based_config.yml

@@ -1,11 +0,0 @@
----
-:pidfile:      /tmp/sidekiq-config-test.pid
-:concurrency:  50
-staging:
-  :verbose:      false
-  :require:      ./test/fake_env.rb
-  :logfile:      /tmp/sidekiq.log
-  :concurrency:  5
-  :queues:
-    - [<%="very_"%>often, 2]
-    - [seldom, 1]

data/test/fake_env.rb

@@ -1 +0,0 @@
-# frozen_string_literal: true

data/test/fixtures/en.yml

@@ -1,2 +0,0 @@
-en:
-  translated_text: 'Changed text from add locals'

data/test/helper.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-$TESTING = true
-# disable minitest/parallel threads
-ENV["N"] = "0"
-
-require 'capybara'
-require 'capybara/dsl'
-require 'capybara/poltergeist'
-
-Capybara.register_driver :poltergeist do |app|
-  Capybara::Poltergeist::Driver.new(app,
-    debug: false, js_errors: false, timeout: 180
-  )
-end
-
-def percy_enabled?
-  !(ENV['PERCY_ENABLE'] == '0')
-end
-require 'percy/capybara' if percy_enabled?
-
-if ENV["COVERAGE"]
-  require 'simplecov'
-  SimpleCov.start do
-    add_filter "/test/"
-    add_filter "/myapp/"
-  end
-end
-ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
-
-trap 'TSTP' do
-  threads = Thread.list
-
-  puts
-  puts "=" * 80
-  puts "Received TSTP signal; printing all #{threads.count} thread backtraces."
-
-  threads.each do |thr|
-    description = thr == Thread.main ? "Main thread" : thr.inspect
-    puts
-    puts "#{description} backtrace: "
-    puts thr.backtrace.join("\n")
-  end
-
-  puts "=" * 80
-end
-
-begin
-  require 'pry-byebug'
-rescue LoadError
-end
-
-require 'minitest/autorun'
-
-require 'sidekiq'
-require 'sidekiq/util'
-Sidekiq.logger.level = Logger::ERROR
-
-Sidekiq::Test = Minitest::Test
-
-require 'sidekiq/redis_connection'
-REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost/15'
-REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'testy')
-
-Sidekiq.configure_client do |config|
-  config.redis = { :url => REDIS_URL, :namespace => 'testy' }
-end
-
-def capture_logging(lvl=Logger::INFO)
-  old = Sidekiq.logger
-  begin
-    out = StringIO.new
-    logger = Logger.new(out)
-    logger.level = lvl
-    Sidekiq.logger = logger
-    yield
-    out.string
-  ensure
-    Sidekiq.logger = old
-  end
-end
-
-def with_logging(lvl=Logger::DEBUG)
-  old = Sidekiq.logger.level
-  begin
-    Sidekiq.logger.level = lvl
-    yield
-  ensure
-    Sidekiq.logger.level = old
-  end
-end
-
-if percy_enabled?
-  # Initialize and finalize Percy.io
-  Percy::Capybara.initialize_build
-  MiniTest.after_run {
-    Percy::Capybara.finalize_build
-  }
-end