sidekiq 4.2.9 → 4.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1bfbae2d6e06595febe27592cf45c3a5f188f7e9
-  data.tar.gz: cae512a4610cd6670d51559e71a3156a00d43eab
+  metadata.gz: 595500a3810c25e8e04e86c644f966ff01cb035f
+  data.tar.gz: 3f92a9f61e61848bc2c0639469ce3588f3848fc0
 SHA512:
-  metadata.gz: 42b2738772c88ba2d22dee47dc8e8bb5755d8d8f769a033be6e0e503f6e15ddc02092db6b745365db93448d110231721838c9bec8c0ced7537c451dfecc940d0
-  data.tar.gz: 8e6861e5b2cec345eeeb10fd2f031431dae82db2af3fa60e58a30c78d874f39a92a7638e0d291c5885a78eaa36dd50010dfb973225ee0dd6cd345059c17598e9
+  metadata.gz: c4e97c3665d1e902f47cfb579fda437dc834ac4b584606f1ca3b8fd9225e8502cacf5fb687673b4735b359de6af179589759a34369f1c74e904b37f8fe70a4ee
+  data.tar.gz: a4e6d41067bf6b88c77577def0c1f9ecc3f5d46f0bdfb0821952c26867e91a71c64231aab09a25096d22c3272dea2dcc37547bc6096dd7622a192fa00e0a4e50

data/.github/issue_template.md

@@ -1,4 +1,9 @@
 Ruby version:
 Sidekiq / Pro / Enterprise version(s):
 
-If relevant, please include your initializer and any error message with the full backtrace.
+Please include your initializer and any error message with the full backtrace.
+
+Are you using an old version?
+Have you checked the changelog to see if your issue has been fixed in a later version?
+
+https://github.com/mperham/sidekiq/blob/master/Changes.md

data/.travis.yml

@@ -6,8 +6,13 @@ services:
 before_install:
   - gem install bundler
   - gem update bundler
-rvm:
-  - 2.2.4
-  - 2.3.0
-  - 2.4.0
-  - jruby-9.1.6.0
+matrix:
+  include:
+    - rvm: 2.2.4
+      env: "PERCY_ENABLE=0"
+    - rvm: 2.3.0
+      env: "PERCY_ENABLE=0"
+    - rvm: 2.4.0
+      env: "PERCY_ENABLE=1"
+    - rvm: jruby-9.1.6.0
+      env: "PERCY_ENABLE=0"

data/Changes.md

@@ -1,5 +1,14 @@
 # Sidekiq Changes
 
+HEAD
+-----------
+
+- Scheduled jobs can now be moved directly to the Dead queue via API [#3390]
+- Fix edge case leading to job duplication when using Sidekiq Pro's
+  reliability feature [#3388]
+- Fix error class name display on retry page [#3348]
+- More robust latency calculation [#3340]
+
 4.2.9
 -----------
 

data/Ent-Changes.md

@@ -3,6 +3,15 @@ Sidekiq Enterprise Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+1.5.2
+-------------
+
+- Fix encrypted arguments double-encrypted by retry or rate limiting [#3368]
+- Fix leak in concurrent rate limiter, run this in Rails console to clean up existing data [#3323]
+```ruby
+expiry = 1.month.to_i; Sidekiq::Limiter.redis { |c| c.scan_each(match: "lmtr-cfree-*") { |key| c.expire(key, expiry) } }
+```
+
 1.5.1
 -------------
 

data/Pro-Changes.md

@@ -3,6 +3,35 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+HEAD
+---------
+
+- Make Batch UI progress bar more friendly to the colorblind [#3387]
+
+3.4.5
+---------
+
+- Fix potential job loss with reliable scheduler when lots of jobs are scheduled
+  at precisely the same time. Thanks to raivil for his hard work in
+  reproducing the bug. [#3371]
+
+3.4.4
+---------
+
+- Optimize super\_fetch shutdown to restart jobs quicker [#3249]
+
+3.4.3
+---------
+
+- Limit reliable scheduler to enqueue up to 100 jobs per call, minimizing Redis latency [#3332]
+- Fix bug in super\_fetch logic for queues with `_` in the name [#3339]
+
+3.4.2
+---------
+
+- Add `Batch::Status#invalidated?` API which returns true if any/all
+  JIDs were invalidated within the batch. [#3326]
+
 3.4.1
 ---------
 

data/bin/sidekiqload

@@ -14,7 +14,7 @@ include Sidekiq::Util
 
 Sidekiq.configure_server do |config|
   #config.options[:concurrency] = 1
-  config.redis = { driver: :hiredis, db: 13, port: 6380 }
+  config.redis = { db: 13 }
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
@@ -36,14 +36,14 @@ end
 # brew tap shopify/shopify
 # brew install toxiproxy
 # gem install toxiproxy
-require 'toxiproxy'
+#require 'toxiproxy'
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
-Toxiproxy.populate([{
-    "name": "redis",
-    "listen": "127.0.0.1:6380",
-    "upstream": "127.0.0.1:6379"
-}])
+#Toxiproxy.populate([{
+    #"name": "redis",
+    #"listen": "127.0.0.1:6380",
+    #"upstream": "127.0.0.1:6379"
+#}])
 
 self_read, self_write = IO.pipe
 %w(INT TERM USR1 USR2 TTIN).each do |sig|
@@ -108,7 +108,7 @@ Sidekiq.logger.error "Created #{count*iter} jobs"
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 2
+      sleep 1
       qsize, retries = Sidekiq.redis do |conn|
         conn.pipelined do
           conn.llen "queue:default"
@@ -130,8 +130,8 @@ begin
   #RubyProf::exclude_threads = [ Monitoring ]
   #RubyProf.start
   fire_event(:startup)
-  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 
@@ -139,7 +139,7 @@ begin
       signal = readable_io.first[0].gets.strip
       handle_signal(launcher, signal)
     end
-  end
+  #end
 rescue SystemExit => e
   #Sidekiq.logger.error("Profiling...")
   #result = RubyProf.stop

data/lib/sidekiq.rb

@@ -149,8 +149,8 @@ module Sidekiq
     require 'sidekiq/middleware/server/logging'
 
     Middleware::Chain.new do |m|
-      m.add Middleware::Server::Logging
       m.add Middleware::Server::RetryJobs
+      m.add Middleware::Server::Logging
     end
   end
 

data/lib/sidekiq/api.rb

@@ -75,7 +75,10 @@ module Sidekiq
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
-                                Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at'.freeze]
+                                job = Sidekiq.load_json(entry)
+                                now = Time.now.to_f
+                                thence = job['enqueued_at'.freeze] || now
+                                now - thence
                               else
                                 0
                               end
@@ -225,7 +228,10 @@ module Sidekiq
         conn.lrange(@rname, -1, -1)
       end.first
       return 0 unless entry
-      Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at']
+      job = Sidekiq.load_json(entry)
+      now = Time.now.to_f
+      thence = job['enqueued_at'] || now
+      now - thence
     end
 
     def each
@@ -351,7 +357,8 @@ module Sidekiq
     end
 
     def latency
-      Time.now.to_f - (@item['enqueued_at'] || @item['created_at'])
+      now = Time.now.to_f
+      now - (@item['enqueued_at'] || @item['created_at'] || now)
     end
 
     ##
@@ -416,10 +423,9 @@ module Sidekiq
     end
 
     def retry
-      raise "Retry not available on jobs which have not failed" unless item["failed_at"]
       remove_job do |message|
         msg = Sidekiq.load_json(message)
-        msg['retry_count'] -= 1
+        msg['retry_count'] -= 1 if msg['retry_count']
         Sidekiq::Client.push(msg)
       end
     end
@@ -427,9 +433,7 @@ module Sidekiq
     ##
     # Place job in the dead set
     def kill
-      raise 'Kill not available on jobs which have not failed' unless item['failed_at']
       remove_job do |message|
-        Sidekiq.logger.info { "Killing job #{message['jid']}" }
         now = Time.now.to_f
         Sidekiq.redis do |conn|
           conn.multi do
@@ -441,6 +445,10 @@ module Sidekiq
       end
     end
 
+    def error?
+      !!item['error_class']
+    end
+
     private
 
     def remove_job
@@ -585,13 +593,13 @@ module Sidekiq
   # Allows enumeration of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
+  # and deleting them from the schedule queue.
   #
   #   r = Sidekiq::ScheduledSet.new
-  #   r.select do |retri|
-  #     retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
-  #     retri.args[0] == 'User' &&
-  #     retri.args[1] == 'setup_new_subscriber'
+  #   r.select do |scheduled|
+  #     scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
+  #     scheduled.args[0] == 'User' &&
+  #     scheduled.args[1] == 'setup_new_subscriber'
   #   end.map(&:delete)
   class ScheduledSet < JobSet
     def initialize

data/lib/sidekiq/middleware/server/logging.rb

@@ -4,28 +4,19 @@ module Sidekiq
       class Logging
 
         def call(worker, item, queue)
-          Sidekiq::Logging.with_context(log_context(worker, item)) do
-            begin
-              start = Time.now
-              logger.info("start".freeze)
-              yield
-              logger.info("done: #{elapsed(start)} sec")
-            rescue Exception
-              logger.info("fail: #{elapsed(start)} sec")
-              raise
-            end
+          begin
+            start = Time.now
+            logger.info("start".freeze)
+            yield
+            logger.info("done: #{elapsed(start)} sec")
+          rescue Exception
+            logger.info("fail: #{elapsed(start)} sec")
+            raise
           end
         end
 
         private
 
-        # If we're using a wrapper class, like ActiveJob, use the "wrapped"
-        # attribute to expose the underlying thing.
-        def log_context(worker, item)
-          klass = item['wrapped'.freeze] || worker.class.to_s
-          "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
-        end
-
         def elapsed(start)
           (Time.now - start).round(3)
         end

data/lib/sidekiq/processor.rb

@@ -129,13 +129,15 @@ module Sidekiq
           worker.jid = job_hash['jid'.freeze]
 
           stats(worker, job_hash, queue) do
-            Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
-              @executor.call do
-                # Only ack if we either attempted to start this job or
-                # successfully completed it. This prevents us from
-                # losing jobs if a middleware raises an exception before yielding
-                ack = true
-                execute_job(worker, cloned(job_hash['args'.freeze]))
+            Sidekiq::Logging.with_context(log_context(job_hash)) do
+              ack = true
+              Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
+                @executor.call do
+                  # Only ack if we either attempted to start this job or
+                  # successfully completed it. This prevents us from
+                  # losing jobs if a middleware raises an exception before yielding
+                  execute_job(worker, cloned(job_hash['args'.freeze]))
+                end
               end
             end
           end
@@ -154,6 +156,13 @@ module Sidekiq
       end
     end
 
+    # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+    # attribute to expose the underlying thing.
+    def log_context(item)
+      klass = item['wrapped'.freeze] || item['class'.freeze]
+      "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+    end
+
     def execute_job(worker, cloned_args)
       worker.perform(*cloned_args)
     end

data/lib/sidekiq/testing.rb

@@ -316,3 +316,9 @@ module Sidekiq
     end
   end
 end
+
+if defined?(::Rails) && !Rails.env.test?
+  puts("**************************************************")
+  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
+  puts("**************************************************")
+end

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "4.2.9"
+  VERSION = "4.2.10"
 end

data/lib/sidekiq/web/application.rb

@@ -85,7 +85,7 @@ module Sidekiq
       name = route_params[:name]
       Sidekiq::Job.new(params['key_val'], name).delete
 
-      redirect_with_query("#{root_path}queues/#{name}")
+      redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
     end
 
     get '/morgue' do
@@ -281,7 +281,7 @@ module Sidekiq
         when :json
           { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }
         when String
-          { "Content-Type" => action.type, "Cache-Control" => "no-cache" }
+          { "Content-Type" => (action.type || "text/html"), "Cache-Control" => "no-cache" }
         else
           { "Content-Type" => "text/html", "Cache-Control" => "no-cache" }
         end

data/lib/sidekiq/web/helpers.rb

@@ -2,6 +2,7 @@
 require 'uri'
 require 'set'
 require 'yaml'
+require 'cgi'
 
 module Sidekiq
   # This is not a public API
@@ -161,7 +162,7 @@ module Sidekiq
     def qparams(options)
       options = options.stringify_keys
       params.merge(options).map do |key, value|
-        SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
+        SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       end.compact.join("&")
     end
 

data/sidekiq.gemspec

@@ -10,8 +10,8 @@ Gem::Specification.new do |gem|
   gem.license       = "LGPL-3.0"
 
   gem.executables   = ['sidekiq', 'sidekiqctl']
-  gem.files         = `git ls-files | grep -Ev '^(myapp|examples)'`.split("\n")
-  gem.test_files    = `git ls-files -- test/*`.split("\n")
+  gem.files         = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
+  gem.test_files    = []
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
@@ -23,4 +23,10 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency      'minitest', '~> 5.10', '>= 5.10.1'
   gem.add_development_dependency      'rake', '~> 10.0'
   gem.add_development_dependency      'rails', '>= 3.2.0'
+
+  gem.add_development_dependency      'capybara', '~> 2.11'
+  gem.add_development_dependency      'poltergeist', '~> 1.12'
+  gem.add_development_dependency      'percy-capybara', '~> 2.3'
+  gem.add_development_dependency      'timecop', '~> 0.8'
+  gem.add_development_dependency      'mocha', '~> 1.1'
 end

data/web/assets/javascripts/dashboard.js

@@ -20,7 +20,7 @@ factory.defined&&factory.defined(function(d){return d.y!==null});return factory}
 var poller;
 
 var realtimeGraph = function(updatePath) {
-  var timeInterval = parseInt(localStorage.timeInterval || '2000');
+  var timeInterval = parseInt(localStorage.timeInterval || '5000');
 
   var graphElement = document.getElementById("realtime");
 

data/web/locales/fa.yml

@@ -0,0 +1,79 @@
+# elements like %{queue} are variables and should not be translated
+fa: # <---- change this to your locale code
+  Dashboard: داشبورد
+  Status: اعلان
+  Time: رمان
+  Namespace: فضای نام
+  Realtime: زنده
+  History: تاریخچه
+  Busy: مشغول
+  Processed: پردازش شده
+  Failed: ناموفق
+  Scheduled: زمان بندی
+  Retries: تکرار
+  Enqueued: صف بندی نشدند
+  Worker: کارگزار
+  LivePoll: Live Poll
+  StopPolling: Stop Polling
+  Queue: صف
+  Class: کلاس
+  Job: کار
+  Arguments: آرگومنت
+  Extras: اضافی
+  Started: شروع شده
+  ShowAll: نمایش همه
+  CurrentMessagesInQueue: کار فعلی در <span class='title'>%{queue}</span>
+  Delete: حذف
+  AddToQueue: افزودن به صف
+  AreYouSureDeleteJob: آیا شما مطمعن هستید از حذف این کار ؟
+  AreYouSureDeleteQueue: ایا شما مطمعنید از حذف %{queue} ?
+  Queues: صف ها
+  Size: سایز
+  Actions: اعمال
+  NextRetry: بار دیگر تلاش کنید
+  RetryCount: تعداد تلاش ها
+  RetryNow: تلاش مجدد
+  Kill: کشتن
+  LastRetry: آخرین تلاش
+  OriginallyFailed: Originally Failed
+  AreYouSure: آیا مطمعن هستید?
+  DeleteAll: حذف همه
+  RetryAll: تلاش مجدد برای همه
+  NoRetriesFound: هیچ تلاش پیدا نشد
+  Error: خطا
+  ErrorClass: خطا کلاس
+  ErrorMessage: پیغام خطا
+  ErrorBacktrace: خطای معکوس
+  GoBack: ← برگشت
+  NoScheduledFound: هیچ کار برنامه ریزی شده ای یافت نشد
+  When: وقتی که
+  ScheduledJobs: کار برنامه ریزی شده
+  idle: بیهودی
+  active: فعال
+  Version: ورژن
+  Connections: ارتباطات
+  MemoryUsage: حافظه استفاده شده
+  PeakMemoryUsage: اوج حافظه استفاده شده
+  Uptime: آپ تایم (روز)
+  OneWeek: ۱ هفته
+  OneMonth: ۱ ماه
+  ThreeMonths: ۳ ماه
+  SixMonths: ۶ ماه
+  Failures: شکست ها
+  DeadJobs: کار مرده
+  NoDeadJobsFound: کار مرده ای یافت نشد
+  Dead: مرده
+  Processes: پردازش ها
+  Thread: رشته
+  Threads: رشته ها
+  Jobs: کار ها
+  Paused: مکث
+  Stop: توقف
+  Quiet: خروج
+  StopAll: توقف همه
+  QuietAll: خروج همه
+  PollingInterval: Polling interval
+  Plugins: پلاگین ها
+  NotYetEnqueued: بدون صف بندی
+  CreatedAt: ساخته شده در
+  BackToApp: برگشت به برنامه