sidekiq 4.1.4 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4e031cb26d8dcc6ab4f404563cd0a711577e586
-  data.tar.gz: 0a05484ed39032f71dcd2db901afec38f4c9d023
+  metadata.gz: db0f25c2fe7ea87588364c3eb86e3b929197139c
+  data.tar.gz: 8fa8c056e6c5fad50ce5d49838515aca61573458
 SHA512:
-  metadata.gz: c807c5e398088ee6b344cdce5fa57a77d51c7206a3cc11c4aeb8bc9af186eb82c292c215facb6ae54589ed5b1d95573b74a07622bc452b7ba970d0fb49c207df
-  data.tar.gz: a56f9815bfc2954b12a9598b6678aee7771ee949d7e886dfc6ddaacbe05f5294e008deccf063c9c39ac430f18322cbdff4468a9ebd3530a9ef8ce780b064bf7e
+  metadata.gz: 4a801ac5f3d70ca6e1856cb7369f557d5bb8d1c579e775f6a4f5ed97d534af2d289f884979de978f3a7cab2d1261fabeef6b7fc9e73b1684c7b1daabbd25e0e7
+  data.tar.gz: 37c617e0ea241fe8cea7c85e6479e2259e06f1b5bfb06ce811b2529d73adc8a09a67655dbdcaff028ba4cd3226169a79f636f813f7848d7fe8eaa084ea9d5739

data/.travis.yml

@@ -7,12 +7,6 @@ before_install:
   - gem install bundler
   - gem update bundler
 rvm:
-  - 2.0.0
-  - 2.1.8
   - 2.2.4
   - 2.3.0
   - jruby-head
-  - rbx-2
-matrix:
-  allow_failures:
-    - rvm: rbx-2

data/Changes.md

@@ -1,5 +1,29 @@
 # Sidekiq Changes
 
+4.2.0
+-----------
+
+- Enable development-mode code reloading.  **With Rails 5.0+, you don't need
+  to restart Sidekiq to pick up your Sidekiq::Worker changes anymore!** [#2457]
+- **Remove Sinatra dependency**.  Sidekiq's Web UI now uses Rack directly.
+  Thank you to Sidekiq's newest committer, **badosu**, for writing the code
+  and doing a lot of testing to ensure compatibility with many different
+  3rd party plugins.  If your Web UI works with 4.1.4 but fails with
+  4.2.0, please open an issue. [#3075]
+- Allow tuning of concurrency with the `RAILS_MAX_THREADS` env var. [#2985]
+  This is the same var used by Puma so you can tune all of your systems
+  the same way:
+```sh
+web: RAILS_MAX_THREADS=5 bundle exec puma ...
+worker: RAILS_MAX_THREADS=10 bundle exec sidekiq ...
+```
+Using `-c` or `config/sidekiq.yml` overrides this setting.  I recommend
+adjusting your `config/database.yml` to use it too so connections are
+auto-scaled:
+```yaml
+  pool: <%= ENV['RAILS_MAX_THREADS'] || 5 %>
+```
+
 4.1.4
 -----------
 

data/Ent-Changes.md

@@ -3,7 +3,30 @@ Sidekiq Enterprise Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
-HEAD
+1.3.2
+-------------
+
+- Upgrade encryption to use OpenSSL's more secure GCM mode. [#3060]
+
+1.3.1
+-------------
+
+- Fix multi-process memory monitoring on CentOS 6.x [#3063]
+- Polish the new encryption feature a bit.
+
+1.3.0
+-------------
+
+- **BETA** [New encryption feature](https://github.com/mperham/sidekiq/wiki/Ent-Encryption)
+  which automatically encrypts the last argument of a Worker, aka the secret bag.
+
+1.2.4
+-------------
+
+- Fix issue causing some minutely jobs to execute every other minute.
+- Log a warning if slow periodic processing causes us to miss a clock tick.
+
+1.2.3
 -------------
 
 - Periodic jobs could stop executing until process restart if Redis goes down [#3047]

data/Gemfile

@@ -1,8 +1,8 @@
 source 'https://rubygems.org'
 gemspec
 
+gem 'rails', '5.0.0'
 gem "hiredis"
-gem 'rails', '~> 4.2'
 gem 'simplecov'
 gem 'minitest'
 gem 'minitest-utils'
@@ -23,7 +23,7 @@ platforms :mri do
   gem 'ruby-prof'
 end
 
-platforms :jruby do
-  gem 'jruby-openssl'
-  gem 'activerecord-jdbcsqlite3-adapter'
-end
+#platforms :jruby do
+  #gem 'jruby-openssl'
+  #gem 'activerecord-jdbcsqlite3-adapter'
+#end

data/Pro-Changes.md

@@ -3,6 +3,17 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+3.3.3
+---------
+
+- Update Web UI extension to work with Sidekiq 4.2.0's new Web UI. [#3075]
+
+3.3.2
+---------
+
+- Minimize batch memory usage after success [#3083]
+- Extract batch's 24 hr linger expiry to a LINGER constant so it can be tuned. [#3011]
+
 3.3.1
 ---------
 

data/lib/sidekiq.rb

@@ -28,9 +28,11 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
+      heartbeat: [],
     },
     dead_max_jobs: 10_000,
-    dead_timeout_in_seconds: 180 * 24 * 60 * 60 # 6 months
+    dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
+    reloader: proc { |&block| block.call },
   }
 
   DEFAULT_WORKER_OPTIONS = {
@@ -169,6 +171,7 @@ module Sidekiq
   def self.default_retries_exhausted=(prok)
     @default_retries_exhausted = prok
   end
+  @default_retries_exhausted = ->(job, ex) { }
   def self.default_retries_exhausted
     @default_retries_exhausted
   end

data/lib/sidekiq/cli.rb

@@ -208,6 +208,8 @@ module Sidekiq
       opts = parse_config(cfile).merge(opts) if cfile
 
       opts[:strict] = true if opts[:strict].nil?
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
+      opts[:identity] = identity
 
       options.merge!(opts)
     end
@@ -227,7 +229,7 @@ module Sidekiq
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
           ::Rails.application.eager_load!
-        else
+        elsif ::Rails::VERSION::MAJOR == 4
           # Painful contortions, see 1791 for discussion
           require File.expand_path("#{options[:require]}/config/application.rb")
           ::Rails::Application.initializer "sidekiq.eager_load" do
@@ -235,10 +237,17 @@ module Sidekiq
           end
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
+        else
+          require 'sidekiq/rails'
+          require File.expand_path("#{options[:require]}/config/environment.rb")
+          Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
         end
         options[:tag] ||= default_tag
       else
-        require options[:require]
+        not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
+            "./#{options[:require]} or /path/to/#{options[:require]}"
+
+        require(options[:require]) || raise(ArgumentError, not_required_message)
       end
     end
 

data/lib/sidekiq/launcher.rb

@@ -94,15 +94,19 @@ module Sidekiq
         end
         fails = procd = 0
 
-        _, _, _, msg = Sidekiq.redis do |conn|
+        _, exists, _, _, msg = Sidekiq.redis do |conn|
           conn.multi do
             conn.sadd('processes', key)
+            conn.exists(key)
             conn.hmset(key, 'info', json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           end
         end
 
+        # first heartbeat or recovering from an outage and need to reestablish our heartbeat
+        fire_event(:heartbeat) if !exists
+
         return unless msg
 
         if JVM_RESERVED_SIGNALS.include?(msg)

data/lib/sidekiq/manager.rb

@@ -4,6 +4,7 @@ require 'sidekiq/util'
 require 'sidekiq/processor'
 require 'sidekiq/fetch'
 require 'thread'
+require 'set'
 
 module Sidekiq
 

data/lib/sidekiq/processor.rb

@@ -36,6 +36,7 @@ module Sidekiq
       @job = nil
       @thread = nil
       @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
+      @reloader = Sidekiq.options[:reloader]
     end
 
     def terminate(wait=false)
@@ -118,33 +119,35 @@ module Sidekiq
       jobstr = work.job
       queue = work.queue_name
 
-      ack = false
-      begin
-        job = Sidekiq.load_json(jobstr)
-        klass  = job['class'.freeze].constantize
-        worker = klass.new
-        worker.jid = job['jid'.freeze]
-
-        stats(worker, job, queue) do
-          Sidekiq.server_middleware.invoke(worker, job, queue) do
-            # Only ack if we either attempted to start this job or
-            # successfully completed it. This prevents us from
-            # losing jobs if a middleware raises an exception before yielding
-            ack = true
-            execute_job(worker, cloned(job['args'.freeze]))
+      @reloader.call do
+        ack = false
+        begin
+          job = Sidekiq.load_json(jobstr)
+          klass  = job['class'.freeze].constantize
+          worker = klass.new
+          worker.jid = job['jid'.freeze]
+
+          stats(worker, job, queue) do
+            Sidekiq.server_middleware.invoke(worker, job, queue) do
+              # Only ack if we either attempted to start this job or
+              # successfully completed it. This prevents us from
+              # losing jobs if a middleware raises an exception before yielding
+              ack = true
+              execute_job(worker, cloned(job['args'.freeze]))
+            end
           end
+          ack = true
+        rescue Sidekiq::Shutdown
+          # Had to force kill this job because it didn't finish
+          # within the timeout.  Don't acknowledge the work since
+          # we didn't properly finish it.
+          ack = false
+        rescue Exception => ex
+          handle_exception(ex, job || { :job => jobstr })
+          raise
+        ensure
+          work.acknowledge if ack
         end
-        ack = true
-      rescue Sidekiq::Shutdown
-        # Had to force kill this job because it didn't finish
-        # within the timeout.  Don't acknowledge the work since
-        # we didn't properly finish it.
-        ack = false
-      rescue Exception => ex
-        handle_exception(ex, job || { :job => jobstr })
-        raise
-      ensure
-        work.acknowledge if ack
       end
     end
 

data/lib/sidekiq/rails.rb

@@ -35,5 +35,22 @@ module Sidekiq
     initializer 'sidekiq' do
       Sidekiq.hook_rails!
     end
+
+    class Reloader
+      def initialize(app = ::Rails.application)
+        Sidekiq.logger.debug "Enabling Rails 5+ live code reloading, so hot!" unless app.config.cache_classes
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
   end if defined?(::Rails)
 end

data/lib/sidekiq/scheduled.rb

@@ -46,6 +46,7 @@ module Sidekiq
         @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
+        @thread = nil
       end
 
       # Shut down this instance, will pause until the thread is dead.

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "4.1.4"
+  VERSION = "4.2.0"
 end

data/lib/sidekiq/web.rb

@@ -1,26 +1,28 @@
 # frozen_string_literal: true
 require 'erb'
-require 'yaml'
-require 'sinatra/base'
 
 require 'sidekiq'
 require 'sidekiq/api'
 require 'sidekiq/paginator'
-require 'sidekiq/web_helpers'
+require 'sidekiq/web/helpers'
 
-module Sidekiq
-  class Web < Sinatra::Base
-    include Sidekiq::Paginator
+require 'sidekiq/web/router'
+require 'sidekiq/web/action'
+require 'sidekiq/web/application'
 
-    enable :sessions
-    use ::Rack::Protection, :use => :authenticity_token unless ENV['RACK_ENV'] == 'test'
+require 'rack/protection'
 
-    set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
-    set :public_folder, proc { "#{root}/assets" }
-    set :views, proc { "#{root}/views" }
-    set :locales, ["#{root}/locales"]
+require 'rack/builder'
+require 'rack/file'
+require 'rack/session/cookie'
 
-    helpers WebHelpers
+module Sidekiq
+  class Web
+    ROOT = File.expand_path("#{File.dirname(__FILE__)}/../../web")
+    VIEWS = "#{ROOT}/views".freeze
+    LOCALES = ["#{ROOT}/locales".freeze]
+    LAYOUT = "#{VIEWS}/layout.erb".freeze
+    ASSETS = "#{ROOT}/assets".freeze
 
     DEFAULT_TABS = {
       "Dashboard" => '',
@@ -32,6 +34,18 @@ module Sidekiq
     }
 
     class << self
+      def settings
+        self
+      end
+
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*middleware_args, &block)
+        middlewares << [middleware_args, block]
+      end
+
       def default_tabs
         DEFAULT_TABS
       end
@@ -41,234 +55,97 @@ module Sidekiq
       end
       alias_method :tabs, :custom_tabs
 
-      attr_accessor :app_url
-    end
-
-    get "/busy" do
-      erb :busy
-    end
-
-    post "/busy" do
-      if params['identity']
-        p = Sidekiq::Process.new('identity' => params['identity'])
-        p.quiet! if params[:quiet]
-        p.stop! if params[:stop]
-      else
-        processes.each do |pro|
-          pro.quiet! if params[:quiet]
-          pro.stop! if params[:stop]
-        end
+      def locales
+        @locales ||= LOCALES
       end
-      redirect "#{root_path}busy"
-    end
 
-    get "/queues" do
-      @queues = Sidekiq::Queue.all
-      erb :queues
-    end
-
-    get "/queues/:name" do
-      halt 404 unless params[:name]
-      @count = (params[:count] || 25).to_i
-      @name = params[:name]
-      @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
-      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
-      erb :queue
-    end
-
-    post "/queues/:name" do
-      Sidekiq::Queue.new(params[:name]).clear
-      redirect "#{root_path}queues"
-    end
-
-    post "/queues/:name/delete" do
-      Sidekiq::Job.new(params[:key_val], params[:name]).delete
-      redirect_with_query("#{root_path}queues/#{params[:name]}")
-    end
-
-    get '/morgue' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @dead) = page("dead", params[:page], @count, reverse: true)
-      @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
-      erb :morgue
-    end
-
-    get "/morgue/:key" do
-      halt 404 unless params['key']
-      @dead = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}morgue" if @dead.nil?
-      erb :dead
-    end
-
-    post '/morgue' do
-      redirect request.path unless params['key']
-
-      params['key'].each do |key|
-        job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
-        retry_or_delete_or_kill job, params if job
+      def views
+        @views ||= VIEWS
       end
-      redirect_with_query("#{root_path}morgue")
-    end
 
-    post "/morgue/all/delete" do
-      Sidekiq::DeadSet.new.clear
-      redirect "#{root_path}morgue"
+      attr_accessor :app_url, :session_secret, :redis_pool
+      attr_writer :locales, :views
     end
 
-    post "/morgue/all/retry" do
-      Sidekiq::DeadSet.new.retry_all
-      redirect "#{root_path}morgue"
+    def settings
+      self.class.settings
     end
 
-    post "/morgue/:key" do
-      halt 404 unless params['key']
-      job = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
-      retry_or_delete_or_kill job, params if job
-      redirect_with_query("#{root_path}morgue")
+    def use(*middleware_args, &block)
+      middlewares << [middleware_args, block]
     end
 
-
-    get '/retries' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @retries) = page("retry", params[:page], @count)
-      @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
-      erb :retries
-    end
-
-    get "/retries/:key" do
-      @retry = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}retries" if @retry.nil?
-      erb :retry
-    end
-
-    post '/retries' do
-      redirect request.path unless params['key']
-
-      params['key'].each do |key|
-        job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
-        retry_or_delete_or_kill job, params if job
-      end
-      redirect_with_query("#{root_path}retries")
+    def middlewares
+      @middlewares ||= Web.middlewares.dup
     end
 
-    post "/retries/all/delete" do
-      Sidekiq::RetrySet.new.clear
-      redirect "#{root_path}retries"
+    def call(env)
+      app.call(env)
     end
 
-    post "/retries/all/retry" do
-      Sidekiq::RetrySet.new.retry_all
-      redirect "#{root_path}retries"
+    def self.call(env)
+      @app ||= new
+      @app.call(env)
     end
 
-    post "/retries/:key" do
-      job = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      retry_or_delete_or_kill job, params if job
-      redirect_with_query("#{root_path}retries")
+    def app
+      @app ||= build
     end
 
-    get '/scheduled' do
-      @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @scheduled) = page("schedule", params[:page], @count)
-      @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
-      erb :scheduled
+    def self.register(extension)
+      extension.registered(WebApplication)
     end
 
-    get "/scheduled/:key" do
-      @job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      redirect "#{root_path}scheduled" if @job.nil?
-      erb :scheduled_job_info
-    end
-
-    post '/scheduled' do
-      redirect request.path unless params['key']
+    private
 
-      params['key'].each do |key|
-        job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
-        delete_or_add_queue job, params if job
+    def using?(middleware)
+      middlewares.any? do |(m,_)|
+        m.kind_of?(Array) && (m[0] == middleware || m[0].kind_of?(middleware))
       end
-      redirect_with_query("#{root_path}scheduled")
-    end
-
-    post "/scheduled/:key" do
-      halt 404 unless params['key']
-      job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      delete_or_add_queue job, params if job
-      redirect_with_query("#{root_path}scheduled")
-    end
-
-    get '/' do
-      @redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
-      stats_history = Sidekiq::Stats::History.new((params[:days] || 30).to_i)
-      @processed_history = stats_history.processed
-      @failed_history = stats_history.failed
-      erb :dashboard
     end
 
-    REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+    def build
+      middlewares = self.middlewares
+      klass = self.class
 
-    get '/dashboard/stats' do
-      redirect "#{root_path}stats"
-    end
-
-    get '/stats' do
-      sidekiq_stats = Sidekiq::Stats.new
-      redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
-
-      content_type :json
-      Sidekiq.dump_json(
-        sidekiq: {
-          processed:       sidekiq_stats.processed,
-          failed:          sidekiq_stats.failed,
-          busy:            sidekiq_stats.workers_size,
-          processes:       sidekiq_stats.processes_size,
-          enqueued:        sidekiq_stats.enqueued,
-          scheduled:       sidekiq_stats.scheduled_size,
-          retries:         sidekiq_stats.retry_size,
-          dead:            sidekiq_stats.dead_size,
-          default_latency: sidekiq_stats.default_queue_latency
-        },
-        redis: redis_stats
-      )
-    end
+      unless using?(::Rack::Protection) || ENV['RACK_ENV'] == 'test'
+        middlewares.unshift [[::Rack::Protection, { use: :authenticity_token }], nil]
+      end
 
-    get '/stats/queues' do
-      queue_stats = Sidekiq::Stats::Queues.new
+      unless using? ::Rack::Session::Cookie
+        unless secret = Web.session_secret
+          require 'securerandom'
+          secret = SecureRandom.hex(64)
+        end
 
-      content_type :json
-      Sidekiq.dump_json(
-        queue_stats.lengths
-      )
-    end
+        middlewares.unshift [[::Rack::Session::Cookie, { secret: secret }], nil]
+      end
 
-    private
+      ::Rack::Builder.new do
+        %w(stylesheets javascripts images).each do |asset_dir|
+          map "/#{asset_dir}" do
+            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", { 'Cache-Control' => 'public, max-age=86400' })
+          end
+        end
 
-    def retry_or_delete_or_kill job, params
-      if params['retry']
-        job.retry
-      elsif params['delete']
-        job.delete
-      elsif params['kill']
-        job.kill
-      end
-    end
+        middlewares.each {|middleware, block| use(*middleware, &block) }
 
-    def delete_or_add_queue job, params
-      if params['delete']
-        job.delete
-      elsif params['add_to_queue']
-        job.add_to_queue
+        run WebApplication.new(klass)
       end
     end
   end
+
+  Sidekiq::WebApplication.helpers WebHelpers
+  Sidekiq::WebApplication.helpers Sidekiq::Paginator
+
+  Sidekiq::WebAction.class_eval "def _render\n#{ERB.new(File.read(Web::LAYOUT)).src}\nend"
 end
 
 if defined?(::ActionDispatch::Request::Session) &&
     !::ActionDispatch::Request::Session.respond_to?(:each)
   # mperham/sidekiq#2460
   # Rack apps can't reuse the Rails session store without
-  # this monkeypatch
+  # this monkeypatch, fixed in Rails 5.
   class ActionDispatch::Request::Session
     def each(&block)
       hash = self.to_hash