sidekiq 4.2.10 → 5.0.5

data/lib/sidekiq/launcher.rb

@@ -61,8 +61,6 @@ module Sidekiq
 
     private unless $TESTING
 
-    JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
-
     def heartbeat
       results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, to_data) }
       results.compact!
@@ -110,11 +108,7 @@ module Sidekiq
 
         return unless msg
 
-        if JVM_RESERVED_SIGNALS.include?(msg)
-          Sidekiq::CLI.instance.handle_signal(msg)
-        else
-          ::Process.kill(msg, $$)
-        end
+        ::Process.kill(msg, $$)
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")

data/lib/sidekiq/logging.rb

@@ -26,6 +26,18 @@ module Sidekiq
       end
     end
 
+    def self.job_hash_context(job_hash)
+      # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+      # attribute to expose the underlying thing.
+      klass = job_hash['wrapped'.freeze] || job_hash["class".freeze]
+      bid = job_hash['bid'.freeze]
+      "#{klass} JID-#{job_hash['jid'.freeze]}#{" BID-#{bid}" if bid}"
+    end
+
+    def self.with_job_hash_context(job_hash, &block)
+      with_context(job_hash_context(job_hash), &block)
+    end
+
     def self.with_context(msg)
       Thread.current[:sidekiq_context] ||= []
       Thread.current[:sidekiq_context] << msg

data/lib/sidekiq/middleware/server/active_record.rb

@@ -2,6 +2,15 @@ module Sidekiq
   module Middleware
     module Server
       class ActiveRecord
+
+        def initialize
+          # With Rails 5+ we must use the Reloader **always**.
+          # The reloader handles code loading and db connection management.
+          if ::Rails::VERSION::MAJOR >= 5
+            raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
+          end
+        end
+
         def call(*args)
           yield
         ensure

data/lib/sidekiq/processor.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 require 'sidekiq/util'
 require 'sidekiq/fetch'
+require 'sidekiq/job_logger'
+require 'sidekiq/job_retry'
 require 'thread'
 require 'concurrent/map'
 require 'concurrent/atomic/atomic_fixnum'
@@ -37,7 +39,8 @@ module Sidekiq
       @thread = nil
       @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
       @reloader = Sidekiq.options[:reloader]
-      @executor = Sidekiq.options[:executor]
+      @logging = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
+      @retrier = Sidekiq::JobRetry.new
     end
 
     def terminate(wait=false)
@@ -116,32 +119,56 @@ module Sidekiq
       nil
     end
 
+    def dispatch(job_hash, queue)
+      # since middleware can mutate the job hash
+      # we clone here so we report the original
+      # job structure to the Web UI
+      pristine = cloned(job_hash)
+
+      Sidekiq::Logging.with_job_hash_context(job_hash) do
+        @retrier.global(pristine, queue) do
+          @logging.call(job_hash, queue) do
+            stats(pristine, queue) do
+              # Rails 5 requires a Reloader to wrap code execution.  In order to
+              # constantize the worker and instantiate an instance, we have to call
+              # the Reloader.  It handles code loading, db connection management, etc.
+              # Effectively this block denotes a "unit of work" to Rails.
+              @reloader.call do
+                klass  = constantize(job_hash['class'.freeze])
+                worker = klass.new
+                worker.jid = job_hash['jid'.freeze]
+                @retrier.local(worker, pristine, queue) do
+                  yield worker
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
     def process(work)
       jobstr = work.job
       queue = work.queue_name
 
       ack = false
       begin
-        job_hash = Sidekiq.load_json(jobstr)
-        @reloader.call do
-          klass  = job_hash['class'.freeze].constantize
-          worker = klass.new
-          worker.jid = job_hash['jid'.freeze]
-
-          stats(worker, job_hash, queue) do
-            Sidekiq::Logging.with_context(log_context(job_hash)) do
-              ack = true
-              Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
-                @executor.call do
-                  # Only ack if we either attempted to start this job or
-                  # successfully completed it. This prevents us from
-                  # losing jobs if a middleware raises an exception before yielding
-                  execute_job(worker, cloned(job_hash['args'.freeze]))
-                end
-              end
-            end
-          end
+        # Treat malformed JSON as a special case: job goes straight to the morgue.
+        job_hash = nil
+        begin
+          job_hash = Sidekiq.load_json(jobstr)
+        rescue => ex
+          handle_exception(ex, { :context => "Invalid JSON for job", :jobstr => jobstr })
+          DeadSet.new.kill(jobstr)
           ack = true
+          raise
+        end
+
+        ack = true
+        dispatch(job_hash, queue) do |worker|
+          Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
+            execute_job(worker, cloned(job_hash['args'.freeze]))
+          end
         end
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
@@ -149,20 +176,14 @@ module Sidekiq
         # we didn't properly finish it.
         ack = false
       rescue Exception => ex
-        handle_exception(ex, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
-        raise
+        e = ex.is_a?(::Sidekiq::JobRetry::Skip) && ex.cause ? ex.cause : ex
+        handle_exception(e, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
+        raise e
       ensure
         work.acknowledge if ack
       end
     end
 
-    # If we're using a wrapper class, like ActiveJob, use the "wrapped"
-    # attribute to expose the underlying thing.
-    def log_context(item)
-      klass = item['wrapped'.freeze] || item['class'.freeze]
-      "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
-    end
-
     def execute_job(worker, cloned_args)
       worker.perform(*cloned_args)
     end
@@ -175,9 +196,9 @@ module Sidekiq
     PROCESSED = Concurrent::AtomicFixnum.new
     FAILURE = Concurrent::AtomicFixnum.new
 
-    def stats(worker, job_hash, queue)
+    def stats(job_hash, queue)
       tid = thread_identity
-      WORKER_STATE[tid] = {:queue => queue, :payload => cloned(job_hash), :run_at => Time.now.to_i }
+      WORKER_STATE[tid] = {:queue => queue, :payload => job_hash, :run_at => Time.now.to_i }
 
       begin
         yield
@@ -193,8 +214,17 @@ module Sidekiq
     # Deep clone the arguments passed to the worker so that if
     # the job fails, what is pushed back onto Redis hasn't
     # been mutated by the worker.
-    def cloned(ary)
-      Marshal.load(Marshal.dump(ary))
+    def cloned(thing)
+      Marshal.load(Marshal.dump(thing))
+    end
+
+    def constantize(str)
+      names = str.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
     end
 
   end

data/lib/sidekiq/rails.rb

@@ -1,36 +1,5 @@
 # frozen_string_literal: true
 module Sidekiq
-  def self.hook_rails!
-    return if defined?(@delay_removed)
-
-    ActiveSupport.on_load(:active_record) do
-      include Sidekiq::Extensions::ActiveRecord
-    end
-
-    ActiveSupport.on_load(:action_mailer) do
-      extend Sidekiq::Extensions::ActionMailer
-    end
-
-    Module.__send__(:include, Sidekiq::Extensions::Klass)
-  end
-
-  # Removes the generic aliases which MAY clash with names of already
-  #  created methods by other applications. The methods `sidekiq_delay`,
-  #  `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
-  def self.remove_delay!
-    @delay_removed = true
-
-    [Extensions::ActiveRecord,
-     Extensions::ActionMailer,
-     Extensions::Klass].each do |mod|
-      mod.module_eval do
-        remove_method :delay if respond_to?(:delay)
-        remove_method :delay_for if respond_to?(:delay_for)
-        remove_method :delay_until if respond_to?(:delay_until)
-      end
-    end
-  end
-
   class Rails < ::Rails::Engine
     # We need to setup this up before any application configuration which might
     # change Sidekiq middleware.
@@ -48,10 +17,6 @@ module Sidekiq
       end
     end
 
-    initializer 'sidekiq' do
-      Sidekiq.hook_rails!
-    end
-
     config.after_initialize do
       # This hook happens after all initializers are run, just before returning
       # from config/environment.rb back to sidekiq/cli.rb.
@@ -62,38 +27,9 @@ module Sidekiq
       #
       Sidekiq.configure_server do |_|
         if ::Rails::VERSION::MAJOR >= 5
-          # The reloader also takes care of ActiveRecord but is incompatible with
-          # the ActiveRecord middleware so make sure it's not in the chain already.
-          if defined?(Sidekiq::Middleware::Server::ActiveRecord) && Sidekiq.server_middleware.exists?(Sidekiq::Middleware::Server::ActiveRecord)
-            raise ArgumentError, "You are using the Sidekiq ActiveRecord middleware and the new Rails 5 reloader which are incompatible. Please remove the ActiveRecord middleware from your Sidekiq middleware configuration."
-          elsif ::Rails.application.config.cache_classes
-            # The reloader API has proven to be troublesome under load in production.
-            # We won't use it at all when classes are cached, see #3154
-            Sidekiq.logger.debug { "Autoload disabled in #{::Rails.env}, Sidekiq will not reload changed classes" }
-            Sidekiq.options[:executor] = Sidekiq::Rails::Executor.new
-          else
-            Sidekiq.logger.debug { "Enabling Rails 5+ live code reloading, so hot!" }
-            Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
-            Psych::Visitors::ToRuby.prepend(Sidekiq::Rails::PsychAutoload)
-          end
-        end
-      end
-    end
-
-    class Executor
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        @app.executor.wrap do
-          yield
+          Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
         end
       end
-
-      def inspect
-        "#<Sidekiq::Rails::Executor @app=#{@app.class.name}>"
-      end
     end
 
     class Reloader
@@ -111,13 +47,5 @@ module Sidekiq
         "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
       end
     end
-
-    module PsychAutoload
-      def resolve_class(klass_name)
-        klass_name && klass_name.constantize
-      rescue NameError
-        super
-      end
-    end
   end if defined?(::Rails)
 end

data/lib/sidekiq/redis_connection.rb

@@ -8,8 +8,11 @@ module Sidekiq
     class << self
 
       def create(options={})
-        options = options.symbolize_keys
+        options.keys.each do |key|
+          options[key.to_sym] = options.delete(key)
+        end
 
+        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{$$}" if !options.has_key?(:id)
         options[:url] ||= determine_redis_provider
 
         size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
@@ -67,7 +70,7 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] ||= 'ruby'
+        opts[:driver] ||= 'ruby'.freeze
 
         # Issue #3303, redis-rb will silently retry an operation.
         # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
@@ -98,7 +101,15 @@ module Sidekiq
       end
 
       def determine_redis_provider
-        ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+        # If you have this in your environment:
+        # MY_REDIS_URL=redis://hostname.example.com:1238/4
+        # then set:
+        # REDIS_PROVIDER=MY_REDIS_URL
+        # and Sidekiq will find your custom URL variable with no custom
+        # initialization code at all.
+        ENV[
+          ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+        ]
       end
 
     end

data/lib/sidekiq/testing.rb

@@ -55,6 +55,15 @@ module Sidekiq
         yield @server_chain if block_given?
         @server_chain
       end
+
+      def constantize(str)
+        names = str.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      end
     end
   end
 
@@ -76,7 +85,7 @@ module Sidekiq
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = job['class'].constantize
+          klass = Sidekiq::Testing.constantize(job['class'])
           job['id'] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
@@ -309,7 +318,7 @@ module Sidekiq
           worker_classes = jobs.map { |job| job["class"] }.uniq
 
           worker_classes.each do |worker_class|
-            worker_class.constantize.drain
+            Sidekiq::Testing.constantize(worker_class).drain
           end
         end
       end
@@ -317,7 +326,7 @@ module Sidekiq
   end
 end
 
-if defined?(::Rails) && !Rails.env.test?
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
   puts("**************************************************")
   puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
   puts("**************************************************")

data/lib/sidekiq/util.rb

@@ -2,7 +2,6 @@
 require 'socket'
 require 'securerandom'
 require 'sidekiq/exception_handler'
-require 'sidekiq/core_ext'
 
 module Sidekiq
   ##
@@ -22,7 +21,7 @@ module Sidekiq
 
     def safe_thread(name, &block)
       Thread.new do
-        Thread.current['sidekiq_label'] = name
+        Thread.current['sidekiq_label'.freeze] = name
         watchdog(name, &block)
       end
     end

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "4.2.10"
+  VERSION = "5.0.5"
 end

data/lib/sidekiq/web/action.rb

@@ -39,10 +39,6 @@ module Sidekiq
       env[RACK_SESSION]
     end
 
-    def content_type(type)
-      @type = type
-    end
-
     def erb(content, options = {})
       if content.kind_of? Symbol
         unless respond_to?(:"_erb_#{content}")

data/lib/sidekiq/web/application.rb

@@ -234,7 +234,6 @@ module Sidekiq
     get '/stats' do
       sidekiq_stats = Sidekiq::Stats.new
       redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
-
       json(
         sidekiq: {
           processed:       sidekiq_stats.processed,
@@ -247,7 +246,8 @@ module Sidekiq
           dead:            sidekiq_stats.dead_size,
           default_latency: sidekiq_stats.default_queue_latency
         },
-        redis: redis_stats
+        redis: redis_stats,
+        server_utc_time: server_utc_time
       )
     end
 
@@ -274,19 +274,14 @@ module Sidekiq
       resp = case resp
       when Array
         resp
-      when Integer
-        [resp, {}, []]
       else
-        type_header = case action.type
-        when :json
-          { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }
-        when String
-          { "Content-Type" => (action.type || "text/html"), "Cache-Control" => "no-cache" }
-        else
-          { "Content-Type" => "text/html", "Cache-Control" => "no-cache" }
-        end
+        headers = {
+          "Content-Type" => "text/html",
+          "Cache-Control" => "no-cache",
+          "Content-Language" => action.locale,
+        }
 
-        [200, type_header, [resp]]
+        [200, headers, [resp]]
       end
 
       resp[1] = resp[1].dup