sidekiq 4.1.4 → 4.2.10

data/lib/sidekiq/launcher.rb

@@ -1,5 +1,5 @@
-# frozen_string_literal: true
 # encoding: utf-8
+# frozen_string_literal: true
 require 'sidekiq/manager'
 require 'sidekiq/fetch'
 require 'sidekiq/scheduled'
@@ -63,15 +63,16 @@ module Sidekiq
 
     JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
 
-    def heartbeat(k, data, json)
-      results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, data) }
+    def heartbeat
+      results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, to_data) }
       results.compact!
       $0 = results.join(' ')
 
-      ❤(k, json)
+      ❤
     end
 
-    def ❤(key, json)
+    def ❤
+      key = identity
       fails = procd = 0
       begin
         Processor::FAILURE.update {|curr| fails = curr; 0 }
@@ -94,15 +95,19 @@ module Sidekiq
         end
         fails = procd = 0
 
-        _, _, _, msg = Sidekiq.redis do |conn|
+        _, exists, _, _, msg = Sidekiq.redis do |conn|
           conn.multi do
             conn.sadd('processes', key)
-            conn.hmset(key, 'info', json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done)
+            conn.exists(key)
+            conn.hmset(key, 'info', to_json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           end
         end
 
+        # first heartbeat or recovering from an outage and need to reestablish our heartbeat
+        fire_event(:heartbeat) if !exists
+
         return unless msg
 
         if JVM_RESERVED_SIGNALS.include?(msg)
@@ -120,28 +125,36 @@ module Sidekiq
     end
 
     def start_heartbeat
-      k = identity
-      data = {
-        'hostname' => hostname,
-        'started_at' => Time.now.to_f,
-        'pid' => $$,
-        'tag' => @options[:tag] || '',
-        'concurrency' => @options[:concurrency],
-        'queues' => @options[:queues].uniq,
-        'labels' => @options[:labels],
-        'identity' => k,
-      }
-      # this data doesn't change so dump it to a string
-      # now so we don't need to dump it every heartbeat.
-      json = Sidekiq.dump_json(data)
-
       while true
-        heartbeat(k, data, json)
+        heartbeat
         sleep 5
       end
       Sidekiq.logger.info("Heartbeat stopping...")
     end
 
+    def to_data
+      @data ||= begin
+        {
+          'hostname' => hostname,
+          'started_at' => Time.now.to_f,
+          'pid' => $$,
+          'tag' => @options[:tag] || '',
+          'concurrency' => @options[:concurrency],
+          'queues' => @options[:queues].uniq,
+          'labels' => @options[:labels],
+          'identity' => identity,
+        }
+      end
+    end
+
+    def to_json
+      @json ||= begin
+        # this data changes infrequently so dump it to a string
+        # now so we don't need to dump it every heartbeat.
+        Sidekiq.dump_json(to_data)
+      end
+    end
+
     def clear_heartbeat
       # Remove record from Redis since we are shutting down.
       # Note we don't stop the heartbeat thread; if the process

data/lib/sidekiq/manager.rb

@@ -1,15 +1,16 @@
-# frozen_string_literal: true
 # encoding: utf-8
+# frozen_string_literal: true
 require 'sidekiq/util'
 require 'sidekiq/processor'
 require 'sidekiq/fetch'
 require 'thread'
+require 'set'
 
 module Sidekiq
 
   ##
   # The Manager is the central coordination point in Sidekiq, controlling
-  # the lifecycle of the Processors and feeding them jobs as necessary.
+  # the lifecycle of the Processors.
   #
   # Tasks:
   #

data/lib/sidekiq/middleware/server/logging.rb

@@ -4,28 +4,19 @@ module Sidekiq
       class Logging
 
         def call(worker, item, queue)
-          Sidekiq::Logging.with_context(log_context(worker, item)) do
-            begin
-              start = Time.now
-              logger.info("start".freeze)
-              yield
-              logger.info("done: #{elapsed(start)} sec")
-            rescue Exception
-              logger.info("fail: #{elapsed(start)} sec")
-              raise
-            end
+          begin
+            start = Time.now
+            logger.info("start".freeze)
+            yield
+            logger.info("done: #{elapsed(start)} sec")
+          rescue Exception
+            logger.info("fail: #{elapsed(start)} sec")
+            raise
           end
         end
 
         private
 
-        # If we're using a wrapper class, like ActiveJob, use the "wrapped"
-        # attribute to expose the underlying thing.
-        def log_context(worker, item)
-          klass = item['wrapped'.freeze] || worker.class.to_s
-          "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
-        end
-
         def elapsed(start)
           (Time.now - start).round(3)
         end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -162,7 +162,7 @@ module Sidekiq
         end
 
         def retry_attempts_from(msg_retry, default)
-          if msg_retry.is_a?(Fixnum)
+          if msg_retry.is_a?(Integer)
             msg_retry
           else
             default

data/lib/sidekiq/processor.rb

@@ -36,6 +36,8 @@ module Sidekiq
       @job = nil
       @thread = nil
       @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
+      @reloader = Sidekiq.options[:reloader]
+      @executor = Sidekiq.options[:executor]
     end
 
     def terminate(wait=false)
@@ -120,34 +122,47 @@ module Sidekiq
 
       ack = false
       begin
-        job = Sidekiq.load_json(jobstr)
-        klass  = job['class'.freeze].constantize
-        worker = klass.new
-        worker.jid = job['jid'.freeze]
-
-        stats(worker, job, queue) do
-          Sidekiq.server_middleware.invoke(worker, job, queue) do
-            # Only ack if we either attempted to start this job or
-            # successfully completed it. This prevents us from
-            # losing jobs if a middleware raises an exception before yielding
-            ack = true
-            execute_job(worker, cloned(job['args'.freeze]))
+        job_hash = Sidekiq.load_json(jobstr)
+        @reloader.call do
+          klass  = job_hash['class'.freeze].constantize
+          worker = klass.new
+          worker.jid = job_hash['jid'.freeze]
+
+          stats(worker, job_hash, queue) do
+            Sidekiq::Logging.with_context(log_context(job_hash)) do
+              ack = true
+              Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
+                @executor.call do
+                  # Only ack if we either attempted to start this job or
+                  # successfully completed it. This prevents us from
+                  # losing jobs if a middleware raises an exception before yielding
+                  execute_job(worker, cloned(job_hash['args'.freeze]))
+                end
+              end
+            end
           end
+          ack = true
         end
-        ack = true
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
         # within the timeout.  Don't acknowledge the work since
         # we didn't properly finish it.
         ack = false
       rescue Exception => ex
-        handle_exception(ex, job || { :job => jobstr })
+        handle_exception(ex, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
         raise
       ensure
         work.acknowledge if ack
       end
     end
 
+    # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+    # attribute to expose the underlying thing.
+    def log_context(item)
+      klass = item['wrapped'.freeze] || item['class'.freeze]
+      "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+    end
+
     def execute_job(worker, cloned_args)
       worker.perform(*cloned_args)
     end
@@ -160,9 +175,9 @@ module Sidekiq
     PROCESSED = Concurrent::AtomicFixnum.new
     FAILURE = Concurrent::AtomicFixnum.new
 
-    def stats(worker, job, queue)
+    def stats(worker, job_hash, queue)
       tid = thread_identity
-      WORKER_STATE[tid] = {:queue => queue, :payload => job, :run_at => Time.now.to_i }
+      WORKER_STATE[tid] = {:queue => queue, :payload => cloned(job_hash), :run_at => Time.now.to_i }
 
       begin
         yield

data/lib/sidekiq/rails.rb

@@ -32,8 +32,92 @@ module Sidekiq
   end
 
   class Rails < ::Rails::Engine
+    # We need to setup this up before any application configuration which might
+    # change Sidekiq middleware.
+    #
+    # This hook happens after `Rails::Application` is inherited within
+    # config/application.rb and before config is touched, usually within the
+    # class block. Definitely before config/environments/*.rb and
+    # config/initializers/*.rb.
+    config.before_configuration do
+      if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
+        Sidekiq.server_middleware do |chain|
+          require 'sidekiq/middleware/server/active_record'
+          chain.add Sidekiq::Middleware::Server::ActiveRecord
+        end
+      end
+    end
+
     initializer 'sidekiq' do
       Sidekiq.hook_rails!
     end
+
+    config.after_initialize do
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      # We have to add the reloader after initialize to see if cache_classes has
+      # been turned on.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      #
+      Sidekiq.configure_server do |_|
+        if ::Rails::VERSION::MAJOR >= 5
+          # The reloader also takes care of ActiveRecord but is incompatible with
+          # the ActiveRecord middleware so make sure it's not in the chain already.
+          if defined?(Sidekiq::Middleware::Server::ActiveRecord) && Sidekiq.server_middleware.exists?(Sidekiq::Middleware::Server::ActiveRecord)
+            raise ArgumentError, "You are using the Sidekiq ActiveRecord middleware and the new Rails 5 reloader which are incompatible. Please remove the ActiveRecord middleware from your Sidekiq middleware configuration."
+          elsif ::Rails.application.config.cache_classes
+            # The reloader API has proven to be troublesome under load in production.
+            # We won't use it at all when classes are cached, see #3154
+            Sidekiq.logger.debug { "Autoload disabled in #{::Rails.env}, Sidekiq will not reload changed classes" }
+            Sidekiq.options[:executor] = Sidekiq::Rails::Executor.new
+          else
+            Sidekiq.logger.debug { "Enabling Rails 5+ live code reloading, so hot!" }
+            Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
+            Psych::Visitors::ToRuby.prepend(Sidekiq::Rails::PsychAutoload)
+          end
+        end
+      end
+    end
+
+    class Executor
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.executor.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Executor @app=#{@app.class.name}>"
+      end
+    end
+
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
+
+    module PsychAutoload
+      def resolve_class(klass_name)
+        klass_name && klass_name.constantize
+      rescue NameError
+        super
+      end
+    end
   end if defined?(::Rails)
 end

data/lib/sidekiq/redis_connection.rb

@@ -67,7 +67,14 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] = opts[:driver] || 'ruby'
+        opts[:driver] ||= 'ruby'
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
 
         opts
       end

data/lib/sidekiq/scheduled.rb

@@ -46,6 +46,7 @@ module Sidekiq
         @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
+        @thread = nil
       end
 
       # Shut down this instance, will pause until the thread is dead.

data/lib/sidekiq/testing.rb

@@ -69,7 +69,9 @@ module Sidekiq
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          Queues.push(job['queue'], job['class'], Sidekiq.load_json(Sidekiq.dump_json(job)))
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
+          Queues.push(job['queue'], job['class'], job)
         end
         true
       elsif Sidekiq::Testing.inline?
@@ -264,7 +266,7 @@ module Sidekiq
       def drain
         while jobs.any?
           next_job = jobs.first
-          Queues.delete_for(next_job["jid"], queue, self.to_s)
+          Queues.delete_for(next_job["jid"], next_job["queue"], self.to_s)
           process_job(next_job)
         end
       end
@@ -314,3 +316,9 @@ module Sidekiq
     end
   end
 end
+
+if defined?(::Rails) && !Rails.env.test?
+  puts("**************************************************")
+  puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
+  puts("**************************************************")
+end

data/lib/sidekiq/util.rb

@@ -22,6 +22,7 @@ module Sidekiq
 
     def safe_thread(name, &block)
       Thread.new do
+        Thread.current['sidekiq_label'] = name
         watchdog(name, &block)
       end
     end
@@ -53,7 +54,7 @@ module Sidekiq
         begin
           block.call
         rescue => ex
-          handle_exception(ex, { event: event })
+          handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
         end
       end
       arr.clear

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "4.1.4"
+  VERSION = "4.2.10"
 end

data/lib/sidekiq/web/action.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class WebAction
+    RACK_SESSION = 'rack.session'.freeze
+
+    attr_accessor :env, :block, :type
+
+    def settings
+      Web.settings
+    end
+
+    def request
+      @request ||= ::Rack::Request.new(env)
+    end
+
+    def halt(res)
+      throw :halt, res
+    end
+
+    def redirect(location)
+      throw :halt, [302, { "Location" => "#{request.base_url}#{location}" }, []]
+    end
+
+    def params
+      indifferent_hash = Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
+
+      indifferent_hash.merge! request.params
+      route_params.each {|k,v| indifferent_hash[k.to_s] = v }
+
+      indifferent_hash
+    end
+
+    def route_params
+      env[WebRouter::ROUTE_PARAMS]
+    end
+
+    def session
+      env[RACK_SESSION]
+    end
+
+    def content_type(type)
+      @type = type
+    end
+
+    def erb(content, options = {})
+      if content.kind_of? Symbol
+        unless respond_to?(:"_erb_#{content}")
+          src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
+          WebAction.class_eval("def _erb_#{content}\n#{src}\n end")
+        end
+      end
+
+      if @_erb
+        _erb(content, options[:locals])
+      else
+        @_erb = true
+        content = _erb(content, options[:locals])
+
+        _render { content }
+      end
+    end
+
+    def render(engine, content, options = {})
+      raise "Only erb templates are supported" if engine != :erb
+
+      erb(content, options)
+    end
+
+    def json(payload)
+      [200, { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }, [Sidekiq.dump_json(payload)]]
+    end
+
+    def initialize(env, block)
+      @_erb = false
+      @env = env
+      @block = block
+      @@files ||= {}
+    end
+
+    private
+
+    def _erb(file, locals)
+      locals.each {|k, v| define_singleton_method(k){ v } } if locals
+
+      if file.kind_of?(String)
+        ERB.new(file).result(binding)
+      else
+        send(:"_erb_#{file}")
+      end
+    end
+  end
+end