sidekiq 7.3.9 → 8.0.0.beta2

data/lib/sidekiq/middleware/current_attributes.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "active_job/arguments"
 require "active_support/current_attributes"
 
 module Sidekiq
@@ -20,6 +21,8 @@ module Sidekiq
   #   Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
   #
   module CurrentAttributes
+    Serializer = ::ActiveJob::Arguments
+
     class Save
       include Sidekiq::ClientMiddleware
 
@@ -33,26 +36,11 @@ module Sidekiq
             attrs = strklass.constantize.attributes
             # Retries can push the job N times, we don't
             # want retries to reset cattr. #5692, #5090
-            if attrs.any?
-              # Older rails has a bug that `CurrentAttributes#attributes` always returns
-              # the same hash instance. We need to dup it to avoid being accidentally mutated.
-              job[key] = if returns_same_object?
-                attrs.dup
-              else
-                attrs
-              end
-            end
+            job[key] = Serializer.serialize(attrs) if attrs.any?
           end
         end
         yield
       end
-
-      private
-
-      def returns_same_object?
-        ActiveSupport::VERSION::MAJOR < 8 ||
-          (ActiveSupport::VERSION::MAJOR == 8 && ActiveSupport::VERSION::MINOR == 0)
-      end
     end
 
     class Load
@@ -68,7 +56,7 @@ module Sidekiq
         @cattrs.each do |(key, strklass)|
           next unless job.has_key?(key)
 
-          klass_attrs[strklass.constantize] = job[key]
+          klass_attrs[strklass.constantize] = Serializer.deserialize(job[key]).to_h
         end
 
         wrap(klass_attrs.to_a, &block)

data/lib/sidekiq/paginator.rb

@@ -17,7 +17,14 @@ module Sidekiq
       ending = starting + page_size - 1
 
       Sidekiq.redis do |conn|
-        type = conn.type(key)
+        # horrible, think you can make this cleaner?
+        type = TYPE_CACHE[key]
+        if type
+        elsif key.start_with?("queue:")
+          type = TYPE_CACHE[key] = "list"
+        else
+          type = TYPE_CACHE[key] = conn.type(key)
+        end
         rev = opts && opts[:reverse]
 
         case type

data/lib/sidekiq/processor.rb

@@ -3,6 +3,7 @@
 require "sidekiq/fetch"
 require "sidekiq/job_logger"
 require "sidekiq/job_retry"
+require "sidekiq/profiler"
 
 module Sidekiq
   ##
@@ -66,7 +67,7 @@ module Sidekiq
       @thread ||= safe_thread("#{config.name}/processor", &method(:run))
     end
 
-    private unless $TESTING
+    private
 
     def run
       # By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
@@ -112,13 +113,17 @@ module Sidekiq
     def handle_fetch_exception(ex)
       unless @down
         @down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-        logger.error("Error fetching job: #{ex}")
         handle_exception(ex)
       end
       sleep(1)
       nil
     end
 
+    def profile(job, &block)
+      return yield unless job["profile"]
+      Sidekiq::Profiler.new(config).call(job, &block)
+    end
+
     def dispatch(job_hash, queue, jobstr)
       # since middleware can mutate the job hash
       # we need to clone it to report the original
@@ -132,17 +137,19 @@ module Sidekiq
         @retrier.global(jobstr, queue) do
           @job_logger.call(job_hash, queue) do
             stats(jobstr, queue) do
-              # Rails 5 requires a Reloader to wrap code execution.  In order to
-              # constantize the worker and instantiate an instance, we have to call
-              # the Reloader.  It handles code loading, db connection management, etc.
-              # Effectively this block denotes a "unit of work" to Rails.
-              @reloader.call do
-                klass = Object.const_get(job_hash["class"])
-                instance = klass.new
-                instance.jid = job_hash["jid"]
-                instance._context = self
-                @retrier.local(instance, jobstr, queue) do
-                  yield instance
+              profile(job_hash) do
+                # Rails 5 requires a Reloader to wrap code execution.  In order to
+                # constantize the worker and instantiate an instance, we have to call
+                # the Reloader.  It handles code loading, db connection management, etc.
+                # Effectively this block denotes a "unit of work" to Rails.
+                @reloader.call do
+                  klass = Object.const_get(job_hash["class"])
+                  instance = klass.new
+                  instance.jid = job_hash["jid"]
+                  instance._context = self
+                  @retrier.local(instance, jobstr, queue) do
+                    yield instance
+                  end
                 end
               end
             end
@@ -165,7 +172,6 @@ module Sidekiq
       begin
         job_hash = Sidekiq.load_json(jobstr)
       rescue => ex
-        handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
         now = Time.now.to_f
         redis do |conn|
           conn.multi do |xa|
@@ -174,6 +180,7 @@ module Sidekiq
             xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
           end
         end
+        handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
         return uow.acknowledge
       end
 

data/lib/sidekiq/profiler.rb

@@ -0,0 +1,59 @@
+require "fileutils"
+require "sidekiq/component"
+
+module Sidekiq
+  # Allows the user to profile jobs running in production.
+  # See details in the Profiling wiki page.
+  class Profiler
+    EXPIRY = 86400 # 1 day
+    DEFAULT_OPTIONS = {
+      mode: :wall
+    }
+
+    include Sidekiq::Component
+    def initialize(config)
+      @config = config
+    end
+
+    def call(job, &block)
+      return yield unless job["profile"]
+
+      token = job["profile"]
+      type = job["class"]
+      jid = job["jid"]
+      started_at = Time.now
+      options = DEFAULT_OPTIONS.merge((job["profiler_options"] || {}).transform_keys!(&:to_sym))
+
+      rundata = {
+        started_at: started_at.to_i,
+        token: token,
+        type: type,
+        jid: jid,
+        # .gz extension tells Vernier to compress the data
+        filename: "#{token}-#{type}-#{jid}-#{started_at.strftime("%Y%m%d-%H%M%S")}.json.gz"
+      }
+
+      require "vernier"
+      begin
+        a = Time.now
+        rc = Vernier.profile(**options.merge(out: rundata[:filename]), &block)
+        b = Time.now
+
+        # Failed jobs will raise an exception on previous line and skip this
+        # block. Only successful jobs will persist profile data to Redis.
+        key = "#{token}-#{jid}"
+        data = File.read(rundata[:filename])
+        redis do |conn|
+          conn.multi do |m|
+            m.zadd("profiles", Time.now.to_f + EXPIRY, key)
+            m.hset(key, rundata.merge(elapsed: (b - a), data: data, size: data.bytesize))
+            m.expire(key, EXPIRY)
+          end
+        end
+        rc
+      ensure
+        FileUtils.rm_f(rundata[:filename])
+      end
+    end
+  end
+end

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
 require "redis_client"
 require "redis_client/decorator"
 

data/lib/sidekiq/testing.rb

@@ -87,7 +87,7 @@ module Sidekiq
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
-          job["enqueued_at"] = Time.now.to_f unless job["at"]
+          job["enqueued_at"] = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) unless job["at"]
           Queues.push(job["queue"], job["class"], job)
         end
         true
@@ -329,6 +329,6 @@ module Sidekiq
   end
 end
 
-if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING # rubocop:disable Style/GlobalVars
   warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end

data/lib/sidekiq/version.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.3.9"
-  MAJOR = 7
+  VERSION = "8.0.0.beta2"
+  MAJOR = 8
 
   def self.gem_version
     Gem::Version.new(VERSION)

data/lib/sidekiq/web/action.rb

@@ -1,115 +1,135 @@
 # frozen_string_literal: true
 
-module Sidekiq
-  class WebAction
-    RACK_SESSION = "rack.session"
+require "erb"
 
-    attr_accessor :env, :block, :type
+module Sidekiq
+  class Web
+    ##
+    # These instance methods are available to all executing ERB
+    # templates.
+    class Action
+      attr_accessor :env, :block
+
+      def initialize(env, block)
+        @_erb = false
+        @env = env
+        @block = block
+      end
 
-    def settings
-      Web.settings
-    end
+      def config
+        env[:web_config]
+      end
 
-    def request
-      @request ||= ::Rack::Request.new(env)
-    end
+      def request
+        @request ||= ::Rack::Request.new(env)
+      end
 
-    def halt(res)
-      throw :halt, [res, {Rack::CONTENT_TYPE => "text/plain"}, [res.to_s]]
-    end
+      def halt(res)
+        throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
+      end
 
-    def redirect(location)
-      throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
-    end
+      # external redirect
+      def redirect_to(url)
+        throw :halt, [302, {"Location" => url}, []]
+      end
 
-    def reload_page
-      current_location = request.referer.gsub(request.base_url, "")
-      redirect current_location
-    end
+      def header(key, value)
+        env["response_headers"][key] = value
+      end
 
-    # deprecated, will warn in 8.0
-    def params
-      indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
+      # internal redirect
+      def redirect(location)
+        throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
+      end
 
-      indifferent_hash.merge! request.params
-      route_params.each { |k, v| indifferent_hash[k.to_s] = v }
+      def reload_page
+        current_location = request.referer.gsub(request.base_url, "")
+        redirect current_location
+      end
 
-      indifferent_hash
-    end
+      # stuff after ? or form input
+      # uses String keys, no Symbols!
+      def url_params(key)
+        warn { "URL parameter `#{key}` should be accessed via String, not Symbol (at #{caller(3..3).first})" } if key.is_a?(Symbol)
+        request.params[key.to_s]
+      end
 
-    # Use like `url_params("page")` within your action blocks
-    def url_params(key)
-      request.params[key]
-    end
+      # variables embedded in path, `/metrics/:name`
+      # uses Symbol keys, no Strings!
+      def route_params(key)
+        warn { "Route parameter `#{key}` should be accessed via Symbol, not String (at #{caller(3..3).first})" } if key.is_a?(String)
+        env["rack.route_params"][key.to_sym]
+      end
 
-    # Use like `route_params(:name)` within your action blocks
-    # key is required in 8.0, nil is only used for backwards compatibility
-    def route_params(key = nil)
-      if key
-        env[WebRouter::ROUTE_PARAMS][key]
-      else
-        env[WebRouter::ROUTE_PARAMS]
+      def params
+        warn { "Direct access to Rack parameters is discouraged, use `url_params` or `route_params` (at #{caller(3..3).first})" }
+        request.params
       end
-    end
 
-    def session
-      env[RACK_SESSION]
-    end
+      def session
+        env["rack.session"]
+      end
 
-    def erb(content, options = {})
-      if content.is_a? Symbol
-        unless respond_to?(:"_erb_#{content}")
-          views = options[:views] || Web.settings.views
-          filename = "#{views}/#{content}.erb"
-          src = ERB.new(File.read(filename)).src
-
-          # Need to use lineno less by 1 because erb generates a
-          # comment before the source code.
-          WebAction.class_eval <<-RUBY, filename, -1 # standard:disable Style/EvalWithLocation
-            def _erb_#{content}
-              #{src}
-            end
-          RUBY
+      def erb(content, options = {})
+        if content.is_a? Symbol
+          unless respond_to?(:"_erb_#{content}")
+            views = options[:views] || Web.views
+            filename = "#{views}/#{content}.erb"
+            src = ERB.new(File.read(filename)).src
+
+            # Need to use lineno less by 1 because erb generates a
+            # comment before the source code.
+            Action.class_eval <<-RUBY, filename, -1 # standard:disable Style/EvalWithLocation
+              def _erb_#{content}
+                #{src}
+              end
+            RUBY
+          end
         end
-      end
 
-      if @_erb
-        _erb(content, options[:locals])
-      else
-        @_erb = true
-        content = _erb(content, options[:locals])
+        if @_erb
+          _erb(content, options[:locals])
+        else
+          @_erb = true
+          content = _erb(content, options[:locals])
 
-        _render { content }
+          _render { content }
+        end
       end
-    end
 
-    def render(engine, content, options = {})
-      raise "Only erb templates are supported" if engine != :erb
+      def render(engine, content, options = {})
+        raise "Only erb templates are supported" if engine != :erb
 
-      erb(content, options)
-    end
+        erb(content, options)
+      end
 
-    def json(payload)
-      [200, {Rack::CONTENT_TYPE => "application/json", Rack::CACHE_CONTROL => "private, no-store"}, [Sidekiq.dump_json(payload)]]
-    end
+      def json(payload)
+        [200,
+          {"content-type" => "application/json", "cache-control" => "private, no-store"},
+          [Sidekiq.dump_json(payload)]]
+      end
 
-    def initialize(env, block)
-      @_erb = false
-      @env = env
-      @block = block
-      @files ||= {}
-    end
+      private
 
-    private
+      def warn
+        Sidekiq.logger.warn yield
+      end
 
-    def _erb(file, locals)
-      locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
+      def _erb(file, locals)
+        locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
 
-      if file.is_a?(String)
-        ERB.new(file).result(binding)
-      else
-        send(:"_erb_#{file}")
+        if file.is_a?(String)
+          ERB.new(file).result(binding)
+        else
+          send(:"_erb_#{file}")
+        end
       end
+
+      class_eval <<-RUBY, ::Sidekiq::Web::LAYOUT, -1 # standard:disable Style/EvalWithLocation
+        def _render
+          #{ERB.new(File.read(::Sidekiq::Web::LAYOUT)).src}
+        end
+      RUBY
     end
   end
 end