sidekiq 7.3.10 → 8.0.10

data/lib/sidekiq/profiler.rb

@@ -0,0 +1,73 @@
+require "fileutils"
+require "sidekiq/component"
+
+module Sidekiq
+  # Allows the user to profile jobs running in production.
+  # See details in the Profiling wiki page.
+  class Profiler
+    EXPIRY = 86400 # 1 day
+    DEFAULT_OPTIONS = {
+      mode: :wall
+    }
+
+    include Sidekiq::Component
+
+    def initialize(config)
+      @config = config
+      @vernier_output_dir = ENV.fetch("VERNIER_OUTPUT_DIR") { Dir.tmpdir }
+    end
+
+    def call(job, &block)
+      return yield unless job["profile"]
+
+      token = job["profile"]
+      type = job["class"]
+      jid = job["jid"]
+      started_at = Time.now
+
+      rundata = {
+        started_at: started_at.to_i,
+        token: token,
+        type: type,
+        jid: jid,
+        # .gz extension tells Vernier to compress the data
+        filename: File.join(
+          @vernier_output_dir,
+          "#{token}-#{type}-#{jid}-#{started_at.strftime("%Y%m%d-%H%M%S")}.json.gz"
+        )
+      }
+      profiler_options = profiler_options(job, rundata)
+
+      require "vernier"
+      begin
+        a = Time.now
+        rc = Vernier.profile(**profiler_options, &block)
+        b = Time.now
+
+        # Failed jobs will raise an exception on previous line and skip this
+        # block. Only successful jobs will persist profile data to Redis.
+        key = "#{token}-#{jid}"
+        data = File.read(rundata[:filename])
+        redis do |conn|
+          conn.multi do |m|
+            m.zadd("profiles", Time.now.to_f + EXPIRY, key)
+            m.hset(key, rundata.merge(elapsed: (b - a), data: data, size: data.bytesize))
+            m.expire(key, EXPIRY)
+          end
+        end
+        rc
+      ensure
+        FileUtils.rm_f(rundata[:filename])
+      end
+    end
+
+    private
+
+    def profiler_options(job, rundata)
+      profiler_options = (job["profiler_options"] || {}).transform_keys(&:to_sym)
+      profiler_options[:mode] = profiler_options[:mode].to_sym if profiler_options[:mode]
+
+      DEFAULT_OPTIONS.merge(profiler_options, {out: rundata[:filename]})
+    end
+  end
+end

data/lib/sidekiq/rails.rb

@@ -1,84 +1,63 @@
 # frozen_string_literal: true
 
-require "sidekiq/job"
-require "rails"
-
 module Sidekiq
-  module ActiveJob
-    # @api private
-    class Wrapper
-      include Sidekiq::Job
-
-      def perform(job_data)
-        ::ActiveJob::Base.execute(job_data.merge("provider_job_id" => jid))
-      end
-    end
-  end
-
-  class Rails < ::Rails::Engine
-    class Reloader
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
-        @app.reloader.wrap(**params) do
-          yield
+  begin
+    gem "railties", ">= 7.0"
+    require "rails"
+    require "sidekiq/job"
+    require_relative "../active_job/queue_adapters/sidekiq_adapter"
+
+    class Rails < ::Rails::Engine
+      class Reloader
+        def initialize(app = ::Rails.application)
+          @app = app
         end
-      end
 
-      def inspect
-        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
-      end
+        def call
+          params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
+          @app.reloader.wrap(**params) do
+            yield
+          end
+        end
 
-      def to_hash
-        {app: @app.class.name}
-      end
-    end
+        def inspect
+          "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+        end
 
-    # By including the Options module, we allow AJs to directly control sidekiq features
-    # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
-    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
-    # manually retried, don't automatically die, etc.
-    #
-    #   class SomeJob < ActiveJob::Base
-    #     queue_as :default
-    #     sidekiq_options retry: 3, backtrace: 10
-    #     def perform
-    #     end
-    #   end
-    initializer "sidekiq.active_job_integration" do
-      ActiveSupport.on_load(:active_job) do
-        require_relative "../active_job/queue_adapters/sidekiq_adapter"
-        include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
+        def to_hash
+          {app: @app.class.name}
+        end
       end
-    end
 
-    initializer "sidekiq.backtrace_cleaner" do
-      Sidekiq.configure_server do |config|
-        config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
+      initializer "sidekiq.backtrace_cleaner" do
+        Sidekiq.configure_server do |config|
+          config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
+        end
       end
-    end
-
-    # This hook happens after all initializers are run, just before returning
-    # from config/environment.rb back to sidekiq/cli.rb.
-    #
-    # None of this matters on the client-side, only within the Sidekiq process itself.
-    config.after_initialize do
-      Sidekiq.configure_server do |config|
-        config[:reloader] = Sidekiq::Rails::Reloader.new
 
-        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
-        # it will appear in the Sidekiq console with all of the job context.
-        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          if ::Rails.logger.respond_to?(:broadcast_to)
-            ::Rails.logger.broadcast_to(config.logger)
-          else
-            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      config.after_initialize do
+        Sidekiq.configure_server do |config|
+          config[:reloader] = Sidekiq::Rails::Reloader.new
+
+          # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
+          # it will appear in the Sidekiq console with all of the job context.
+          unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+            if ::Rails.logger.respond_to?(:broadcast_to)
+              ::Rails.logger.broadcast_to(config.logger)
+            elsif ::ActiveSupport::Logger.respond_to?(:broadcast)
+              ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+            else
+              ::Rails.logger = ::ActiveSupport::BroadcastLogger.new(::Rails.logger, config.logger)
+            end
           end
         end
       end
     end
+  rescue Gem::LoadError
+    # Rails not available or version requirement not met
   end
 end

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
 require "redis_client"
 require "redis_client/decorator"
 

data/lib/sidekiq/redis_connection.rb

@@ -23,7 +23,7 @@ module Sidekiq
 
         size = symbolized_options.delete(:size) || 5
         pool_timeout = symbolized_options.delete(:pool_timeout) || 1
-        pool_name = symbolized_options.delete(:pool_name)
+        symbolized_options.delete(:pool_name)
 
         # Default timeout in redis-client is 1 second, which can be too aggressive
         # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
@@ -33,7 +33,7 @@ module Sidekiq
         symbolized_options[:timeout] ||= 3
 
         redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
-        ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
+        ConnectionPool.new(timeout: pool_timeout, size: size) do
           redis_config.new_client
         end
       end

data/lib/sidekiq/scheduled.rb

@@ -115,9 +115,9 @@ module Sidekiq
       private
 
       def wait
-        @sleeper.pop(random_poll_interval)
+        @sleeper.pop(timeout: random_poll_interval)
       rescue Timeout::Error
-        # expected
+        # TODO move to exception: false
       rescue => ex
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
@@ -155,7 +155,7 @@ module Sidekiq
         else
           # With 10+ processes, we should have enough randomness to get decent polling
           # across the entire timespan
-          interval * rand
+          interval * rand * 2
         end
       end
 
@@ -223,7 +223,7 @@ module Sidekiq
         total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
-        @sleeper.pop(total)
+        @sleeper.pop(timeout: total)
       rescue Timeout::Error
       ensure
         # periodically clean out the `processes` set in Redis which can collect

data/lib/sidekiq/testing.rb

@@ -83,11 +83,11 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   module TestingClient
-    def atomic_push(conn, payloads)
+    private def atomic_push(conn, payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
-          job["enqueued_at"] = Time.now.to_f unless job["at"]
+          job["enqueued_at"] = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) unless job["at"]
           Queues.push(job["queue"], job["class"], job)
         end
         true
@@ -329,6 +329,6 @@ module Sidekiq
   end
 end
 
-if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING # rubocop:disable Style/GlobalVars
   warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end

data/lib/sidekiq/version.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.3.10"
-  MAJOR = 7
+  VERSION = "8.0.10"
+  MAJOR = 8
 
   def self.gem_version
     Gem::Version.new(VERSION)

data/lib/sidekiq/web/action.rb

@@ -1,115 +1,178 @@
 # frozen_string_literal: true
 
+require "erb"
+
 module Sidekiq
-  class WebAction
-    RACK_SESSION = "rack.session"
+  class Web
+    ##
+    # These instance methods are available to all executing ERB
+    # templates.
+    class Action
+      attr_accessor :env, :block
+
+      def initialize(env, block)
+        @_erb = false
+        @env = env
+        @block = block
+      end
 
-    attr_accessor :env, :block, :type
+      def config
+        env[:web_config]
+      end
 
-    def settings
-      Web.settings
-    end
+      def request
+        @request ||= ::Rack::Request.new(env)
+      end
 
-    def request
-      @request ||= ::Rack::Request.new(env)
-    end
+      def halt(res)
+        throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
+      end
 
-    def halt(res)
-      throw :halt, [res, {Rack::CONTENT_TYPE => "text/plain"}, [res.to_s]]
-    end
+      # external redirect
+      def redirect_to(url)
+        throw :halt, [302, {"location" => url}, []]
+      end
 
-    def redirect(location)
-      throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
-    end
+      def header(key, value)
+        env["response_headers"][key] = value.to_s
+      end
 
-    def reload_page
-      current_location = request.referer.gsub(request.base_url, "")
-      redirect current_location
-    end
+      # internal redirect
+      def redirect(location)
+        throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
+      end
 
-    # deprecated, will warn in 8.0
-    def params
-      indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
+      def reload_page
+        current_location = request.referer.gsub(request.base_url, "")
+        redirect current_location
+      end
 
-      indifferent_hash.merge! request.params
-      route_params.each { |k, v| indifferent_hash[k.to_s] = v }
+      # stuff after ? or form input
+      # uses String keys, no Symbols!
+      def url_params(key)
+        warn { "URL parameter `#{key}` should be accessed via String, not Symbol (at #{caller(3..3).first})" } if key.is_a?(Symbol)
+        request.params[key.to_s]
+      end
 
-      indifferent_hash
-    end
+      # variables embedded in path, `/metrics/:name`
+      # uses Symbol keys, no Strings!
+      def route_params(key)
+        warn { "Route parameter `#{key}` should be accessed via Symbol, not String (at #{caller(3..3).first})" } if key.is_a?(String)
+        env["rack.route_params"][key.to_sym]
+      end
 
-    # Use like `url_params("page")` within your action blocks
-    def url_params(key)
-      request.params[key]
-    end
+      def params
+        warn { "Direct access to Rack parameters is discouraged, use `url_params` or `route_params` (at #{caller(3..3).first})" }
+        request.params
+      end
+
+      def session
+        env["rack.session"] || fail(<<~EOM)
+          Sidekiq::Web needs a valid Rack session. If this is a Rails app, make
+          sure you mount Sidekiq::Web *inside* your application routes:
+
+
+          Rails.application.routes.draw do
+            mount Sidekiq::Web => "/sidekiq"
+            ....
+          end
+
+
+          If this is a Rails app in API mode, you need to enable sessions.
+
+            https://guides.rubyonrails.org/api_app.html#using-session-middlewares
 
-    # Use like `route_params(:name)` within your action blocks
-    # key is required in 8.0, nil is only used for backwards compatibility
-    def route_params(key = nil)
-      if key
-        env[WebRouter::ROUTE_PARAMS][key]
-      else
-        env[WebRouter::ROUTE_PARAMS]
+          If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+            # first, use IRB to create a shared secret key for sessions and commit it
+            require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+            # now use the secret with a session cookie middleware
+            use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+            run Sidekiq::Web
+
+        EOM
       end
-    end
 
-    def session
-      env[RACK_SESSION]
-    end
+      def logger
+        Sidekiq.logger
+      end
 
-    def erb(content, options = {})
-      if content.is_a? Symbol
-        unless respond_to?(:"_erb_#{content}")
-          views = options[:views] || Web.settings.views
-          filename = "#{views}/#{content}.erb"
-          src = ERB.new(File.read(filename)).src
-
-          # Need to use lineno less by 1 because erb generates a
-          # comment before the source code.
-          WebAction.class_eval <<-RUBY, filename, -1 # standard:disable Style/EvalWithLocation
-            def _erb_#{content}
-              #{src}
-            end
-          RUBY
-        end
+      # flash { "Some message to show on redirect" }
+      def flash
+        msg = yield
+        logger.info msg
+        session[:skq_flash] = msg
       end
 
-      if @_erb
-        _erb(content, options[:locals])
-      else
-        @_erb = true
-        content = _erb(content, options[:locals])
+      def flash?
+        session&.[](:skq_flash)
+      end
 
-        _render { content }
+      def get_flash
+        @flash ||= session.delete(:skq_flash)
       end
-    end
 
-    def render(engine, content, options = {})
-      raise "Only erb templates are supported" if engine != :erb
+      def erb(content, options = {})
+        if content.is_a? Symbol
+          unless respond_to?(:"_erb_#{content}")
+            views = options[:views] || Web.views
+            filename = "#{views}/#{content}.erb"
+            src = ERB.new(File.read(filename)).src
+
+            # Need to use lineno less by 1 because erb generates a
+            # comment before the source code.
+            Action.class_eval <<-RUBY, filename, -1 # standard:disable Style/EvalWithLocation
+              def _erb_#{content}
+                #{src}
+              end
+            RUBY
+          end
+        end
 
-      erb(content, options)
-    end
+        if @_erb
+          _erb(content, options[:locals])
+        else
+          @_erb = true
+          content = _erb(content, options[:locals])
 
-    def json(payload)
-      [200, {Rack::CONTENT_TYPE => "application/json", Rack::CACHE_CONTROL => "private, no-store"}, [Sidekiq.dump_json(payload)]]
-    end
+          _render { content }
+        end
+      end
 
-    def initialize(env, block)
-      @_erb = false
-      @env = env
-      @block = block
-      @files ||= {}
-    end
+      def render(engine, content, options = {})
+        raise "Only erb templates are supported" if engine != :erb
+
+        erb(content, options)
+      end
+
+      def json(payload)
+        [200,
+          {"content-type" => "application/json", "cache-control" => "private, no-store"},
+          [Sidekiq.dump_json(payload)]]
+      end
+
+      private
 
-    private
+      def warn
+        Sidekiq.logger.warn yield
+      end
 
-    def _erb(file, locals)
-      locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
+      def _erb(file, locals)
+        locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
 
-      if file.is_a?(String)
-        ERB.new(file).result(binding)
-      else
-        send(:"_erb_#{file}")
+        if file.is_a?(String)
+          ERB.new(file).result(binding)
+        else
+          send(:"_erb_#{file}")
+        end
       end
+
+      class_eval <<-RUBY, ::Sidekiq::Web::LAYOUT, -1 # standard:disable Style/EvalWithLocation
+        def _render
+          #{ERB.new(File.read(::Sidekiq::Web::LAYOUT)).src}
+        end
+      RUBY
     end
   end
 end