sidekiq 3.5.4 → 7.2.0

data/lib/sidekiq/testing.rb

@@ -1,26 +1,46 @@
-require 'securerandom'
-require 'sidekiq'
+# frozen_string_literal: true
 
-module Sidekiq
+require "securerandom"
+require "sidekiq"
 
+module Sidekiq
   class Testing
+    class TestModeAlreadySetError < RuntimeError; end
     class << self
-      attr_accessor :__test_mode
+      attr_accessor :__global_test_mode
 
+      # Calling without a block sets the global test mode, affecting
+      # all threads. Calling with a block only affects the current Thread.
       def __set_test_mode(mode)
         if block_given?
-          current_mode = self.__test_mode
+          # Reentrant testing modes will lead to a rat's nest of code which is
+          # hard to reason about. You can set the testing mode once globally and
+          # you can override that global setting once per-thread.
+          raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
+
+          self.__local_test_mode = mode
           begin
-            self.__test_mode = mode
             yield
           ensure
-            self.__test_mode = current_mode
+            self.__local_test_mode = nil
           end
         else
-          self.__test_mode = mode
+          self.__global_test_mode = mode
         end
       end
 
+      def __test_mode
+        __local_test_mode || __global_test_mode
+      end
+
+      def __local_test_mode
+        Thread.current[:__sidekiq_test_mode]
+      end
+
+      def __local_test_mode=(value)
+        Thread.current[:__sidekiq_test_mode] = value
+      end
+
       def disable!(&block)
         __set_test_mode(:disable, &block)
       end
@@ -34,23 +54,23 @@ module Sidekiq
       end
 
       def enabled?
-        self.__test_mode != :disable
+        __test_mode != :disable
       end
 
       def disabled?
-        self.__test_mode == :disable
+        __test_mode == :disable
       end
 
       def fake?
-        self.__test_mode == :fake
+        __test_mode == :fake
       end
 
       def inline?
-        self.__test_mode == :inline
+        __test_mode == :inline
       end
 
       def server_middleware
-        @server_chain ||= Middleware::Chain.new
+        @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
         yield @server_chain if block_given?
         @server_chain
       end
@@ -62,30 +82,134 @@ module Sidekiq
 
   class EmptyQueueError < RuntimeError; end
 
-  class Client
-    alias_method :raw_push_real, :raw_push
-
-    def raw_push(payloads)
+  module TestingClient
+    def atomic_push(conn, payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+          job = Sidekiq.load_json(Sidekiq.dump_json(job))
+          job["enqueued_at"] = Time.now.to_f unless job["at"]
+          Queues.push(job["queue"], job["class"], job)
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          job['jid'] ||= SecureRandom.hex(12)
-          klass = job['class'].constantize
-          klass.jobs.unshift Sidekiq.load_json(Sidekiq.dump_json(job))
-          klass.perform_one
+          klass = Object.const_get(job["class"])
+          job["id"] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
         end
         true
       else
-        raw_push_real(payloads)
+        super
+      end
+    end
+  end
+
+  Sidekiq::Client.prepend TestingClient
+
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # There are 2 data structures involved in tandem. This is due to the
+    # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
+    # to the array. Because the array was dervied from a filter of the total
+    # jobs enqueued, it appeared as though the array didn't change.
+    #
+    # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
+    # on the queue, and another with keys of the job type, so the array for
+    # HardJob.jobs is a straight reference to a real array.
+    #
+    # Queue-based hash:
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::HardJob",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Job-based hash:
+    #
+    # {
+    #   "TestTesting::HardJob"=>[
+    #     {
+    #       "class"=>"TestTesting::HardJob",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardJob.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs_by_queue[queue]
+      end
+
+      def push(queue, klass, job)
+        jobs_by_queue[queue] << job
+        jobs_by_class[klass] << job
+      end
+
+      def jobs_by_queue
+        @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def jobs_by_class
+        @jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+      alias_method :jobs_by_worker, :jobs_by_class
+
+      def delete_for(jid, queue, klass)
+        jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
+        jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
+      end
+
+      def clear_for(queue, klass)
+        jobs_by_queue[queue.to_s].clear
+        jobs_by_class[klass].clear
+      end
+
+      def clear_all
+        jobs_by_queue.clear
+        jobs_by_class.clear
       end
     end
   end
 
-  module Worker
+  module Job
     ##
     # The Sidekiq testing infrastructure overrides perform_async
     # so that it does not actually touch the network.  Instead it
@@ -99,80 +223,71 @@ module Sidekiq
     #
     #   require 'sidekiq/testing'
     #
-    #   assert_equal 0, HardWorker.jobs.size
-    #   HardWorker.perform_async(:something)
-    #   assert_equal 1, HardWorker.jobs.size
-    #   assert_equal :something, HardWorker.jobs[0]['args'][0]
-    #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #
-    # You can also clear and drain all workers' jobs:
+    #   assert_equal 0, HardJob.jobs.size
+    #   HardJob.perform_async(:something)
+    #   assert_equal 1, HardJob.jobs.size
+    #   assert_equal :something, HardJob.jobs[0]['args'][0]
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
+    # You can also clear and drain all job types:
     #
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   MyModel.delay.do_something_hard
-    #
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   Sidekiq::Worker.clear_all # or .drain_all
-    #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
+    #   Sidekiq::Job.clear_all # or .drain_all
     #
     # This can be useful to make sure jobs don't linger between tests:
     #
     #   RSpec.configure do |config|
     #     config.before(:each) do
-    #       Sidekiq::Worker.clear_all
+    #       Sidekiq::Job.clear_all
     #     end
     #   end
     #
     # or for acceptance testing, i.e. with cucumber:
     #
     #   AfterStep do
-    #     Sidekiq::Worker.drain_all
+    #     Sidekiq::Job.drain_all
     #   end
     #
     #   When I sign up as "foo@example.com"
     #   Then I should receive a welcome email to "foo@example.com"
     #
     module ClassMethods
+      # Queue for this worker
+      def queue
+        get_sidekiq_options["queue"]
+      end
 
       # Jobs queued for this worker
       def jobs
-        Worker.jobs[self]
+        Queues.jobs_by_class[to_s]
       end
 
       # Clear all jobs for this worker
       def clear
-        jobs.clear
+        Queues.clear_for(queue, to_s)
       end
 
       # Drain and run all jobs for this worker
       def drain
-        while job = jobs.shift do
-          process_job(job)
+        while jobs.any?
+          next_job = jobs.first
+          Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
+          process_job(next_job)
         end
       end
 
       # Pop out a single job and perform it
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
-        job = jobs.shift
-        process_job(job)
+        next_job = jobs.first
+        Queues.delete_for(next_job["jid"], queue, to_s)
+        process_job(next_job)
       end
 
       def process_job(job)
-        worker = new
-        worker.jid = job['jid']
-        worker.bid = job['bid'] if worker.respond_to?(:bid=)
-        Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
-          execute_job(worker, job['args'])
+        inst = new
+        inst.jid = job["jid"]
+        inst.bid = job["bid"] if inst.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
+          execute_job(inst, job["args"])
         end
       end
 
@@ -183,20 +298,37 @@ module Sidekiq
 
     class << self
       def jobs # :nodoc:
-        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+        Queues.jobs_by_queue.values.flatten
       end
 
-      # Clear all queued jobs across all workers
+      # Clear all queued jobs
       def clear_all
-        jobs.clear
+        Queues.clear_all
       end
 
-      # Drain all queued jobs across all workers
+      # Drain (execute) all queued jobs
       def drain_all
-        until jobs.values.all?(&:empty?) do
-          jobs.keys.each(&:drain)
+        while jobs.any?
+          job_classes = jobs.map { |job| job["class"] }.uniq
+
+          job_classes.each do |job_class|
+            Object.const_get(job_class).drain
+          end
         end
       end
     end
   end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
+      end
+    end
+  end
+end
+
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
+  warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.", uplevel: 1)
 end

data/lib/sidekiq/transaction_aware_client.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq/client"
+
+module Sidekiq
+  class TransactionAwareClient
+    def initialize(pool: nil, config: nil)
+      @redis_client = Client.new(pool: pool, config: config)
+    end
+
+    def push(item)
+      # pre-allocate the JID so we can return it immediately and
+      # save it to the database as part of the transaction.
+      item["jid"] ||= SecureRandom.hex(12)
+      AfterCommitEverywhere.after_commit { @redis_client.push(item) }
+      item["jid"]
+    end
+
+    ##
+    # We don't provide transactionality for push_bulk because we don't want
+    # to hold potentially hundreds of thousands of job records in memory due to
+    # a long running enqueue process.
+    def push_bulk(items)
+      @redis_client.push_bulk(items)
+    end
+  end
+end
+
+##
+# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
+module Sidekiq
+  def self.transactional_push!
+    begin
+      require "after_commit_everywhere"
+    rescue LoadError
+      raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
+    end
+
+    Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
+    true
+  end
+end

data/lib/sidekiq/version.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
 module Sidekiq
-  VERSION = "3.5.4"
+  VERSION = "7.2.0"
+  MAJOR = 7
 end

data/lib/sidekiq/web/action.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class WebAction
+    RACK_SESSION = "rack.session"
+
+    attr_accessor :env, :block, :type
+
+    def settings
+      Web.settings
+    end
+
+    def request
+      @request ||= ::Rack::Request.new(env)
+    end
+
+    def halt(res)
+      throw :halt, [res, {Rack::CONTENT_TYPE => "text/plain"}, [res.to_s]]
+    end
+
+    def redirect(location)
+      throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
+    end
+
+    def params
+      indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
+
+      indifferent_hash.merge! request.params
+      route_params.each { |k, v| indifferent_hash[k.to_s] = v }
+
+      indifferent_hash
+    end
+
+    def route_params
+      env[WebRouter::ROUTE_PARAMS]
+    end
+
+    def session
+      env[RACK_SESSION]
+    end
+
+    def erb(content, options = {})
+      if content.is_a? Symbol
+        unless respond_to?(:"_erb_#{content}")
+          src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
+          WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            def _erb_#{content}
+              #{src}
+            end
+          RUBY
+        end
+      end
+
+      if @_erb
+        _erb(content, options[:locals])
+      else
+        @_erb = true
+        content = _erb(content, options[:locals])
+
+        _render { content }
+      end
+    end
+
+    def render(engine, content, options = {})
+      raise "Only erb templates are supported" if engine != :erb
+
+      erb(content, options)
+    end
+
+    def json(payload)
+      [200, {Rack::CONTENT_TYPE => "application/json", Rack::CACHE_CONTROL => "private, no-store"}, [Sidekiq.dump_json(payload)]]
+    end
+
+    def initialize(env, block)
+      @_erb = false
+      @env = env
+      @block = block
+      @files ||= {}
+    end
+
+    private
+
+    def _erb(file, locals)
+      locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
+
+      if file.is_a?(String)
+        ERB.new(file).result(binding)
+      else
+        send(:"_erb_#{file}")
+      end
+    end
+  end
+end