sidekiq 2.15.2 → 2.16.0

data/lib/sidekiq/core_ext.rb

@@ -53,6 +53,7 @@ end
 
 begin
   require 'active_support/core_ext/hash/keys'
+  require 'active_support/core_ext/hash/deep_merge'
 rescue LoadError
   class Hash
     def stringify_keys
@@ -68,6 +69,22 @@ rescue LoadError
       end
       self
     end if !{}.respond_to?(:symbolize_keys)
+
+    def deep_merge(other_hash, &block)
+      dup.deep_merge!(other_hash, &block)
+    end if !{}.respond_to?(:deep_merge)
+
+    def deep_merge!(other_hash, &block)
+      other_hash.each_pair do |k,v|
+        tv = self[k]
+        if tv.is_a?(Hash) && v.is_a?(Hash)
+          self[k] = tv.deep_merge(v, &block)
+        else
+          self[k] = block && tv ? block.call(k, tv, v) : v
+        end
+      end
+      self
+    end if !{}.respond_to?(:deep_merge!)
   end
 end
 

data/lib/sidekiq/middleware/chain.rb

@@ -64,6 +64,10 @@ module Sidekiq
       include Enumerable
       attr_reader :entries
 
+      def initialize_copy(copy)
+        copy.instance_variable_set(:@entries, entries.dup)
+      end
+
       def each(&block)
         entries.each(&block)
       end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -46,7 +46,7 @@ module Sidekiq
       #
       #   Sidekiq.configure_server do |config|
       #     config.server_middleware do |chain|
-      #       chain.add Middleware::Server::RetryJobs, {:max_retries => 7}
+      #       chain.add Middleware::Server::RetryJobs, :max_retries => 7
       #     end
       #   end
       class RetryJobs
@@ -106,6 +106,8 @@ module Sidekiq
           raise e
         end
 
+        private
+
         def retries_exhausted(worker, msg)
           logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
           if worker.respond_to?(:retries_exhausted)
@@ -140,7 +142,7 @@ module Sidekiq
           begin
             worker.sidekiq_retry_in_block.call(count)
           rescue Exception => e
-            logger.error { "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default: #{e.message}"}
+            handle_exception(e, { :context => "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
             nil
           end
         end

data/lib/sidekiq/redis_connection.rb

@@ -6,22 +6,28 @@ module Sidekiq
     class << self
 
       def create(options={})
-        url = options[:url] || determine_redis_provider || 'redis://localhost:6379/0'
+        url = options[:url] || determine_redis_provider
+        if url
+          options[:url] = url
+        end
+        
         # need a connection for Fetcher and Retry
         size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 2) : 5)
         pool_timeout = options[:pool_timeout] || 1
 
-        log_info(url, options)
+        log_info(options)
 
         ConnectionPool.new(:timeout => pool_timeout, :size => size) do
-          build_client(url, options[:namespace], options[:driver] || 'ruby', options[:network_timeout])
+          build_client(options)
         end
       end
 
       private
 
-      def build_client(url, namespace, driver, network_timeout)
-        client = Redis.new client_opts(url, driver, network_timeout)
+      def build_client(options)
+        namespace = options[:namespace]
+
+        client = Redis.new client_opts(options)
         if namespace
           require 'redis/namespace'
           Redis::Namespace.new(namespace, :redis => client)
@@ -30,21 +36,27 @@ module Sidekiq
         end
       end
 
-      def client_opts(url, driver, timeout)
-        if timeout
-          { :url => url, :driver => driver, :timeout => timeout }
-        else
-          { :url => url, :driver => driver }
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
         end
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:driver] = opts[:driver] || 'ruby'
+
+        opts
       end
 
-      def log_info(url, options)
-        opts = options.dup
-        opts.delete(:url)
+      def log_info(options)
         if Sidekiq.server?
-          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} using #{url} with options #{opts}")
+          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{options}")
         else
-          Sidekiq.logger.info("#{Sidekiq::NAME} client using #{url} with options #{opts}")
+          Sidekiq.logger.info("#{Sidekiq::NAME} client with redis options #{options}")
         end
       end
 

data/lib/sidekiq/tasks/sidekiq.rake

@@ -0,0 +1,92 @@
+namespace :load do
+  task :defaults do
+
+    # If you need a special boot commands
+    # 
+    # set :sidekiq_cmd,           ->{ "bundle exec sidekiq"  }
+    # set :sidekiqctl_cmd,        ->{ "bundle exec sidekiqctl" }
+    set :sidekiq_cmd,           ->{  }
+    set :sidekiqctl_cmd,        ->{  }
+
+    # must be relative to Rails.root. If this changes, you'll need to manually
+    # stop the existing sidekiq process.
+    set :sidekiq_pid,             ->{ "tmp/sidekiq.pid" }
+
+    # "-d -i INT -P PATH" are added automatically.
+    set :sidekiq_options,       ->{ "-e #{fetch(:rails_env, 'production')} -C #{current_path}/config/sidekiq.yml -L #{current_path}/log/sidekiq.log" }
+
+    set :sidekiq_timeout,       ->{ 10 }
+    set :sidekiq_role,            ->{ :app }
+    set :sidekiq_processes,   ->{ 1 }
+  end
+end
+
+namespace :sidekiq do
+  def for_each_process(&block)
+    fetch(:sidekiq_processes).times do |idx|
+      yield((idx == 0 ? "#{fetch(:sidekiq_pid)}" : "#{fetch(:sidekiq_pid)}-#{idx}"), idx)
+    end
+  end
+
+  desc "Quiet sidekiq (stop accepting new work)"
+  task :quiet do
+    on roles fetch(:sidekiq_role) do
+      within current_path do
+        for_each_process do |pid_file, idx|
+          if test "[ -f #{current_path}/#{pid_file} ]"
+            if fetch(:sidekiqctl_cmd)
+              execute fetch(:sidekiqctl_cmd), 'quiet', "#{current_path}/#{pid_file}"
+            else
+              execute :bundle, :exec, :sidekiqctl, 'quiet', "#{current_path}/#{pid_file}"
+            end
+          end
+        end
+      end
+    end
+  end
+
+  desc "Stop sidekiq"
+  task :stop do
+    on roles fetch(:sidekiq_role) do
+      within current_path do
+        for_each_process do |pid_file, idx|
+          if test "[ -f #{current_path}/#{pid_file} ]"
+            if fetch(:sidekiqctl_cmd)
+              execute fetch(:sidekiqctl_cmd), 'stop', "#{current_path}/#{pid_file}", fetch(:sidekiq_timeout)
+            else
+              execute :bundle, :exec, :sidekiqctl, 'stop', "#{current_path}/#{pid_file}", fetch(:sidekiq_timeout)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  desc "Start sidekiq"
+  task :start do
+    on roles fetch(:sidekiq_role) do
+      rails_env = fetch(:rails_env, "production")
+      within current_path do
+        for_each_process do |pid_file, idx|
+          if fetch(:sidekiq_cmd)
+            execute fetch(:sidekiq_cmd), "-d -i #{idx} -P #{pid_file} #{fetch(:sidekiq_options)}" 
+          else
+            execute :bundle, :exec, :sidekiq, "-d -i #{idx} -P #{pid_file} #{fetch(:sidekiq_options)}" 
+          end
+        end
+      end
+    end
+  end
+
+  desc "Restart sidekiq"
+  task :restart do
+    invoke 'sidekiq:stop'
+    invoke 'sidekiq:start'
+  end
+
+  after 'deploy:starting',  'sidekiq:quiet'
+  after 'deploy:updated',   'sidekiq:stop'
+  after 'deploy:reverted',  'sidekiq:stop'
+  after 'deploy:published', 'sidekiq:start'
+  
+end

data/lib/sidekiq/testing.rb

@@ -54,24 +54,22 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   class Client
-    class << self
-      alias_method :raw_push_real, :raw_push
+    alias_method :raw_push_real, :raw_push
 
-      def raw_push(payloads)
-        if Sidekiq::Testing.fake?
-          payloads.each do |job|
-            job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
-          end
-          true
-        elsif Sidekiq::Testing.inline?
-          payloads.each do |item|
-            marshalled = Sidekiq.load_json(Sidekiq.dump_json(item))
-            marshalled['class'].constantize.new.perform(*marshalled['args'])
-          end
-          true
-        else
-          raw_push_real(payloads)
+    def raw_push(payloads)
+      if Sidekiq::Testing.fake?
+        payloads.each do |job|
+          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+        end
+        true
+      elsif Sidekiq::Testing.inline?
+        payloads.each do |item|
+          marshalled = Sidekiq.load_json(Sidekiq.dump_json(item))
+          marshalled['class'].constantize.new.perform(*marshalled['args'])
         end
+        true
+      else
+        raw_push_real(payloads)
       end
     end
   end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.15.2"
+  VERSION = "2.16.0"
 end

data/lib/sidekiq/web.rb

@@ -14,7 +14,7 @@ module Sidekiq
     set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
     set :public_folder, Proc.new { "#{root}/assets" }
     set :views, Proc.new { "#{root}/views" }
-    set :locales, Proc.new { "#{root}/locales" }
+    set :locales, ["#{root}/locales"]
 
     helpers WebHelpers
 

data/lib/sidekiq/web_helpers.rb

@@ -5,8 +5,13 @@ module Sidekiq
   module WebHelpers
     def strings
       @@strings ||= begin
-        Dir["#{settings.locales}/*.yml"].inject({}) do |memo, file|
-          memo.merge(YAML.load(File.open(file)))
+        # Allow sidekiq-web extensions to add locale paths
+        # so extensions can be localized
+        settings.locales.each_with_object({}) do |path,global|
+          Dir["#{path}/*.yml"].each_with_object(global) do |file,hash|
+            strs = YAML.load(File.open(file))
+            hash.deep_merge!(strs)
+          end
         end
       end
     end
@@ -72,6 +77,10 @@ module Sidekiq
       Sidekiq.redis { |conn| conn.client.location }
     end
 
+    def redis_connection
+      Sidekiq.redis { |conn| conn.client.id }
+    end
+
     def namespace
       @@ns ||= Sidekiq.redis {|conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
     end
@@ -101,6 +110,16 @@ module Sidekiq
       [score.to_f, jid]
     end
 
+    SAFE_QPARAMS = %w(page poll)
+
+    # Merge options with current params, filter safe params, and stringify to query string
+    def qparams(options)
+      options = options.stringify_keys
+      params.merge(options).map { |key, value|
+        SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
+      }.join("&")
+    end
+
     def truncate(text, truncate_after_chars = 2000)
       truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
     end

data/test/fixtures/en.yml

@@ -0,0 +1,2 @@
+en:
+  translated_text: 'Changed text from add locals'

data/test/test_client.rb

@@ -46,7 +46,30 @@ class TestClient < Sidekiq::Test
       assert_raises ArgumentError do
         Sidekiq::Client.push('queue' => 'foo', 'class' => MyWorker, 'args' => 1)
       end
+    end
+
+    describe 'as instance' do
+      it 'can push' do
+        @redis.expect :lpush, 1, ['queue:default', Array]
+        client = Sidekiq::Client.new
+        jid = client.push('class' => 'Blah', 'args' => [1,2,3])
+        assert_equal 24, jid.size
+      end
 
+      it 'allows local middleware modification' do
+        @redis.expect :lpush, 1, ['queue:default', Array]
+        $called = false
+        mware = Class.new { def call(worker_klass,msg,q); $called = true; msg;end }
+        client = Sidekiq::Client.new
+        client.middleware do |chain|
+          chain.add mware
+        end
+        client.push('class' => 'Blah', 'args' => [1,2,3])
+
+        assert $called
+        assert client.middleware.exists?(mware)
+        refute Sidekiq.client_middleware.exists?(mware)
+      end
     end
 
     it 'pushes messages to redis' do
@@ -136,12 +159,7 @@ class TestClient < Sidekiq::Test
 
     it 'retrieves queues' do
       @redis.expect :smembers, ['bob'], ['queues']
-      assert_equal ['bob'], Sidekiq::Client.registered_queues
-    end
-
-    it 'retrieves workers' do
-      @redis.expect :smembers, ['bob'], ['workers']
-      assert_equal ['bob'], Sidekiq::Client.registered_workers
+      assert_equal ['bob'], Sidekiq::Queue.all.map(&:name)
     end
   end
 
@@ -201,11 +219,11 @@ class TestClient < Sidekiq::Test
 
   describe 'item normalization' do
     it 'defaults retry to true' do
-      assert_equal true, Sidekiq::Client.send(:normalize_item, 'class' => QueuedWorker, 'args' => [])['retry']
+      assert_equal true, Sidekiq::Client.new.send(:normalize_item, 'class' => QueuedWorker, 'args' => [])['retry']
     end
 
     it "does not normalize numeric retry's" do
-      assert_equal 2, Sidekiq::Client.send(:normalize_item, 'class' => CWorker, 'args' => [])['retry']
+      assert_equal 2, Sidekiq::Client.new.send(:normalize_item, 'class' => CWorker, 'args' => [])['retry']
     end
   end
 end

data/test/test_extensions.rb

@@ -22,18 +22,18 @@ class TestExtensions < Sidekiq::Test
     end
 
     it 'allows delayed execution of ActiveRecord class methods' do
-      assert_equal [], Sidekiq::Client.registered_queues
+      assert_equal [], Sidekiq::Queue.all.map(&:name)
       assert_equal 0, Sidekiq.redis {|c| c.llen('queue:default') }
       MyModel.delay.long_class_method
-      assert_equal ['default'], Sidekiq::Client.registered_queues
+      assert_equal ['default'], Sidekiq::Queue.all.map(&:name)
       assert_equal 1, Sidekiq.redis {|c| c.llen('queue:default') }
     end
 
     it 'uses and stringifies specified options' do
-      assert_equal [], Sidekiq::Client.registered_queues
+      assert_equal [], Sidekiq::Queue.all.map(&:name)
       assert_equal 0, Sidekiq.redis {|c| c.llen('queue:notdefault') }
       MyModel.delay(queue: :notdefault).long_class_method
-      assert_equal ['notdefault'], Sidekiq::Client.registered_queues
+      assert_equal ['notdefault'], Sidekiq::Queue.all.map(&:name)
       assert_equal 1, Sidekiq.redis {|c| c.llen('queue:notdefault') }
     end
 
@@ -56,10 +56,10 @@ class TestExtensions < Sidekiq::Test
     end
 
     it 'allows delayed delivery of ActionMailer mails' do
-      assert_equal [], Sidekiq::Client.registered_queues
+      assert_equal [], Sidekiq::Queue.all.map(&:name)
       assert_equal 0, Sidekiq.redis {|c| c.llen('queue:default') }
       UserMailer.delay.greetings(1, 2)
-      assert_equal ['default'], Sidekiq::Client.registered_queues
+      assert_equal ['default'], Sidekiq::Queue.all.map(&:name)
       assert_equal 1, Sidekiq.redis {|c| c.llen('queue:default') }
     end
 

data/test/test_redis_connection.rb

@@ -39,6 +39,20 @@ class TestRedisConnection < Sidekiq::Test
       end
     end
 
+    describe "socket path" do
+      it "uses a given :path" do
+        pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock")
+        assert_equal "unix", pool.checkout.client.scheme
+        assert_equal "redis:///var/run/redis.sock/0", pool.checkout.client.id
+      end
+
+      it "uses a given :path and :db" do
+        pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock", :db => 8)
+        assert_equal "unix", pool.checkout.client.scheme
+        assert_equal "redis:///var/run/redis.sock/8", pool.checkout.client.id
+      end
+    end
+
     describe "pool_timeout" do
       it "uses a given :timeout over the default of 1" do
         pool = Sidekiq::RedisConnection.create(:pool_timeout => 5)