sidekiq 2.15.2 → 2.16.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 08f16a145d63e3e2a04ec513885c855f1a0b12d7
+  data.tar.gz: 5596a8f758154a052f1b0e2eedfbe837bc801f2e
+SHA512:
+  metadata.gz: 5867e076904084e5b63c9f7a3dbe9358173405b48c3903954d7cf75129d2afe7d92aac884da37fa492d4542e82233d867d9927c3467e4cd11496abc123a965cc
+  data.tar.gz: fe962090ddb489623737ccc89279f6cd2bf9e36d53dadcc9a7fdd9f41bca067e1743d04a98ce5afe49c83f3e467819695221ac8bd5338bbb5b2f89c9294d6afe

data/Changes.md

@@ -1,3 +1,17 @@
+2.16.0
+-----------
+
+- Deprecate `Sidekiq::Client.registered_workers` and `Sidekiq::Client.registered_queues`
+- Refactor Sidekiq::Client to be instance-based [#1279]
+- Pass all Redis options to the Redis driver so Unix sockets
+  can be fully configured. [#1270, salimane]
+- Allow sidekiq-web extensions to add locale paths so extensions
+  can be localized. [#1261, ondrejbartas]
+- Capistrano 3 support [#1254, phallstrom]
+- Use Ruby's `resolv-replace` to enable pure Ruby DNS lookups.
+  This ensures that any DNS resolution that takes place in worker
+  threads won't lock up the entire VM on MRI. [#1258]
+
 2.15.2
 -----------
 

data/Contributing.md

@@ -13,9 +13,7 @@ If you're interested in helping or contributing to Sidekiq, here's
 some known areas which need improvement:
 
 * The Web UI and sidekiq.org website could always use more polish.  If you have an eye for design and an idea for improvement, please open an issue and let us know.
-* The Sidekiq API has a serious issue: deleting elements and iterating
-  at the same time is broken, see #866, #1060.
-* Make normal testing and inline testing dynamic: #1053
+* The [Sidekiq issue tracker](https://github.com/mperham/sidekiq/issues) usually has a few ideas
 * Your brilliant idea which I haven't listed!
 
 It's always best to open an issue before investing a lot of time into a

data/Pro-Changes.md

@@ -3,6 +3,13 @@ Sidekiq Pro Changelog
 
 Please see http://sidekiq.org/pro for more details and how to buy.
 
+1.2.5
+-----------
+
+- Convert Batch UI to use Sidekiq 2.16's support for extension
+  localization.
+- Pro now requires Sidekiq 2.16.0
+
 1.2.4
 -----------
 

data/lib/sidekiq.rb

@@ -78,7 +78,7 @@ module Sidekiq
   end
 
   def self.client_middleware
-    @client_chain ||= Client.default_middleware
+    @client_chain ||= Middleware::Chain.new
     yield @client_chain if block_given?
     @client_chain
   end

data/lib/sidekiq/api.rb

@@ -257,6 +257,7 @@ module Sidekiq
 
     def initialize(name)
       @zset = name
+      @_size = size
     end
 
     def size
@@ -270,14 +271,14 @@ module Sidekiq
     end
 
     def each(&block)
-      initial_size = size
-      deleted_size = 0
+      initial_size = @_size
+      offset_size = 0
       page = -1
       page_size = 50
 
       loop do
-        range_start = page * page_size + deleted_size
-        range_end   = page * page_size + deleted_size + (page_size - 1)
+        range_start = page * page_size + offset_size
+        range_end   = page * page_size + offset_size + (page_size - 1)
         elements = Sidekiq.redis do |conn|
           conn.zrange @zset, range_start, range_end, :with_scores => true
         end
@@ -286,7 +287,7 @@ module Sidekiq
         elements.each do |element, score|
           block.call SortedEntry.new(self, score, element)
         end
-        deleted_size = initial_size - size
+        offset_size = initial_size - @_size
       end
     end
 
@@ -320,13 +321,21 @@ module Sidekiq
           message = Sidekiq.load_json(element)
 
           if message["jid"] == jid
-            Sidekiq.redis { |conn| conn.zrem(@zset, element) }
+            _, @_size = Sidekiq.redis do |conn|
+              conn.multi do
+                conn.zrem(@zset, element)
+                conn.zcard @zset
+              end
+            end
           end
         end
         elements_with_jid.count != 0
       else
-        count = Sidekiq.redis do |conn|
-          conn.zremrangebyscore(@zset, score, score)
+        count, @_size = Sidekiq.redis do |conn|
+          conn.multi do
+            conn.zremrangebyscore(@zset, score, score)
+            conn.zcard @zset
+          end
         end
         count != 0
       end

data/lib/sidekiq/capistrano.rb

@@ -1,54 +1,5 @@
-Capistrano::Configuration.instance.load do
-
-  _cset(:sidekiq_default_hooks) { true }
-  _cset(:sidekiq_cmd) { "#{fetch(:bundle_cmd, "bundle")} exec sidekiq" }
-  _cset(:sidekiqctl_cmd) { "#{fetch(:bundle_cmd, "bundle")} exec sidekiqctl" }
-  _cset(:sidekiq_timeout)   { 10 }
-  _cset(:sidekiq_role)      { :app }
-  _cset(:sidekiq_pid)       { "#{current_path}/tmp/pids/sidekiq.pid" }
-  _cset(:sidekiq_processes) { 1 }
-
-  if fetch(:sidekiq_default_hooks)
-    before "deploy:update_code", "sidekiq:quiet"
-    after "deploy:stop",    "sidekiq:stop"
-    after "deploy:start",   "sidekiq:start"
-    before "deploy:restart", "sidekiq:restart"
-  end
-
-  namespace :sidekiq do
-    def for_each_process(&block)
-      fetch(:sidekiq_processes).times do |idx|
-        yield((idx == 0 ? "#{fetch(:sidekiq_pid)}" : "#{fetch(:sidekiq_pid)}-#{idx}"), idx)
-      end
-    end
-
-    desc "Quiet sidekiq (stop accepting new work)"
-    task :quiet, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
-      for_each_process do |pid_file, idx|
-        run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} quiet #{pid_file} ; else echo 'Sidekiq is not running'; fi"
-      end
-    end
-
-    desc "Stop sidekiq"
-    task :stop, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
-      for_each_process do |pid_file, idx|
-        run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} stop #{pid_file} #{fetch :sidekiq_timeout} ; else echo 'Sidekiq is not running'; fi"
-      end
-    end
-
-    desc "Start sidekiq"
-    task :start, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
-      rails_env = fetch(:rails_env, "production")
-      for_each_process do |pid_file, idx|
-        run "cd #{current_path} ; nohup #{fetch(:sidekiq_cmd)} -e #{rails_env} -C #{current_path}/config/sidekiq.yml -i #{idx} -P #{pid_file} >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
-      end
-    end
-
-    desc "Restart sidekiq"
-    task :restart, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
-      stop
-      start
-    end
-
-  end
+if Gem::Version.new(Capistrano::VERSION).release >= Gem::Version.new('3.0.0')
+  load File.expand_path("../tasks/sidekiq.rake", __FILE__)
+else
+  require_relative 'capistrano2'
 end

data/lib/sidekiq/capistrano2.rb

@@ -0,0 +1,54 @@
+Capistrano::Configuration.instance.load do
+
+  _cset(:sidekiq_default_hooks) { true }
+  _cset(:sidekiq_cmd) { "#{fetch(:bundle_cmd, "bundle")} exec sidekiq" }
+  _cset(:sidekiqctl_cmd) { "#{fetch(:bundle_cmd, "bundle")} exec sidekiqctl" }
+  _cset(:sidekiq_timeout)   { 10 }
+  _cset(:sidekiq_role)      { :app }
+  _cset(:sidekiq_pid)       { "#{current_path}/tmp/pids/sidekiq.pid" }
+  _cset(:sidekiq_processes) { 1 }
+
+  if fetch(:sidekiq_default_hooks)
+    before "deploy:update_code", "sidekiq:quiet"
+    after "deploy:stop",    "sidekiq:stop"
+    after "deploy:start",   "sidekiq:start"
+    before "deploy:restart", "sidekiq:restart"
+  end
+
+  namespace :sidekiq do
+    def for_each_process(&block)
+      fetch(:sidekiq_processes).times do |idx|
+        yield((idx == 0 ? "#{fetch(:sidekiq_pid)}" : "#{fetch(:sidekiq_pid)}-#{idx}"), idx)
+      end
+    end
+
+    desc "Quiet sidekiq (stop accepting new work)"
+    task :quiet, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      for_each_process do |pid_file, idx|
+        run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} quiet #{pid_file} ; else echo 'Sidekiq is not running'; fi"
+      end
+    end
+
+    desc "Stop sidekiq"
+    task :stop, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      for_each_process do |pid_file, idx|
+        run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} stop #{pid_file} #{fetch :sidekiq_timeout} ; else echo 'Sidekiq is not running'; fi"
+      end
+    end
+
+    desc "Start sidekiq"
+    task :start, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      rails_env = fetch(:rails_env, "production")
+      for_each_process do |pid_file, idx|
+        run "cd #{current_path} ; nohup #{fetch(:sidekiq_cmd)} -e #{rails_env} -C #{current_path}/config/sidekiq.yml -i #{idx} -P #{pid_file} >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
+      end
+    end
+
+    desc "Restart sidekiq"
+    task :restart, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
+      stop
+      start
+    end
+
+  end
+end

data/lib/sidekiq/cli.rb

@@ -1,5 +1,6 @@
 $stdout.sync = true
 
+require 'resolv-replace'
 require 'yaml'
 require 'singleton'
 require 'optparse'

data/lib/sidekiq/client.rb

@@ -1,73 +1,104 @@
 require 'securerandom'
-
 require 'sidekiq/middleware/chain'
 
 module Sidekiq
   class Client
-    class << self
 
-      def default_middleware
-        Middleware::Chain.new do
-        end
+    ##
+    # Define client-side middleware:
+    #
+    #   client = Sidekiq::Client.new
+    #   client.middleware do |chain|
+    #     chain.use MyClientMiddleware
+    #   end
+    #   client.push('class' => 'SomeWorker', 'args' => [1,2,3])
+    #
+    # All client instances default to the globally-defined
+    # Sidekiq.client_middleware but you can change as necessary.
+    #
+    def middleware(&block)
+      @chain ||= Sidekiq.client_middleware
+      if block_given?
+        @chain = @chain.dup
+        yield @chain
       end
+      @chain
+    end
+
+    ##
+    # The main method used to push a job to Redis.  Accepts a number of options:
+    #
+    #   queue - the named queue to use, default 'default'
+    #   class - the worker class to call, required
+    #   args - an array of simple arguments to the perform method, must be JSON-serializable
+    #   retry - whether to retry this job if it fails, true or false, default true
+    #   backtrace - whether to save any error backtrace, default false
+    #
+    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # symbols in 'args' will be converted to strings.
+    #
+    # Returns nil if not pushed to Redis or a unique Job ID if pushed.
+    #
+    # Example:
+    #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #
+    def push(item)
+      normed = normalize_item(item)
+      payload = process_single(item['class'], normed)
+
+      pushed = false
+      pushed = raw_push([payload]) if payload
+      pushed ? payload['jid'] : nil
+    end
+
+    ##
+    # Push a large number of jobs to Redis.  In practice this method is only
+    # useful if you are pushing tens of thousands of jobs or more.  This method
+    # basically cuts down on the redis round trip latency.
+    #
+    # Takes the same arguments as #push except that args is expected to be
+    # an Array of Arrays.  All other keys are duplicated for each job.  Each job
+    # is run through the client middleware pipeline and each job gets its own Job ID
+    # as normal.
+    #
+    # Returns the number of jobs pushed or nil if the pushed failed.  The number of jobs
+    # pushed can be less than the number given if the middleware stopped processing for one
+    # or more jobs.
+    def push_bulk(items)
+      normed = normalize_item(items)
+      payloads = items['args'].map do |args|
+        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !args.is_a?(Array)
+        process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f))
+      end.compact
+
+      pushed = false
+      pushed = raw_push(payloads) if !payloads.empty?
+      pushed ? payloads.size : nil
+    end
 
+    class << self
+      def default
+        @default ||= new
+      end
+
+      # deprecated
       def registered_workers
+        puts "registered_workers is deprecated, please use Sidekiq::Workers.new"
         Sidekiq.redis { |x| x.smembers('workers') }
       end
 
+      # deprecated
       def registered_queues
-        Sidekiq.redis { |x| x.smembers('queues') }
+        puts "registered_queues is deprecated, please use Sidekiq::Queue.all"
+        Sidekiq::Queue.all.map(&:name)
       end
 
-      ##
-      # The main method used to push a job to Redis.  Accepts a number of options:
-      #
-      #   queue - the named queue to use, default 'default'
-      #   class - the worker class to call, required
-      #   args - an array of simple arguments to the perform method, must be JSON-serializable
-      #   retry - whether to retry this job if it fails, true or false, default true
-      #   backtrace - whether to save any error backtrace, default false
-      #
-      # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
-      # symbols in 'args' will be converted to strings.
-      #
-      # Returns nil if not pushed to Redis or a unique Job ID if pushed.
-      #
-      # Example:
-      #   Sidekiq::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
-      #
       def push(item)
-        normed = normalize_item(item)
-        payload = process_single(item['class'], normed)
-
-        pushed = false
-        pushed = raw_push([payload]) if payload
-        pushed ? payload['jid'] : nil
+        default.push(item)
       end
 
-      ##
-      # Push a large number of jobs to Redis.  In practice this method is only
-      # useful if you are pushing tens of thousands of jobs or more.  This method
-      # basically cuts down on the redis round trip latency.
-      #
-      # Takes the same arguments as Client.push except that args is expected to be
-      # an Array of Arrays.  All other keys are duplicated for each job.  Each job
-      # is run through the client middleware pipeline and each job gets its own Job ID
-      # as normal.
-      #
-      # Returns the number of jobs pushed or nil if the pushed failed.  The number of jobs
-      # pushed can be less than the number given if the middleware stopped processing for one
-      # or more jobs.
       def push_bulk(items)
-        normed = normalize_item(items)
-        payloads = items['args'].map do |args|
-          raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !args.is_a?(Array)
-          process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f))
-        end.compact
-
-        pushed = false
-        pushed = raw_push(payloads) if !payloads.empty?
-        pushed ? payloads.size : nil
+        default.push_bulk(items)
       end
 
       # Resque compatibility helpers.  Note all helpers
@@ -109,56 +140,56 @@ module Sidekiq
       def enqueue_in(interval, klass, *args)
         klass.perform_in(interval, *args)
       end
+    end
+
+    private
 
-      private
-
-      def raw_push(payloads)
-        pushed = false
-        Sidekiq.redis do |conn|
-          if payloads.first['at']
-            pushed = conn.zadd('schedule', payloads.map do |hash|
-              at = hash.delete('at').to_s
-              [at, Sidekiq.dump_json(hash)]
-            end)
-          else
-            q = payloads.first['queue']
-            to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
-            _, pushed = conn.multi do
-              conn.sadd('queues', q)
-              conn.lpush("queue:#{q}", to_push)
-            end
+    def raw_push(payloads)
+      pushed = false
+      Sidekiq.redis do |conn|
+        if payloads.first['at']
+          pushed = conn.zadd('schedule', payloads.map do |hash|
+            at = hash.delete('at').to_s
+            [at, Sidekiq.dump_json(hash)]
+          end)
+        else
+          q = payloads.first['queue']
+          to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
+          _, pushed = conn.multi do
+            conn.sadd('queues', q)
+            conn.lpush("queue:#{q}", to_push)
           end
         end
-        pushed
       end
+      pushed
+    end
 
-      def process_single(worker_class, item)
-        queue = item['queue']
+    def process_single(worker_class, item)
+      queue = item['queue']
 
-        Sidekiq.client_middleware.invoke(worker_class, item, queue) do
-          item
-        end
+      middleware.invoke(worker_class, item, queue) do
+        item
       end
+    end
 
-      def normalize_item(item)
-        raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
-        raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
-        raise(ArgumentError, "Message args must be an Array") unless item['args'].is_a?(Array)
-        raise(ArgumentError, "Message class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
-
-        if item['class'].is_a?(Class)
-          raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].respond_to?('get_sidekiq_options')
-          normalized_item = item['class'].get_sidekiq_options.merge(item)
-          normalized_item['class'] = normalized_item['class'].to_s
-        else
-          normalized_item = Sidekiq.default_worker_options.merge(item)
-        end
-
-        normalized_item['jid'] ||= SecureRandom.hex(12)
-        normalized_item['enqueued_at'] ||= Time.now.to_f
-        normalized_item
+    def normalize_item(item)
+      raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
+      raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
+      raise(ArgumentError, "Message args must be an Array") unless item['args'].is_a?(Array)
+      raise(ArgumentError, "Message class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
+
+      if item['class'].is_a?(Class)
+        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].respond_to?('get_sidekiq_options')
+        normalized_item = item['class'].get_sidekiq_options.merge(item)
+        normalized_item['class'] = normalized_item['class'].to_s
+      else
+        normalized_item = Sidekiq.default_worker_options.merge(item)
       end
 
+      normalized_item['jid'] ||= SecureRandom.hex(12)
+      normalized_item['enqueued_at'] ||= Time.now.to_f
+      normalized_item
     end
+
   end
 end