sidekiq 3.0.2 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a7c0bb226bdf38ccb6a57bcf5fee433966e8f641
-  data.tar.gz: c90d47b16b78595ae858c209a89f34d4eb698ce5
+  metadata.gz: 2c396adca6a73c134f306d045e66ee17a6f462f5
+  data.tar.gz: aa1be6ef05116a539c57390f4ca6f23dd035cd74
 SHA512:
-  metadata.gz: 28d2b3599cd512c2de0a524df4b60a9381698ff1b4cdf4c14251868f3a08a8e85ef5297c6fb3bcbe6e80a6780b4767e1637798bbfb8249c4de29f22b9c2c7119
-  data.tar.gz: 81f5f2949d44afe92709e4a078e56f727cb5315a2fb9bf86a0f0379b1d8a7f315f9a9508e7ed6c2d63958048c2bdf402daf3634f87086140cd8d2e5a506e0f9c
+  metadata.gz: e5b673786330ff3a01c5f5577d194fb34feba04a579853036ba41ef6e612440745cf340a7f279da7bb82e1712466ee864d6688dca7c23e79631e94d520acdcd4
+  data.tar.gz: 33c32fbea3ea8f04a85c1134f2181c4549d630631dd1894dfb7e04ae9d872fbbcfc0ce22d79051e9ea2e7a348060eefd7cf1c6be560f8d06bb2293c79b6474f8

data/.travis.yml

@@ -6,7 +6,7 @@ rvm:
   - jruby-19mode
   - rbx
   - 2.0.0
-  - 2.1.0
+  - 2.1
 matrix:
   allow_failures:
     - rvm: rbx

data/Changes.md

@@ -1,3 +1,26 @@
+3.1.0
+-----------
+
+- New **remote control** feature: you can remotely trigger Sidekiq to quiet
+  or terminate via API, without signals.  This is most useful on JRuby
+  or Heroku which does not support the USR1 'quiet' signal.  Now you can
+  run a rake task like this at the start of your deploy to quiet your
+  set of Sidekiq processes. [#1703]
+```ruby
+namespace :sidekiq do
+  task :quiet => :environment do
+    Sidekiq::ProcessSet.new.each(&:quiet!)
+  end
+end
+```
+- The Web UI can use the API to quiet or stop all processes via the Busy page.
+- The Web UI understands and hides the `Sidekiq::Extensions::Delay*`
+  classes, instead showing `Class.method` as the Job. [#1718]
+- The poll interval is now configurable in the Web UI [madebydna, #1713]
+- Delay extensions can be removed so they don't conflict with
+  DelayedJob: put `Sidekiq.remove_delay!` in your initializer. [devaroop, #1674]
+
+
 3.0.2
 -----------
 

data/Pro-Changes.md

@@ -3,6 +3,14 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/pro](http://sidekiq.org/pro) for more details and how to buy.
 
+1.7.1
+-----------
+
+- Fix for paused queues being processed for a few seconds when starting
+  a new Sidekiq process.
+- Add a 5 sec delay when starting reliable fetch on Heroku to minimize
+  any duplicate job processing with another process shutting down.
+
 1.7.0
 -----------
 

data/lib/sidekiq/api.rb

@@ -189,11 +189,12 @@ module Sidekiq
   # removed from the queue via Job#delete.
   #
   class Job
+    KNOWN_WRAPPERS = [/\ASidekiq::Extensions::Delayed/, "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"]
     attr_reader :item
 
     def initialize(item, queue_name=nil)
       @value = item
-      @item = Sidekiq.load_json(item)
+      @item = item.is_a?(Hash) ? item : Sidekiq.load_json(item)
       @queue = queue_name || @item['queue']
     end
 
@@ -201,6 +202,32 @@ module Sidekiq
       @item['class']
     end
 
+    def display_class
+      # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
+      @klass ||= case klass
+                 when /\ASidekiq::Extensions::Delayed/
+                   (target, method, _) = YAML.load(args[0])
+                   "#{target}.#{method}"
+                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+                   args[0]
+                 else
+                   klass
+                 end
+    end
+
+    def display_args
+      # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
+      @args ||= case klass
+                when /\ASidekiq::Extensions::Delayed/
+                  (_, _, arg) = YAML.load(args[0])
+                  arg
+                when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+                  args[1..-1]
+                else
+                  args
+                end
+    end
+
     def args
       @item['args']
     end
@@ -449,18 +476,8 @@ module Sidekiq
   # right now.  Each process send a heartbeat to Redis every 5 seconds
   # so this set should be relatively accurate, barring network partitions.
   #
-  # Yields a hash of data which looks something like this:
+  # Yields a Sidekiq::Process.
   #
-  # {
-  #   'hostname' => 'app-1.example.com',
-  #   'started_at' => <process start time>,
-  #   'pid' => 12345,
-  #   'tag' => 'myapp'
-  #   'concurrency' => 25,
-  #   'queues' => ['default', 'low'],
-  #   'busy' => 10,
-  #   'beat' => <last heartbeat>,
-  # }
 
   class ProcessSet
     include Enumerable
@@ -486,7 +503,7 @@ module Sidekiq
           # in to Redis and probably died.
           (to_prune << sorted[i]; next) if info.nil?
           hash = Sidekiq.load_json(info)
-          yield hash.merge('busy' => busy.to_i, 'beat' => at_s.to_f)
+          yield Process.new(hash.merge('busy' => busy.to_i, 'beat' => at_s.to_f))
         end
       end
 
@@ -503,6 +520,53 @@ module Sidekiq
     end
   end
 
+  #
+  # Sidekiq::Process has a set of attributes which look like this:
+  #
+  # {
+  #   'hostname' => 'app-1.example.com',
+  #   'started_at' => <process start time>,
+  #   'pid' => 12345,
+  #   'tag' => 'myapp'
+  #   'concurrency' => 25,
+  #   'queues' => ['default', 'low'],
+  #   'busy' => 10,
+  #   'beat' => <last heartbeat>,
+  # }
+  class Process
+    def initialize(hash)
+      @attribs = hash
+    end
+
+    def [](key)
+      @attribs[key]
+    end
+
+    def quiet!
+      signal('USR1')
+    end
+
+    def stop!
+      signal('TERM')
+    end
+
+    private
+
+    def signal(sig)
+      key = "#{identity}-signals"
+      Sidekiq.redis do |c|
+        c.multi do
+          c.lpush(key, sig)
+          c.expire(key, 60)
+        end
+      end
+    end
+
+    def identity
+      @id ||= "#{self['hostname']}:#{self['pid']}"
+    end
+  end
+
   ##
   # Programmatic access to the current active worker set.
   #

data/lib/sidekiq/cli.rb

@@ -20,7 +20,7 @@ module Sidekiq
 
   class CLI
     include Util
-    include Singleton
+    include Singleton unless $TESTING
 
     # Used for CLI testing
     attr_accessor :code
@@ -40,16 +40,14 @@ module Sidekiq
       daemonize
       write_pid
       load_celluloid
-      boot_system
+      print_banner
     end
 
+    # Code within this method is not tested because it alters
+    # global process state irreversibly.  PRs which improve the
+    # test coverage of Sidekiq::CLI are welcomed.
     def run
-      # Print logo and banner for development
-      if environment == 'development' && $stdout.tty?
-        puts "\e[#{31}m"
-        puts Sidekiq::BANNER
-        puts "\e[0m"
-      end
+      boot_system
 
       self_read, self_write = IO.pipe
 
@@ -95,13 +93,12 @@ module Sidekiq
 
     private
 
-    def fire_event(event)
-      Sidekiq.options[:lifecycle_events][event].each do |block|
-        begin
-          block.call
-        rescue => ex
-          handle_exception(ex, { :event => event })
-        end
+    def print_banner
+      # Print logo and banner for development
+      if environment == 'development' && $stdout.tty?
+        puts "\e[#{31}m"
+        puts Sidekiq::BANNER
+        puts "\e[0m"
       end
     end
 
@@ -158,7 +155,7 @@ module Sidekiq
         files_to_reopen << file unless file.closed?
       end
 
-      Process.daemon(true, true)
+      ::Process.daemon(true, true)
 
       files_to_reopen.each do |file|
         begin
@@ -323,7 +320,7 @@ module Sidekiq
       if path = options[:pidfile]
         pidfile = File.expand_path(path)
         File.open(pidfile, 'w') do |f|
-          f.puts Process.pid
+          f.puts ::Process.pid
         end
       end
     end

data/lib/sidekiq/logging.rb

@@ -7,7 +7,7 @@ module Sidekiq
     class Pretty < Logger::Formatter
       # Provide a call() method that returns the formatted message.
       def call(severity, time, program_name, message)
-        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+        "#{time.utc.iso8601} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
       end
 
       def context

data/lib/sidekiq/manager.rb

@@ -149,12 +149,15 @@ module Sidekiq
 
     def ❤(key)
       begin
-        Sidekiq.redis do |conn|
+        _, _, msg = Sidekiq.redis do |conn|
           conn.multi do
             conn.hmset(key, 'busy', @busy.size, 'beat', Time.now.to_f)
             conn.expire(key, 60)
+            conn.rpop("#{key}-signals")
           end
         end
+
+        ::Process.kill(msg, $$) if msg
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")

data/lib/sidekiq/processor.rb

@@ -119,9 +119,6 @@ module Sidekiq
       end
     end
 
-    # Singleton classes are not clonable.
-    SINGLETON_CLASSES = [ NilClass, TrueClass, FalseClass, Symbol, Fixnum, Float, Bignum ].freeze
-
     # Deep clone the arguments passed to the worker so that if
     # the message fails, what is pushed back onto Redis hasn't
     # been mutated by the worker.

data/lib/sidekiq/rails.rb

@@ -9,6 +9,21 @@ module Sidekiq
     end
   end
 
+  # Removes the generic aliases which MAY clash with names of already
+  #  created methods by other applications. The methods `sidekiq_delay`,
+  #  `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
+  def self.remove_delay!
+    [Extensions::ActiveRecord,
+     Extensions::ActionMailer,
+     Extensions::Klass].each do |mod|
+      mod.module_eval do
+        remove_method :delay if respond_to?(:delay)
+        remove_method :delay_for if respond_to?(:delay_for)
+        remove_method :delay_until if respond_to?(:delay_until)
+      end
+    end
+  end
+
   class Rails < ::Rails::Engine
     initializer 'sidekiq' do
       Sidekiq.hook_rails!

data/lib/sidekiq/util.rb

@@ -33,5 +33,16 @@ module Sidekiq
     def identity
       @@identity ||= "#{hostname}:#{$$}"
     end
+
+    def fire_event(event)
+      Sidekiq.options[:lifecycle_events][event].each do |block|
+        begin
+          block.call
+        rescue => ex
+          handle_exception(ex, { :event => event })
+        end
+      end
+    end
+
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.0.2"
+  VERSION = "3.1.0"
 end

data/lib/sidekiq/web.rb

@@ -42,6 +42,20 @@ module Sidekiq
       erb :busy
     end
 
+    post "/busy" do
+      if params['hostname']
+        pro = Sidekiq::Process.new('hostname' => params["hostname"], 'pid' => params['pid'])
+        pro.quiet! if params[:quiet]
+        pro.stop! if params[:stop]
+      else
+        Sidekiq::ProcessSet.new.each do |pro|
+          pro.quiet! if params[:quiet]
+          pro.stop! if params[:stop]
+        end
+      end
+      redirect "#{root_path}busy"
+    end
+
     get "/queues" do
       @queues = Sidekiq::Queue.all
       erb :queues
@@ -53,7 +67,7 @@ module Sidekiq
       @name = params[:name]
       @queue = Sidekiq::Queue.new(@name)
       (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
-      @messages = @messages.map {|msg| Sidekiq.load_json(msg) }
+      @messages = @messages.map {|msg| Sidekiq::Job.new(msg, @name) }
       erb :queue
     end
 
@@ -86,12 +100,7 @@ module Sidekiq
 
       params['key'].each do |key|
         job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
-        next unless job
-        if params['retry']
-          job.retry
-        elsif params['delete']
-          job.delete
-        end
+        retry_or_delete job, params if job
       end
       redirect_with_query("#{root_path}morgue")
     end
@@ -109,13 +118,7 @@ module Sidekiq
     post "/morgue/:key" do
       halt 404 unless params['key']
       job = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
-      if job
-        if params['retry']
-          job.retry
-        elsif params['delete']
-          job.delete
-        end
-      end
+      retry_or_delete job, params if job
       redirect_with_query("#{root_path}morgue")
     end
 
@@ -139,12 +142,7 @@ module Sidekiq
 
       params['key'].each do |key|
         job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
-        next unless job
-        if params['retry']
-          job.retry
-        elsif params['delete']
-          job.delete
-        end
+        retry_or_delete job, params if job
       end
       redirect_with_query("#{root_path}retries")
     end
@@ -162,13 +160,7 @@ module Sidekiq
     post "/retries/:key" do
       halt 404 unless params['key']
       job = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      if job
-        if params['retry']
-          job.retry
-        elsif params['delete']
-          job.delete
-        end
-      end
+      retry_or_delete job, params if job
       redirect_with_query("#{root_path}retries")
     end
 
@@ -191,13 +183,7 @@ module Sidekiq
 
       params['key'].each do |key|
         job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
-        if job
-          if params['delete']
-            job.delete
-          elsif params['add_to_queue']
-            job.add_to_queue
-          end
-        end
+        delete_or_add_queue job, params if job
       end
       redirect_with_query("#{root_path}scheduled")
     end
@@ -205,13 +191,7 @@ module Sidekiq
     post "/scheduled/:key" do
       halt 404 unless params['key']
       job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      if job
-        if params['add_to_queue']
-          job.add_to_queue
-        elsif params['delete']
-          job.delete
-        end
-      end
+      delete_or_add_queue job, params if job
       redirect_with_query("#{root_path}scheduled")
     end
 
@@ -223,7 +203,7 @@ module Sidekiq
       erb :dashboard
     end
 
-    REDIS_KEYS = %w(redis_stats uptime_in_days connected_clients used_memory_human used_memory_peak_human)
+    REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
 
     get '/dashboard/stats' do
       sidekiq_stats = Sidekiq::Stats.new
@@ -245,5 +225,22 @@ module Sidekiq
       })
     end
 
+    private
+
+    def retry_or_delete job, params
+      if params['retry']
+        job.retry
+      elsif params['delete']
+        job.delete
+      end
+    end
+
+    def delete_or_add_queue job, params
+      if params['delete']
+        job.delete
+      elsif params['add_to_queue']
+        job.add_to_queue
+      end
+    end
   end
 end