sidekiq 2.14.1 → 2.15.0

data/Changes.md

@@ -1,3 +1,27 @@
+2.15.0
+-----------
+
+- The Core Sidekiq actors are now monitored.  If any crash, the
+  Sidekiq process logs the error and exits immediately.  This is to
+  help prevent "stuck" Sidekiq processes which are running but don't
+  appear to be doing any work. [#1194]
+- Sidekiq's testing behavior is now dynamic.  You can choose between
+  `inline` and `fake` behavior in your tests. See
+[Testing](wiki/Testing) for detail. [#1193]
+- The Retries table has a new column for the error message.
+- The Web UI topbar now contains the status and live poll button.
+- Orphaned worker records are now auto-vacuumed when you vist the
+  Workers page in the Web UI.
+- Sidekiq.default\_worker\_options allows you to configure default
+  options for all Sidekiq worker types.
+
+```ruby
+Sidekiq.default_worker_options = { 'queue' => 'default', 'backtrace' => true }
+```
+- Added two Sidekiq::Client class methods for compatibility with resque-scheduler:
+  `enqueue_to_in` and `enqueue_in` [#1212]
+- Upgrade Web UI to Bootstrap 3.0. [#1211, jeffboek]
+
 2.14.1
 -----------
 

data/README.md

@@ -8,7 +8,7 @@ Simple, efficient background processing for Ruby.
 
 Sidekiq uses threads to handle many jobs at the same time in the
 same process.  It does not require Rails but will integrate tightly with
-Rails 3 to make background processing dead simple.
+Rails 3/4 to make background processing dead simple.
 
 Sidekiq is compatible with Resque.  It uses the exact same
 message format as Resque so it can integrate into an existing Resque processing farm.
@@ -24,8 +24,11 @@ the same CPU and perform the same amount of work.
 Requirements
 -----------------
 
-I test on Ruby 1.9.3 and JRuby 1.7.x.  Other versions/VMs are
-untested but I will do my best to support them.  Ruby 1.8 is not supported.
+I test with the latest Ruby (2.0) and JRuby versions (1.7).  Other versions/VMs
+are untested but might work fine.
+
+The last two major Rails releases (3.2 and 4.0) are officially supported, other
+versions might work fine.
 
 Redis 2.4 or greater is required.
 

data/lib/sidekiq.rb

@@ -89,6 +89,14 @@ module Sidekiq
     @server_chain
   end
 
+  def self.default_worker_options=(hash)
+    @default_worker_options = default_worker_options.merge(hash)
+  end
+
+  def self.default_worker_options
+    @default_worker_options || { 'retry' => true, 'queue' => 'default' }
+  end
+
   def self.load_json(string)
     JSON.parse(string)
   end

data/lib/sidekiq/api.rb

@@ -260,7 +260,7 @@ module Sidekiq
 
     def schedule(timestamp, message)
       Sidekiq.redis do |conn|
-        conn.zadd(@zset, timestamp.to_s, Sidekiq.dump_json(message))
+        conn.zadd(@zset, timestamp.to_f.to_s, Sidekiq.dump_json(message))
       end
     end
 

data/lib/sidekiq/cli.rb

@@ -220,7 +220,7 @@ module Sidekiq
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
-        logger.info "  Please point sidekiq to a Rails 3 application or a Ruby file    "
+        logger.info "  Please point sidekiq to a Rails 3/4 application or a Ruby file  "
         logger.info "  to load your worker classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser

data/lib/sidekiq/client.rb

@@ -70,7 +70,8 @@ module Sidekiq
         pushed ? payloads.size : nil
       end
 
-      # Resque compatibility helpers.
+      # Resque compatibility helpers.  Note all helpers
+      # should go through Worker#client_push.
       #
       # Example usage:
       #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
@@ -88,6 +89,27 @@ module Sidekiq
         klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
       end
 
+      # Example usage:
+      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #
+      def enqueue_to_in(queue, interval, klass, *args)
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = (int < 1_000_000_000 ? now + int : int)
+
+        item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
+        item.delete('at') if ts <= now
+
+        klass.client_push(item)
+      end
+
+      # Example usage:
+      #   Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #
+      def enqueue_in(interval, klass, *args)
+        klass.perform_in(interval, *args)
+      end
+
       private
 
       def raw_push(payloads)
@@ -129,7 +151,7 @@ module Sidekiq
           normalized_item = item['class'].get_sidekiq_options.merge(item)
           normalized_item['class'] = normalized_item['class'].to_s
         else
-          normalized_item = Sidekiq::Worker::ClassMethods::DEFAULT_OPTIONS.merge(item)
+          normalized_item = Sidekiq.default_worker_options.merge(item)
         end
 
         normalized_item['jid'] ||= SecureRandom.hex(12)

data/lib/sidekiq/exception_handler.rb

@@ -1,8 +1,8 @@
 module Sidekiq
   module ExceptionHandler
 
-    def handle_exception(ex, ctxHash)
-      Sidekiq.logger.warn ctxHash
+    def handle_exception(ex, ctxHash={})
+      Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
       Sidekiq.logger.warn ex
       Sidekiq.logger.warn ex.backtrace.join("\n")
       # This list of services is getting a bit ridiculous.

data/lib/sidekiq/launcher.rb

@@ -1,25 +1,54 @@
-require 'sidekiq/util'
+require 'sidekiq/actor'
 require 'sidekiq/manager'
+require 'sidekiq/fetch'
 require 'sidekiq/scheduled'
 
 module Sidekiq
+  # The Launcher is a very simple Actor whose job is to
+  # start, monitor and stop the core Actors in Sidekiq.
+  # If any of these actors die, the Sidekiq process exits
+  # immediately.
   class Launcher
-    attr_reader :manager, :poller, :options
+    include Actor
+    include Util
+
+    trap_exit :actor_died
+
+    attr_reader :manager, :poller, :fetcher
+
     def initialize(options)
+      @manager = Sidekiq::Manager.new_link options
+      @poller = Sidekiq::Scheduled::Poller.new_link
+      @fetcher = Sidekiq::Fetcher.new_link @manager, options
+      @manager.fetcher = @fetcher
+      @done = false
       @options = options
-      @manager = Sidekiq::Manager.new(options)
-      @poller  = Sidekiq::Scheduled::Poller.new
+    end
+
+    def actor_died(actor, reason)
+      return if @done
+      Sidekiq.logger.warn("Sidekiq died due to the following error, cannot recover, process exiting")
+      handle_exception(reason)
+      exit(1)
     end
 
     def run
-      manager.async.start
-      poller.async.poll(true)
+      watchdog('Launcher#run') do
+        manager.async.start
+        poller.async.poll(true)
+      end
     end
 
     def stop
-      poller.async.terminate if poller.alive?
-      manager.async.stop(:shutdown => true, :timeout => options[:timeout])
-      manager.wait(:shutdown)
+      watchdog('Launcher#stop') do
+        @done = true
+        Sidekiq::Fetcher.done!
+        fetcher.async.terminate if fetcher.alive?
+        poller.async.terminate if poller.alive?
+
+        manager.async.stop(:shutdown => true, :timeout => @options[:timeout])
+        manager.wait(:shutdown)
+      end
     end
 
     def procline(tag)

data/lib/sidekiq/manager.rb

@@ -28,7 +28,6 @@ module Sidekiq
       @threads = {}
       @done = false
       @busy = []
-      @fetcher = Fetcher.new(current_actor, options)
       @ready = @count.times.map do
         p = Processor.new_link(current_actor)
         p.proxy_id = p.object_id
@@ -42,8 +41,6 @@ module Sidekiq
         timeout = options[:timeout]
 
         @done = true
-        Sidekiq::Fetcher.done!
-        @fetcher.async.terminate if @fetcher.alive?
 
         logger.info { "Shutting down #{@ready.size} quiet workers" }
         @ready.each { |x| x.terminate if x.alive? }

data/lib/sidekiq/testing.rb

@@ -1,16 +1,77 @@
 module Sidekiq
 
+  class Testing
+    class << self
+      attr_accessor :__test_mode
+
+      def __set_test_mode(mode, &block)
+        if block
+          current_mode = self.__test_mode
+          begin
+            self.__test_mode = mode
+            block.call
+          ensure
+            self.__test_mode = current_mode
+          end
+        else
+          self.__test_mode = mode
+        end
+      end
+
+      def disable!(&block)
+        __set_test_mode(:disable, &block)
+      end
+
+      def fake!(&block)
+        __set_test_mode(:fake, &block)
+      end
+
+      def inline!(&block)
+        __set_test_mode(:inline, &block)
+      end
+
+      def enabled?
+        self.__test_mode != :disable
+      end
+
+      def disabled?
+        self.__test_mode == :disable
+      end
+
+      def fake?
+        self.__test_mode == :fake
+      end
+
+      def inline?
+        self.__test_mode == :inline
+      end
+    end
+  end
+
+  # Default to fake testing to keep old behavior
+  Sidekiq::Testing.fake!
+
   class EmptyQueueError < RuntimeError; end
 
   class Client
     class << self
-      alias_method :raw_push_old, :raw_push
+      alias_method :raw_push_real, :raw_push
 
       def raw_push(payloads)
-        payloads.each do |job|
-          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+        if Sidekiq::Testing.fake?
+          payloads.each do |job|
+            job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+          end
+          true
+        elsif Sidekiq::Testing.inline?
+          payloads.each do |item|
+            marshalled = Sidekiq.load_json(Sidekiq.dump_json(item))
+            marshalled['class'].constantize.new.perform(*marshalled['args'])
+          end
+          true
+        else
+          raw_push_real(payloads)
         end
-        true
       end
     end
   end

data/lib/sidekiq/testing/inline.rb

@@ -1,41 +1,28 @@
-module Sidekiq
-  class Client
+require 'sidekiq/testing'
 
-    ##
-    # The Sidekiq inline infrastructure overrides perform_async so that it
-    # actually calls perform instead. This allows workers to be run inline in a
-    # testing environment.
-    #
-    # This is similar to `Resque.inline = true` functionality.
-    #
-    # Example:
-    #
-    #   require 'sidekiq/testing/inline'
-    #
-    #   $external_variable = 0
-    #
-    #   class ExternalWorker
-    #     include Sidekiq::Worker
-    #
-    #     def perform
-    #       $external_variable = 1
-    #     end
-    #   end
-    #
-    #   assert_equal 0, $external_variable
-    #   ExternalWorker.perform_async
-    #   assert_equal 1, $external_variable
-    #
-    singleton_class.class_eval do
-      alias_method :raw_push_old, :raw_push
-      def raw_push(payload)
-        [payload].flatten.each do |item|
-          marshalled = Sidekiq.load_json(Sidekiq.dump_json(item))
-          marshalled['class'].constantize.new.perform(*marshalled['args'])
-        end
-
-        true
-      end
-    end
-  end
-end
+##
+# The Sidekiq inline infrastructure overrides perform_async so that it
+# actually calls perform instead. This allows workers to be run inline in a
+# testing environment.
+#
+# This is similar to `Resque.inline = true` functionality.
+#
+# Example:
+#
+#   require 'sidekiq/testing/inline'
+#
+#   $external_variable = 0
+#
+#   class ExternalWorker
+#     include Sidekiq::Worker
+#
+#     def perform
+#       $external_variable = 1
+#     end
+#   end
+#
+#   assert_equal 0, $external_variable
+#   ExternalWorker.perform_async
+#   assert_equal 1, $external_variable
+#
+Sidekiq::Testing.inline!

data/lib/sidekiq/util.rb

@@ -15,6 +15,7 @@ module Sidekiq
       yield
     rescue Exception => ex
       handle_exception(ex, { :context => last_words })
+      raise ex
     end
 
     def logger

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.14.1"
+  VERSION = "2.15.0"
 end

data/lib/sidekiq/web.rb

@@ -53,12 +53,21 @@ module Sidekiq
 
       def workers
         @workers ||= begin
-          Sidekiq.redis do |conn|
+          to_rem = []
+          workers = Sidekiq.redis do |conn|
             conn.smembers('workers').map do |w|
               msg = conn.get("worker:#{w}")
-              msg ? [w, Sidekiq.load_json(msg)] : nil
+              msg ? [w, Sidekiq.load_json(msg)] : (to_rem << w; nil)
             end.compact.sort { |x| x[1] ? -1 : 1 }
           end
+
+          # Detect and clear out any orphaned worker records.
+          # These can be left in Redis if Sidekiq crashes hard
+          # while processing jobs.
+          if to_rem.size > 0
+            Sidekiq.redis { |conn| conn.srem('workers', to_rem) }
+          end
+          workers
         end
       end
 
@@ -107,10 +116,14 @@ module Sidekiq
         [score.to_f, jid]
       end
 
+      def truncate(text, truncate_after_chars = 2000)
+        truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
+      end
+
       def display_args(args, truncate_after_chars = 2000)
         args.map do |arg|
           a = arg.inspect
-          truncate_after_chars && a.size > truncate_after_chars ? "#{a[0..truncate_after_chars]}..." : a
+          truncate(a)
         end.join(", ")
       end
 

data/lib/sidekiq/worker.rb

@@ -45,12 +45,12 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
+        item = { 'class' => self, 'args' => args, 'at' => ts }
+
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        if ts <= now
-          perform_async(*args)
-        else
-          client_push('class' => self, 'args' => args, 'at' => ts)
-        end
+        item.delete('at') if ts <= now
+
+        client_push(item)
       end
       alias_method :perform_at, :perform_in
 
@@ -75,10 +75,8 @@ module Sidekiq
         self.sidekiq_retries_exhausted_block = block
       end
 
-      DEFAULT_OPTIONS = { 'retry' => true, 'queue' => 'default' }
-
       def get_sidekiq_options # :nodoc:
-        self.sidekiq_options_hash ||= DEFAULT_OPTIONS
+        self.sidekiq_options_hash ||= Sidekiq.default_worker_options
       end
 
       def client_push(item) # :nodoc: