sidekiq 2.7.2 → 2.7.3

data/Changes.md

@@ -1,3 +1,10 @@
+2.7.3
+-----------
+
+- Real-time dashboard is now the default web page
+- Make config file optional for capistrano
+- Fix Retry All button in the Web UI
+
 2.7.2
 -----------
 

data/Gemfile

@@ -7,6 +7,7 @@ gem 'sqlite3', :platform => :mri
 
 group :test do
   gem 'simplecov', :require => false
+  gem 'minitest-emoji', :require => false
 end
 
 group :development do

data/README.md

@@ -63,7 +63,7 @@ Problems?
 
 **Please do not directly email any Sidekiq committers with questions or problems.**  A community is best served when discussions are held in public.
 
-If you have a problem, please review the [FAQ](/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages. Searching the issues for your problem is also a good idea.  If that doesn't help, feel free to email the Sidekiq mailing list or open a new issue.
+If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages. Searching the issues for your problem is also a good idea.  If that doesn't help, feel free to email the Sidekiq mailing list or open a new issue.
 The mailing list is the preferred place to ask questions on usage. If you are encountering what you think is a bug, please open an issue.
 
 

data/examples/upstart/sidekiq.conf

@@ -0,0 +1,46 @@
+# /etc/init/sidekiq.conf - Sidekiq config
+
+# This example config should work with Ubuntu 12.04+.  It
+# allows you to manage multiple Sidekiq instances with
+# Upstart, Ubuntu's native service management tool.
+#
+# See workers.conf for how to manage all Sidekiq instances at once.
+#
+# Save this config as /etc/init/sidekiq.conf then mange sidekiq with:
+#   sudo start sidekiq index=0
+#   sudo stop sidekiq index=0
+#   sudo status sidekiq index=0
+#
+# or use the service command:
+#   sudo service sidekiq {start,stop,restart,status}
+#
+
+description "Sidekiq Background Worker"
+
+# no "start on", we don't want to automatically start
+stop on (stopping workers or runlevel [06])
+
+# change to match your deployment user
+setuid deploy
+setgid deploy
+
+respawn
+respawn limit 3 30
+
+instance $index
+
+script
+# this script runs in /bin/sh by default
+# respawn as bash so we can source in rbenv
+exec /bin/bash <<EOT
+  # use syslog for logging
+  exec &> /dev/kmsg
+
+  # pull in system rbenv
+  export HOME=/home/deploy
+  source /etc/profile.d/rbenv.sh
+
+  cd /opt/theclymb/current
+  exec bin/sidekiq -i ${index} -e production -C config/sidekiq.yml -P tmp/pids/sidekiq-${index}.pid
+EOT
+end script

data/examples/upstart/workers.conf

@@ -0,0 +1,30 @@
+# /etc/init/workers.conf - manage a set of Sidekiqs
+
+# This example config should work with Ubuntu 12.04+.  It
+# allows you to manage multiple Sidekiq instances with
+# Upstart, Ubuntu's native service management tool.
+#
+# See sidekiq.conf for how to manage a single Sidekiq instance.
+#
+# Use "stop workers" to stop all Sidekiq instances.
+# Use "start workers" to start all instances.
+# Use "restart workers" to restart all instances.
+# Crazy, right?
+#
+
+description "manages the set of sidekiq processes"
+
+# This starts upon bootup and stops on shutdown
+start on runlevel [2345]
+stop on runlevel [06]
+
+# Set this to the number of Sidekiq processes you want
+# to run on this machine
+env NUM_WORKERS=2
+
+pre-start script
+  for i in `seq 0 $((${NUM_WORKERS} - 1))`
+  do
+    start sidekiq index=$i
+  done
+end script

data/lib/sidekiq.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'sidekiq/version'
 require 'sidekiq/logging'
 require 'sidekiq/client'
@@ -26,6 +27,10 @@ module Sidekiq
     :profile => false,
   }
 
+  def self.❨╯°□°❩╯︵ ┻━┻
+    puts "Calm down, bro"
+  end
+
   def self.options
     @options ||= DEFAULTS.dup
   end
@@ -91,7 +96,7 @@ module Sidekiq
   end
 
   def self.load_json(string)
-    MultiJson.decode(string)
+    MultiJson.decode(string, :symbolize_keys => false)
   end
 
   def self.dump_json(object)

data/lib/sidekiq/api.rb

@@ -342,6 +342,12 @@ module Sidekiq
     def initialize
       super 'retry'
     end
+
+    def retry_all
+      while size > 0
+        each(&:retry)
+      end
+    end
   end
 
 

data/lib/sidekiq/capistrano.rb

@@ -36,7 +36,7 @@ Capistrano::Configuration.instance.load do
     task :start, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
       rails_env = fetch(:rails_env, "production")
       for_each_process do |pid_file, idx|
-        run "cd #{current_path} ; #{fetch(:sidekiq_cmd)} -d -e #{rails_env} -C #{current_path}/config/sidekiq.yml -i #{idx} -P #{pid_file} -L #{current_path}/log/sidekiq.log"
+        run "cd #{current_path} ; #{fetch(:sidekiq_cmd)} -d -e #{rails_env} -C #{current_path}/config/sidekiq.yml -i #{idx} -P #{pid_file} -L #{current_path}/log/sidekiq.log", :pty => false
       end
     end
 

data/lib/sidekiq/cli.rb

@@ -18,7 +18,7 @@ end
 
 trap 'USR1' do
   Sidekiq.logger.info "Received USR1, no longer accepting new work"
-  mgr = Sidekiq::CLI.instance.manager
+  mgr = Sidekiq::CLI.instance.launcher.manager
   mgr.async.stop if mgr
 end
 
@@ -49,6 +49,7 @@ require 'erb'
 
 require 'sidekiq'
 require 'sidekiq/util'
+require 'sidekiq/launcher'
 
 module Sidekiq
   class CLI
@@ -57,7 +58,7 @@ module Sidekiq
 
     # Used for CLI testing
     attr_accessor :code
-    attr_accessor :manager
+    attr_accessor :launcher
     attr_accessor :environment
 
     def initialize
@@ -89,21 +90,19 @@ module Sidekiq
         logger.info 'Starting processing, hit Ctrl-C to stop'
       end
 
-      @manager = Sidekiq::Manager.new(options)
-      poller = Sidekiq::Scheduled::Poller.new
+      @launcher = Sidekiq::Launcher.new(options)
+      launcher.procline(options[:tag] ? "#{options[:tag]} " : '')
+
       begin
         if options[:profile]
           require 'ruby-prof'
           RubyProf.start
         end
-        @manager.async.start
-        poller.async.poll(true)
+        launcher.run
         sleep
       rescue Interrupt
         logger.info 'Shutting down'
-        poller.async.terminate if poller.alive?
-        @manager.async.stop(:shutdown => true, :timeout => options[:timeout])
-        @manager.wait(:shutdown)
+        launcher.stop
         # Explicitly exit so busy Processor threads can't block
         # process shutdown.
         exit(0)
@@ -316,9 +315,10 @@ module Sidekiq
       if File.exist?(cfile)
         opts = YAML.load(ERB.new(IO.read(cfile)).result)
         opts = opts.merge(opts.delete(environment) || {})
-        parse_queues opts, opts.delete(:queues) || []
+        parse_queues(opts, opts.delete(:queues) || [])
       else
-        raise ArgumentError, "can't find config file #{cfile}"
+        # allow a non-existent config file so Sidekiq
+        # can be deployed by cap with just the defaults.
       end
       opts
     end

data/lib/sidekiq/launcher.rb

@@ -0,0 +1,31 @@
+
+require 'sidekiq/util'
+require 'sidekiq/manager'
+require 'sidekiq/scheduled'
+
+module Sidekiq
+  class Launcher
+    attr_reader :manager, :poller, :options
+    def initialize(options)
+      @options = options
+      @manager = Sidekiq::Manager.new(options)
+      @poller  = Sidekiq::Scheduled::Poller.new
+    end
+
+    def run
+      manager.async.start
+      poller.async.poll(true)
+    end
+
+    def stop
+      poller.async.terminate if poller.alive?
+      manager.async.stop(:shutdown => true, :timeout => options[:timeout])
+      manager.wait(:shutdown)
+    end
+
+    def procline(tag)
+      $0 = manager.procline(tag)
+      manager.after(5) { procline(tag) }
+    end
+  end
+end

data/lib/sidekiq/manager.rb

@@ -27,7 +27,6 @@ module Sidekiq
       @busy = []
       @fetcher = Fetcher.new(current_actor, options)
       @ready = @count.times.map { Processor.new_link(current_actor) }
-      procline(options[:tag] ? "#{options[:tag]} " : '')
     end
 
     def stop(options={})
@@ -103,6 +102,10 @@ module Sidekiq
       end
     end
 
+    def procline(tag)
+      "sidekiq #{Sidekiq::VERSION} #{tag}[#{@busy.size} of #{@count} busy]#{stopped? ? ' stopping' : ''}"
+    end
+
     private
 
     def hard_shutdown_in(delay)
@@ -130,7 +133,7 @@ module Sidekiq
             workers_to_remove = workers.select do |worker_name|
               worker_name =~ /:#{process_id}-/
             end
-            conn.srem('workers', workers_to_remove)
+            conn.srem('workers', workers_to_remove) if !workers_to_remove.empty?
           end
 
           logger.debug { "Terminating worker threads" }
@@ -156,10 +159,5 @@ module Sidekiq
     def stopped?
       @done
     end
-
-    def procline(tag)
-      $0 = "sidekiq #{Sidekiq::VERSION} #{tag}[#{@busy.size} of #{@count} busy]#{stopped? ? ' stopping' : ''}"
-      after(5) { procline(tag) }
-    end
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.7.2"
+  VERSION = "2.7.3"
 end

data/lib/sidekiq/web.rb

@@ -105,7 +105,7 @@ module Sidekiq
       end
     end
 
-    get "/" do
+    get "/workers" do
       slim :index
     end
 
@@ -172,7 +172,7 @@ module Sidekiq
     end
 
     post "/retries/all/retry" do
-      Sidekiq::RetrySet.new.each { |job| job.retry }
+      Sidekiq::RetrySet.new.retry_all
       redirect "#{root_path}retries"
     end
 
@@ -203,7 +203,7 @@ module Sidekiq
       redirect "#{root_path}scheduled"
     end
 
-    get '/dashboard' do
+    get '/' do
       @redis_info = Sidekiq.redis { |conn| conn.info }.select{ |k, v| redis_keys.include? k }
       stats_history = Sidekiq::Stats::History.new((params[:days] || 30).to_i)
       @processed_history = stats_history.processed
@@ -220,6 +220,7 @@ module Sidekiq
         sidekiq: {
           processed:  sidekiq_stats.processed,
           failed:     sidekiq_stats.failed,
+          busy:       workers.size,
           enqueued:   sidekiq_stats.enqueued,
           scheduled:  sidekiq_stats.scheduled_size,
           retries:    sidekiq_stats.retry_size,
@@ -230,11 +231,11 @@ module Sidekiq
 
     def self.tabs
       @tabs ||= {
-        "Workers"   =>'',
-        "Queues"    =>'queues',
-        "Retries"   =>'retries',
-        "Scheduled" =>'scheduled',
-        "Dashboard" =>'dashboard'
+        "Dashboard" => '',
+        "Workers"   => 'workers',
+        "Queues"    => 'queues',
+        "Retries"   => 'retries',
+        "Scheduled" => 'scheduled',
       }
     end
 

data/test/helper.rb

@@ -10,8 +10,8 @@ rescue LoadError
 end
 
 require 'minitest/unit'
-require 'minitest/pride'
 require 'minitest/autorun'
+require 'minitest/emoji'
 
 require 'sidekiq'
 require 'sidekiq/util'

data/test/test_web.rb

@@ -28,8 +28,8 @@ class TestWeb < MiniTest::Unit::TestCase
       end
     end
 
-    it 'can display home' do
-      get '/'
+    it 'can display workers' do
+      get '/workers'
       assert_equal 200, last_response.status
       assert_match /status-idle/, last_response.body
     end
@@ -194,8 +194,8 @@ class TestWeb < MiniTest::Unit::TestCase
       end
     end
 
-    it 'can display the dashboard' do
-      get '/dashboard'
+    it 'can display home' do
+      get '/'
       assert_equal 200, last_response.status
     end
 
@@ -207,6 +207,8 @@ class TestWeb < MiniTest::Unit::TestCase
         end
         2.times { add_retry }
         3.times { add_scheduled }
+        4.times { add_worker }
+
         get '/dashboard/stats'
         @response = Sidekiq.load_json(last_response.body)
       end
@@ -221,19 +223,23 @@ class TestWeb < MiniTest::Unit::TestCase
         end
 
         it 'reports processed' do
-          assert_equal @response["sidekiq"]["processed"], 5
+          assert_equal 5, @response["sidekiq"]["processed"]
         end
 
         it 'reports failed' do
-          assert_equal @response["sidekiq"]["failed"], 2
+          assert_equal 2, @response["sidekiq"]["failed"]
+        end
+
+        it 'reports busy' do
+          assert_equal 4, @response["sidekiq"]["busy"]
         end
 
         it 'reports retries' do
-          assert_equal @response["sidekiq"]["retries"], 2
+          assert_equal 2, @response["sidekiq"]["retries"]
         end
 
         it 'reports scheduled' do
-          assert_equal @response["sidekiq"]["scheduled"], 3
+          assert_equal 3, @response["sidekiq"]["scheduled"]
         end
       end
 
@@ -271,5 +277,14 @@ class TestWeb < MiniTest::Unit::TestCase
       end
       [msg, score]
     end
+
+    def add_worker
+      process_id = rand(1000)
+      msg = "{\"queue\":\"default\",\"payload\":{\"retry\":true,\"queue\":\"default\",\"timeout\":20,\"backtrace\":5,\"class\":\"HardWorker\",\"args\":[\"bob\",10,5],\"jid\":\"2b5ad2b016f5e063a1c62872\"},\"run_at\":1361208995}"
+      Sidekiq.redis do |conn|
+        conn.sadd("workers", "mercury.home:#{process_id}-70215157189060:started")
+        conn.set("worker:mercury.home:#{process_id}-70215157189060:started", msg)
+      end
+    end
   end
 end

data/web/assets/javascripts/dashboard.js

@@ -42,18 +42,18 @@ var realtimeGraph = function(updatePath) {
     $.getJSON($("#history").data("update-url"), function(data) {
 
       if (i === 0) {
-        var processed = data.processed;
-        var failed = data.failed;
+        var processed = data.sidekiq.processed;
+        var failed = data.sidekiq.failed;
       } else {
-        var processed = data.processed - Sidekiq.processed;
-        var failed = data.failed - Sidekiq.failed;
+        var processed = data.sidekiq.processed - Sidekiq.processed;
+        var failed = data.sidekiq.failed - Sidekiq.failed;
       }
 
       graph.series.addData({ failed: failed, processed: processed });
       graph.render();
 
-      Sidekiq.processed = data.processed;
-      Sidekiq.failed = data.failed;
+      Sidekiq.processed = data.sidekiq.processed;
+      Sidekiq.failed = data.sidekiq.failed;
 
       updateStatsSummary(data.sidekiq);
       updateRedisStats(data.redis);
@@ -114,6 +114,7 @@ var createSeries = function(obj) {
 var updateStatsSummary = function(data) {
   $('ul.summary li.processed span.count').html(data.processed.numberWithDelimiter())
   $('ul.summary li.failed span.count').html(data.failed.numberWithDelimiter())
+  $('ul.summary li.busy span.count').html(data.busy.numberWithDelimiter())
   $('ul.summary li.scheduled span.count').html(data.scheduled.numberWithDelimiter())
   $('ul.summary li.retries span.count').html(data.retries.numberWithDelimiter())
   $('ul.summary li.enqueued span.count').html(data.enqueued.numberWithDelimiter())

data/web/views/_summary.slim

@@ -5,7 +5,7 @@ ul.unstyled.summary
   li.failed
     span.count #{number_with_delimiter(stats.failed)}
     span.desc  Failed
-  li
+  li.busy
     span.count #{number_with_delimiter(workers.size)}
     span.desc  Busy
   li.scheduled

data/web/views/dashboard.slim

@@ -15,7 +15,7 @@ h5
   a href="#{{current_path}}" class="history-graph #{{"active" if params[:days].nil? || params[:days] == "30"}}" 1 month
   a href="#{{current_path}}?days=90" class="history-graph #{{"active" if params[:days] == "90"}}" 3 month
   a href="#{{current_path}}?days=180" class="history-graph #{{"active" if params[:days] == "180"}}" 6 month
-#history data-processed="#{Sidekiq.dump_json(@processed_history)}" data-failed="#{Sidekiq.dump_json(@failed_history)}" data-update-url="#{{current_path}}/stats"
+#history data-processed="#{Sidekiq.dump_json(@processed_history)}" data-failed="#{Sidekiq.dump_json(@failed_history)}" data-update-url="#{{current_path}}dashboard/stats"
 
 br
 h5 Redis

data/web/views/layout.slim

@@ -36,7 +36,7 @@ html
               span.title Status
               == slim :_status
             == slim :_summary
-            - unless current_path.include?("dashboard")
+            - unless current_path == ''
               .row
                 .span2
                   - if params[:poll]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 2.7.2
+  version: 2.7.3
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain:
 - .public_cert.pem
-date: 2013-02-08 00:00:00.000000000 Z
+date: 2013-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -225,7 +225,8 @@ files:
 - examples/scheduling.rb
 - examples/sidekiq
 - examples/sinkiq.rb
-- examples/upstart.conf
+- examples/upstart/sidekiq.conf
+- examples/upstart/workers.conf
 - examples/web-ui.png
 - lib/sidekiq.rb
 - lib/sidekiq/api.rb
@@ -239,6 +240,7 @@ files:
 - lib/sidekiq/extensions/class_methods.rb
 - lib/sidekiq/extensions/generic_proxy.rb
 - lib/sidekiq/fetch.rb
+- lib/sidekiq/launcher.rb
 - lib/sidekiq/logging.rb
 - lib/sidekiq/manager.rb
 - lib/sidekiq/middleware/chain.rb

data/examples/upstart.conf

@@ -1,39 +0,0 @@
-# /etc/init/sidekiq.conf - Sidekiq config
-
-# This example config should work with Ubuntu 12.04+
-#
-# Save this config as /etc/init/sidekiq.conf then mange sidekiq with:
-#   sudo start sidekiq
-#   sudo stop sidekiq
-#   sudo status sidekiq
-#
-# or use the service command:
-#   sudo service sidekiq {start,stop,restart,status}
-#
-# This is the exact script used to manage Sidekiq on TheClymb.com
-#
-
-description "Sidekiq Background Worker System"
-
-start on runlevel [2345]
-stop  on shutdown
-
-setuid deploy
-setgid deploy
-
-respawn
-respawn limit 3 30
-
-script
-# this script runs in /bin/sh by default
-# respawn as bash so we can source in rbenv
-exec /bin/bash <<EOT
-  # use syslog for logging
-  exec >/dev/kmsg 2>&1
-  # pull in system rbenv
-  source /etc/profile.d/rbenv.sh
-
-  cd /opt/theclymb/current
-  exec bin/sidekiq -e production -C config/sidekiq.yml -P tmp/pids/sidekiq.pid
-EOT
-end script