sidekiq 2.13.1 → 2.14.0

data/Changes.md

@@ -1,3 +1,14 @@
+2.14.0
+-----------
+
+- Removed slim gem dependency, Web UI now uses ERB [Locke23rus, #1120]
+- Fix more race conditions in Web UI actions
+- Don't reset Job enqueued\_at when retrying
+- Timestamp tooltips in the Web UI should use UTC
+- Fix invalid usage of handle\_exception causing issues in Airbrake
+  [#1134]
+
+
 2.13.1
 -----------
 
@@ -12,7 +23,7 @@
 ```ruby
 Sidekiq.configure_server do |config|
   config.server_middleware do |chain|
-    chain.add Sidekiq::Middleare::Server::RetryJobs, :max_retries => 10
+    chain.add Sidekiq::Middleware::Server::RetryJobs, :max_retries => 10
   end
 end
 ```

data/Contributing.md

@@ -3,6 +3,9 @@
 First of all, thank you for even opening this file up!  I hope you find
 it worthwhile to help out with Sidekiq.
 
+## Issues
+
+When opening an issue, please include a **backtrace** with your error.
 
 ## Code
 

data/Gemfile

@@ -1,17 +1,2 @@
-source 'http://rubygems.org'
+source 'https://rubygems.org'
 gemspec
-
-gem 'slim', '1.1.0' # earliest verson supported
-
-gem 'sqlite3', :platform => :mri
-
-group :test do
-  gem 'simplecov', :require => false
-  gem 'minitest-emoji', :require => false
-end
-
-group :development do
-  gem 'pry', :platform => :mri
-  gem 'shotgun'
-  gem 'rack', '~> 1.4.0'
-end

data/Pro-Changes.md

@@ -3,6 +3,24 @@ Sidekiq Pro Changelog
 
 Please see http://sidekiq.org/pro for more details and how to buy.
 
+1.2.3
+-----------
+
+- Pro now requires Sidekiq 2.14.0
+- Fix bad exception handling in batch callbacks [#1134]
+- Convert Batch UI to ERB
+
+1.2.2
+-----------
+
+- Problem with reliable fetch which could lead to lost jobs when Sidekiq
+  is shut down normally.  Thanks to MikaelAmborn for the report. [#1109]
+
+1.2.1
+-----------
+
+- Forgot to push paging code necessary for `delete_job` performance.
+
 1.2.0
 -----------
 

data/lib/sidekiq/api.rb

@@ -175,7 +175,7 @@ module Sidekiq
     end
 
     def enqueued_at
-      Time.at(@item['enqueued_at'] || 0)
+      Time.at(@item['enqueued_at'] || 0).utc
     end
 
     def queue
@@ -210,7 +210,7 @@ module Sidekiq
     end
 
     def at
-      Time.at(score)
+      Time.at(score).utc
     end
 
     def delete

data/lib/sidekiq/cli.rb

@@ -9,10 +9,13 @@ require 'sidekiq'
 require 'sidekiq/util'
 
 module Sidekiq
-  # Used to raise in workers that have not finished within the
-  # hard timeout limit.  This is needed to rollback db transactions,
+  # We are shutting down Sidekiq but what about workers that
+  # are working on some long job?  This error is
+  # raised in workers that have not finished within the hard
+  # timeout limit.  This is needed to rollback db transactions,
   # otherwise Ruby's Thread#kill will commit.  See #377.
-  class Shutdown < RuntimeError; end
+  # DO NOT RESCUE THIS ERROR.
+  class Shutdown < Interrupt; end
 
   class CLI
     include Util
@@ -105,7 +108,7 @@ module Sidekiq
       when 'USR2'
         if Sidekiq.options[:logfile]
           Sidekiq.logger.info "Received USR2, reopening log file"
-          Sidekiq::Logging.initialize_logger(Sidekiq.options[:logfile])
+          initialize_logger
         end
       when 'TTIN'
         Thread.list.each do |thread|

data/lib/sidekiq/client.rb

@@ -133,7 +133,7 @@ module Sidekiq
         end
 
         normalized_item['jid'] ||= SecureRandom.hex(12)
-        normalized_item['enqueued_at'] = Time.now.to_f
+        normalized_item['enqueued_at'] ||= Time.now.to_f
         normalized_item
       end
 

data/lib/sidekiq/exception_handler.rb

@@ -1,39 +1,39 @@
 module Sidekiq
   module ExceptionHandler
 
-    def handle_exception(ex, msg)
-      Sidekiq.logger.warn msg
+    def handle_exception(ex, ctxHash)
+      Sidekiq.logger.warn ctxHash
       Sidekiq.logger.warn ex
       Sidekiq.logger.warn ex.backtrace.join("\n")
       # This list of services is getting a bit ridiculous.
       # For future error services, please add your own
       # middleware like BugSnag does:
       # https://github.com/bugsnag/bugsnag-ruby/blob/master/lib/bugsnag/sidekiq.rb
-      send_to_airbrake(msg, ex) if defined?(::Airbrake)
-      send_to_honeybadger(msg, ex) if defined?(::Honeybadger)
-      send_to_exceptional(msg, ex) if defined?(::Exceptional)
-      send_to_exception_notifier(msg, ex) if defined?(::ExceptionNotifier)
+      send_to_airbrake(ctxHash, ex) if defined?(::Airbrake)
+      send_to_honeybadger(ctxHash, ex) if defined?(::Honeybadger)
+      send_to_exceptional(ctxHash, ex) if defined?(::Exceptional)
+      send_to_exception_notifier(ctxHash, ex) if defined?(::ExceptionNotifier)
     end
 
     private
 
-    def send_to_airbrake(msg, ex)
-      ::Airbrake.notify_or_ignore(ex, :parameters => msg)
+    def send_to_airbrake(hash, ex)
+      ::Airbrake.notify_or_ignore(ex, :parameters => hash)
     end
 
-    def send_to_honeybadger(msg, ex)
-      ::Honeybadger.notify_or_ignore(ex, :parameters => msg)
+    def send_to_honeybadger(hash, ex)
+      ::Honeybadger.notify_or_ignore(ex, :parameters => hash)
     end
 
-    def send_to_exceptional(msg, ex)
+    def send_to_exceptional(hash, ex)
       if ::Exceptional::Config.should_send_to_api?
-        ::Exceptional.context(msg)
+        ::Exceptional.context(hash)
         ::Exceptional::Remote.error(::Exceptional::ExceptionData.new(ex))
       end
     end
 
-    def send_to_exception_notifier(msg, ex)
-      ::ExceptionNotifier.notify_exception(ex, :data => {:message => msg})
+    def send_to_exception_notifier(hash, ex)
+      ::ExceptionNotifier.notify_exception(ex, :data => {:message => hash})
     end
   end
 end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -116,7 +116,7 @@ module Sidekiq
           end
 
         rescue Exception => e
-          handle_exception(e, "Error calling retries_exhausted")
+          handle_exception(e, { :context => "Error calling retries_exhausted" })
         end
 
         def retry_attempts_from(msg_retry, default)

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.13.1"
+  VERSION = "2.14.0"
 end

data/lib/sidekiq/web.rb

@@ -1,8 +1,6 @@
+require 'erb'
 require 'yaml'
 require 'sinatra/base'
-require 'slim'
-
-raise "The Sidekiq Web UI requires slim 1.1.0 or greater.  You have slim v#{Slim::VERSION}" if Gem::Version.new(Slim::VERSION) < Gem::Version.new('1.1.0')
 
 require 'sidekiq'
 require 'sidekiq/api'
@@ -16,7 +14,6 @@ module Sidekiq
     set :public_folder, Proc.new { "#{root}/assets" }
     set :views, Proc.new { "#{root}/views" }
     set :locales, Proc.new { "#{root}/locales" }
-    set :slim, :pretty => true
 
     helpers do
       def strings
@@ -161,15 +158,19 @@ module Sidekiq
       def redis_keys
         ["redis_stats", "uptime_in_days", "connected_clients", "used_memory_human", "used_memory_peak_human"]
       end
+
+      def h(text)
+        ERB::Util.h(text)
+      end
     end
 
     get "/workers" do
-      slim :index
+      erb :index
     end
 
     get "/queues" do
       @queues = Sidekiq::Queue.all
-      slim :queues
+      erb :queues
     end
 
     get "/queues/:name" do
@@ -178,7 +179,7 @@ module Sidekiq
       @name = params[:name]
       (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
       @messages = @messages.map {|msg| Sidekiq.load_json(msg) }
-      slim :queue
+      erb :queue
     end
 
     post "/reset" do
@@ -200,14 +201,14 @@ module Sidekiq
       @count = (params[:count] || 25).to_i
       (@current_page, @total_size, @retries) = page("retry", params[:page], @count)
       @retries = @retries.map {|msg, score| [Sidekiq.load_json(msg), score] }
-      slim :retries
+      erb :retries
     end
 
     get "/retries/:key" do
       halt 404 unless params['key']
       @retry = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
       redirect "#{root_path}retries" if @retry.nil?
-      slim :retry
+      erb :retry
     end
 
     post '/retries' do
@@ -238,10 +239,12 @@ module Sidekiq
     post "/retries/:key" do
       halt 404 unless params['key']
       job = Sidekiq::RetrySet.new.fetch(*parse_params(params['key'])).first
-      if params['retry']
-        job.retry
-      elsif params['delete']
-        job.delete
+      if job
+        if params['retry']
+          job.retry
+        elsif params['delete']
+          job.delete
+        end
       end
       redirect "#{root_path}retries"
     end
@@ -250,14 +253,14 @@ module Sidekiq
       @count = (params[:count] || 25).to_i
       (@current_page, @total_size, @scheduled) = page("schedule", params[:page], @count)
       @scheduled = @scheduled.map {|msg, score| [Sidekiq.load_json(msg), score] }
-      slim :scheduled
+      erb :scheduled
     end
 
     get "/scheduled/:key" do
       halt 404 unless params['key']
       @job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
       redirect "#{root_path}scheduled" if @job.nil?
-      slim :scheduled_job_info
+      erb :scheduled_job_info
     end
 
     post '/scheduled' do
@@ -265,10 +268,12 @@ module Sidekiq
 
       params['key'].each do |key|
         job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
-        if params['delete']
-          job.delete
-        elsif params['add_to_queue']
-          job.add_to_queue
+        if job
+          if params['delete']
+            job.delete
+          elsif params['add_to_queue']
+            job.add_to_queue
+          end
         end
       end
       redirect "#{root_path}scheduled"
@@ -277,10 +282,12 @@ module Sidekiq
     post "/scheduled/:key" do
       halt 404 unless params['key']
       job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
-      if params['add_to_queue']
-        job.add_to_queue
-      elsif params['delete']
-        job.delete
+      if job
+        if params['add_to_queue']
+          job.add_to_queue
+        elsif params['delete']
+          job.delete
+        end
       end
       redirect "#{root_path}scheduled"
     end
@@ -290,7 +297,7 @@ module Sidekiq
       stats_history = Sidekiq::Stats::History.new((params[:days] || 30).to_i)
       @processed_history = stats_history.processed
       @failed_history = stats_history.failed
-      slim :dashboard
+      erb :dashboard
     end
 
     get '/dashboard/stats' do

data/sidekiq.gemspec

@@ -14,13 +14,12 @@ Gem::Specification.new do |gem|
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
-  gem.add_dependency                  'redis', '>= 3.0'
+  gem.add_dependency                  'redis', '>= 3.0.4'
   gem.add_dependency                  'redis-namespace'
   gem.add_dependency                  'connection_pool', '>= 1.0.0'
   gem.add_dependency                  'celluloid', '>= 0.14.1'
   gem.add_dependency                  'json'
   gem.add_development_dependency      'sinatra'
-  gem.add_development_dependency      'slim', '>= 1.1.0'
   gem.add_development_dependency      'minitest', '~> 5'
   gem.add_development_dependency      'rake'
   gem.add_development_dependency      'actionmailer'

data/web/views/_job_info.erb

@@ -0,0 +1,76 @@
+<header>
+  <h3><%= t('Job') %></h3>
+</header>
+
+<table class="table table-bordered table-striped">
+  <tbody>
+    <tr>
+      <th><%= t('Queue') %></th>
+      <td>
+        <a href="<%= root_path %>queues/<%= job['queue'] %>"><%= job['queue'] %></a>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('Class') %></th>
+      <td>
+        <code><%= job['class'] %></code>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('Arguments') %></th>
+      <td>
+        <code>
+          <!-- We don't want to truncate any job arguments when viewing a single job's status page -->
+          <div class="args-extended"><%= display_args(job['args'], nil) %></div>
+        </code>
+      </td>
+    </tr>
+    <tr>
+      <th>JID</th>
+      <td>
+        <code><%= job.jid %></code>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('Enqueued') %></th>
+      <td><%= relative_time(job.enqueued_at) %></td>
+    </tr>
+    <% unless retry_extra_items(job).empty? %>
+      <tr>
+        <th><%= t('Extras') %></th>
+        <td>
+          <code>
+            <%= retry_extra_items(job).inspect %>
+          </code>
+        </td>
+      </tr>
+    <% end %>
+    <% if type == :retry %>
+      <% if job['retry_count'] && job['retry_count'] > 0 %>
+        <tr>
+          <th><%= t('RetryCount') %></th>
+          <td><%= job['retry_count'] %></td>
+        </tr>
+        <tr>
+          <th><%= t('LastRetry') %></th>
+          <td><%= relative_time(job['retried_at'].is_a?(Numeric) ? Time.at(job['retried_at']) : Time.parse(job['retried_at'])) %></td>
+        </tr>
+      <% else %>
+        <tr>
+          <th><%= t('OriginallyFailed') %></th>
+          <td><%= relative_time(job['failed_at'].is_a?(Numeric) ? Time.at(job['failed_at']) : Time.parse(job['failed_at'])) %></td>
+        </tr>
+      <% end %>
+      <tr>
+        <th><%= t('NextRetry') %></th>
+        <td><%= relative_time(job.at) %></td>
+      </tr>
+    <% end %>
+    <% if type == :scheduled %>
+      <tr>
+        <th><%= t('Scheduled') %></th>
+        <td><%= relative_time(job.at) %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

data/web/views/_nav.erb

@@ -0,0 +1,23 @@
+<div class="navbar-inner">
+  <div class="container">
+    <a class="brand" href="<%= root_path %>"><%= Sidekiq::NAME %></a>
+    <ul class="nav">
+      <% tabs.each do |title, url| %>
+        <% if url == '' %>
+          <li class="<%= current_path == url ? 'active' : '' %>">
+            <a href="<%= root_path %><%= url %>"><%= t(title) %></a>
+          </li>
+        <% else %>
+          <li class="<%= current_path.start_with?(url) ? 'active' : '' %>">
+            <a href="<%= root_path %><%= url %>"><%= t(title) %></a>
+          </li>
+        <% end %>
+      <% end %>
+      <% custom_tabs.each do |title, url| %>
+        <li class="<%= current_path.start_with?(url) ? 'active' : '' %>">
+          <a href="<%= root_path %><%= url %>"><%= t(title) %></a>
+        </li>
+      <% end %>
+    </ul>
+  </div>
+</div>

data/web/views/_paging.erb

@@ -0,0 +1,25 @@
+<% if @total_size > @count %>
+  <div class="pagination pagination-right">
+    <ul>
+      <li class="<%= 'disabled' if @current_page == 1 %>">
+        <a href="<%= url %>?page=1">«</a>
+      </li>
+      <% if @current_page > 1 %>
+        <li>
+          <a href="<%= url %>?page=<%= @current_page - 1 %>"><%= @current_page - 1 %></a>
+        </li>
+      <% end %>
+      <li class="disabled">
+        <a href="<%= url %>?page=<%= @current_page %>"><%= @current_page %></a>
+      </li>
+      <% if @total_size > @current_page * @count %>
+        <li>
+          <a href="<%= url %>?page=<%= @current_page + 1 %>"><%= @current_page + 1 %></a>
+        </li>
+      <% end %>
+      <li class="<%= 'disabled' if @total_size <= @current_page * @count %>">
+        <a href="<%= url %>?page=<%= (@total_size.to_f / @count).ceil %>">»</a>
+      </li>
+    </ul>
+  </div>
+<% end %>

data/web/views/_status.erb

@@ -0,0 +1,4 @@
+<span class="status">
+  <i class="status-sprite status-<%= current_status %>"></i>
+  <% t(current_status) %>
+</span>

data/web/views/_summary.erb

@@ -0,0 +1,26 @@
+<ul class="unstyled summary row">
+  <li class="processed span2">
+    <span class="count"><%= number_with_delimiter(stats.processed) %></span>
+    <span class="desc"><%= t('Processed') %></span>
+  </li>
+  <li class="failed span2">
+    <span class="count"><%= number_with_delimiter(stats.failed) %></span>
+    <span class="desc"><%= t('Failed') %></span>
+  </li>
+  <li class="busy span2">
+    <span class="count"><%= number_with_delimiter(workers_size) %></span>
+    <span class="desc"><%= t('Busy') %></span>
+  </li>
+  <li class="scheduled span2">
+    <span class="count"><%= number_with_delimiter(stats.scheduled_size) %></span>
+    <span class="desc"><%= t('Scheduled') %></span>
+  </li>
+  <li class="retries span2">
+    <span class="count"><%= number_with_delimiter(stats.retry_size) %></span>
+    <span class="desc"><%= t('Retries') %></span>
+  </li>
+  <li class="enqueued span2">
+    <span class="count"><%= number_with_delimiter(stats.enqueued) %></span>
+    <span class="desc"><%= t('Enqueued') %></span>
+  </li>
+</ul>

data/web/views/_workers.erb

@@ -0,0 +1,29 @@
+<table class="workers table table-hover table-bordered table-striped table-white">
+  <thead>
+    <th><%= t('Worker') %></th>
+    <th><%= t('Queue') %></th>
+    <th><%= t('Class') %></th>
+    <th><%= t('Arguments') %></th>
+    <th><%= t('Started') %></th>
+  </thead>
+  <% workers.each_with_index do |(worker, msg), index| %>
+    <tr>
+      <td><%= worker %></td>
+      <td>
+        <a href="<%= root_path %>queues/<%= msg['queue'] %>">= msg['queue'] %></a>
+      </td>
+
+      <td><%= msg['payload']['class'] %></td>
+      <td>
+        <% if msg['payload']['args'].to_s.size > 100 %>
+          <%= msg['payload']['args'].inspect[0..100] + "... " %>
+          <button data-toggle="collapse" data-target="#worker_<%= index %>" class="btn btn-mini"><%= t('ShowAll') %></button>
+          <div class="toggle" id="worker_<%= index %>" style="display: none;max-width: 750px;"><%= msg['payload']['args'] %></div>
+        <% else %>
+          <%= msg['payload']['args'] %>
+        <% end %>
+      </td>
+      <td><%= relative_time(msg['run_at'].is_a?(Numeric) ? Time.at(msg['run_at']) : Time.parse(msg['run_at'])) %></td>
+    </tr>
+  <% end %>
+</table>

data/web/views/dashboard.erb

@@ -0,0 +1,59 @@
+<script type="text/javascript" src="<%= root_path %>javascripts/dashboard.js"></script>
+
+<h3>
+  <%= t('Dashboard') %>
+  <span class="beacon">
+    <span class="ring"></span>
+    <span class="dot"></span>
+  </span>
+</h3>
+
+<h5></h5>
+<div id="realtime"></div>
+
+<h5>
+  <span class="history-heading"><%= t('History') %></span>
+  <a href="<%= root_path %>?days=7" class="history-graph <%= "active" if params[:days] == "7" %>"><%= t('OneWeek') %></a>
+  <a href="<%= root_path %>" class="history-graph <%= "active" if params[:days].nil? || params[:days] == "30" %>" ><%= t('OneMonth') %></a>
+  <a href="<%= root_path %>?days=90" class="history-graph <%= "active" if params[:days] == "90" %>"><%= t('ThreeMonths') %></a>
+  <a href="<%= root_path %>?days=180" class="history-graph <%= "active" if params[:days] == "180" %>"><%= t('SixMonths') %></a>
+</h5>
+<div id="history" data-processed="<%= h Sidekiq.dump_json(@processed_history) %>" data-failed="<%= h Sidekiq.dump_json(@failed_history) %>" data-update-url="<%= root_path %>dashboard/stats"></div>
+
+<br/>
+<h5>Redis</h5>
+
+<% if @redis_info.fetch("redis_version", nil) %>
+  <div class="stat">
+    <h3 class="redis_version"><%= @redis_info.fetch("redis_version") %></h3>
+    <p><%= t('Version') %></p>
+  </div>
+<% end %>
+
+<% if @redis_info.fetch("uptime_in_days", nil) %>
+  <div class="stat">
+    <h3 class="uptime_in_days"><%= @redis_info.fetch("uptime_in_days") %></h3>
+    <p><%= t('Uptime') %></p>
+  </div>
+<% end %>
+
+<% if @redis_info.fetch("connected_clients", nil) %>
+  <div class="stat">
+    <h3 class="connected_clients"><%= @redis_info.fetch("connected_clients") %></h3>
+    <p><%= t('Connections') %></p>
+  </div>
+<% end %>
+
+<%  if @redis_info.fetch("used_memory_human", nil) %>
+  <div class="stat">
+    <h3 class="used_memory_human"><%= @redis_info.fetch("used_memory_human") %></h3>
+    <p><%= t('MemoryUsage') %></p>
+  </div>
+<% end %>
+
+<%  if @redis_info.fetch("used_memory_peak_human", nil) %>
+  <div class="stat">
+    <h3 class="used_memory_peak_human"><%= @redis_info.fetch("used_memory_peak_human") %></h3>
+    <p><%= @redis_info.fetch("used_memory_peak_human") %></p>
+  </div>
+<% end %>

data/web/views/index.erb

@@ -0,0 +1,16 @@
+<div class="row header">
+  <div class="span7">
+    <h3><%= t('Workers') %></h3>
+  </div>
+</div>
+
+<%= erb :_workers %>
+<% if workers_size > 0 %>
+  <div class="row">
+    <div class="span2 pull-right">
+      <form action="<%= root_path %>reset" method="post">
+        <button class="btn btn-primary btn-block" type="submit"><%= t('ClearWorkerList') %></button>
+      </form>
+    </div>
+  </div>
+<% end %>