sidekiq-failures-discourse 0.3.0

data/lib/sidekiq/failures/sorted_entry.rb

@@ -0,0 +1,14 @@
+module Sidekiq
+  class SortedEntry
+    def retry_failure
+      Sidekiq.redis do |conn|
+        results = conn.zrangebyscore(Sidekiq::Failures::LIST_KEY, score, score)
+        conn.zremrangebyscore(Sidekiq::Failures::LIST_KEY, score, score)
+        results.map do |message|
+          msg = Sidekiq.load_json(message)
+          Sidekiq::Client.push(msg)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/failures/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Failures
+    VERSION = "0.3.0"
+  end
+end

data/lib/sidekiq/failures/views/failure.erb

@@ -0,0 +1,30 @@
+<%= erb :_job_info, :locals => {:job => @failure, :type => :failure} %>
+
+<h3><%= t('Error') %></h3>
+<table class="error table table-bordered table-striped">
+  <tbody>
+    <tr>
+      <th><%= t('ErrorClass') %></th>
+      <td>
+        <code><%= @failure['error_class'] %></code>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('ErrorMessage') %></th>
+      <td><%= @failure['error_message'] %></td>
+    </tr>
+    <% if !@failure['error_backtrace'].nil? %>
+      <tr>
+        <th><%= t('ErrorBacktrace') %></th>
+        <td>
+          <code><%= @failure['error_backtrace'].join("<br/>") %></code>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+<form class="form-horizontal" action="<%= root_path %>failures/<%= job_params(@failure, @failure.score) %>" method="post">
+  <a class="btn" href="<%= root_path %>failures"><%= t('GoBack') %></a>
+  <input class="btn btn-primary" type="submit" name="retry" value="<%= t('RetryNow') %>" />
+  <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" />
+</form>

data/lib/sidekiq/failures/views/failures.erb

@@ -0,0 +1,75 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3><%= t('FailedJobs') %></h3>
+  </div>
+  <% if @failures.size > 0 && @total_size > @count %>
+    <div class="col-sm-4">
+      <%= erb :_paging, :locals => { :url => "#{root_path}failures" } %>
+    </div>
+  <% end %>
+  <%= filtering('failures') %>
+</header>
+
+<% if @failures.size > 0 %>
+  <form action="<%= root_path %>failures" method="post">
+    <table class="table table-striped table-bordered table-white">
+      <thead>
+        <tr>
+          <th width="20px" class="table-checkbox">
+            <label>
+              <input type="checkbox" class="check_all" />
+            </label>
+          </th>
+          <th style="width: 20%"><%= t('Worker') %></th>
+          <th style="width: 10%"><%= t('Arguments') %></th>
+          <th style="width: 10%"><%= t('Queue') %></th>
+          <th style="width: 10%"><%= t('FailedAt') %></th>
+          <th style="width: 50%"><%= t('Error') %></th>
+        </tr>
+      </thead>
+      <% @failures.each do |entry| %>
+        <tr>
+          <td class="table-checkbox">
+            <label>
+              <input type='checkbox' name='key[]' value='<%= job_params(entry.item, entry.score) %>' />
+            </label>
+          </td>
+          <td><a href="<%= root_path %>failures/<%= job_params(entry.item, entry.score) %>"><%= entry.klass %></a></td>
+          <td>
+            <div class="args"><%= display_args(entry.args) %></div>
+          </td>
+          <td>
+            <a href="<%= root_path %>queues/<%= entry.queue %>"><%= entry.queue %></a>
+          </td>
+          <td><%= relative_time(Time.at(entry['failed_at'])) %></td>
+          <td style="overflow: auto; padding: 10px;">
+            <a class="backtrace" href="#" onclick="$(this).next().toggle(); return false">
+              <%= h entry['error_class'] %>: <%= h entry['error_message'] %>
+            </a>
+            <pre style="display: none; background: none; border: 0; width: 100%; max-height: 30em; font-size: 0.8em; white-space: nowrap;">
+              <%= entry['error_backtrace'].join("<br />") if entry['error_backtrace'] %>
+            </pre>
+            <p>
+              <span>Processor: <%= entry['processor'] %></span>
+            </p>
+          </td>
+        </tr>
+      <% end %>
+    </table>
+    <input class="btn btn-primary btn-xs pull-left" type="submit" name="retry" value="<%= t('RetryNow') %>" />
+    <input class="btn btn-danger btn-xs pull-left" type="submit" name="delete" value="<%= t('Delete') %>" />
+  </form>
+
+  <form action="<%= root_path %>failures/all/reset" method="post">
+    <input class="btn btn-danger btn-xs pull-right" type="submit" name="reset" value="<%= t('Reset Counter') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+  <form action="<%= root_path %>failures/all/delete" method="post">
+    <input class="btn btn-danger btn-xs pull-right" type="submit" name="delete" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+  <form action="<%= root_path %>failures/all/retry" method="post">
+    <input class="btn btn-danger btn-xs pull-right" type="submit" name="retry" value="<%= t('RetryAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+
+<% else %>
+  <div class="alert alert-success"><%= t('NoFailedJobsFound') %></div>
+<% end %>

data/lib/sidekiq/failures/web_extension.rb

@@ -0,0 +1,72 @@
+module Sidekiq
+  module Failures
+    module WebExtension
+
+      def self.registered(app)
+        view_path = File.join(File.expand_path("..", __FILE__), "views")
+
+        app.get "/failures" do
+          @count = (params[:count] || 25).to_i
+          (@current_page, @total_size, @failures) = page(LIST_KEY, params[:page], @count)
+          @failures = @failures.map {|msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+
+          render(:erb, File.read(File.join(view_path, "failures.erb")))
+        end
+
+        app.get "/failures/:key" do
+          halt 404 unless params['key']
+
+          @failure = FailureSet.new.fetch(*parse_params(params['key'])).first
+          redirect "#{root_path}failures" if @failure.nil?
+          render(:erb, File.read(File.join(view_path, "failure.erb")))
+        end
+
+        app.post "/failures" do
+          halt 404 unless params['key']
+
+          params['key'].each do |key|
+            job = FailureSet.new.fetch(*parse_params(key)).first
+            next unless job
+
+            if params['retry']
+              job.retry_failure
+            elsif params['delete']
+              job.delete
+            end
+          end
+
+          redirect_with_query("#{root_path}failures")
+        end
+
+        app.post "/failures/:key" do
+          halt 404 unless params['key']
+
+          job = FailureSet.new.fetch(*parse_params(params['key'])).first
+          if job
+            if params['retry']
+              job.retry_failure
+            elsif params['delete']
+              job.delete
+            end
+          end
+          redirect_with_query("#{root_path}failures")
+        end
+
+        app.post "/failures/all/reset" do
+          Sidekiq::Failures.reset_failures
+          redirect "#{root_path}failures"
+        end
+
+        app.post "/failures/all/delete" do
+          FailureSet.new.clear
+          redirect "#{root_path}failures"
+        end
+
+        app.post "/failures/all/retry" do
+          FailureSet.new.retry_all_failures
+          redirect "#{root_path}failures"
+        end
+      end
+    end
+  end
+end

data/sidekiq-failures.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/sidekiq/failures/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Marcelo Silveira"]
+  gem.email         = ["marcelo@mhfs.com.br"]
+  gem.description   = %q{Keep track of Sidekiq failed jobs}
+  gem.summary       = %q{Keeps track of Sidekiq failed jobs and adds a tab to the Web UI to let you browse them. Makes use of Sidekiq's custom tabs and middleware chain.}
+  gem.homepage      = "https://github.com/mhfs/sidekiq-failures/"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "sidekiq-failures-discourse"
+  gem.require_paths = ["lib"]
+  gem.version       = Sidekiq::Failures::VERSION
+
+  gem.add_dependency "sidekiq", ">= 2.14.0"
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rack-test"
+  gem.add_development_dependency "sprockets"
+  gem.add_development_dependency "sinatra"
+end

data/test/middleware_test.rb

@@ -0,0 +1,283 @@
+require "test_helper"
+
+module Sidekiq
+  module Failures
+    describe "Middleware" do
+      before do
+        Celluloid.boot
+        $invokes = 0
+        @boss = MiniTest::Mock.new
+        @processor = ::Sidekiq::Processor.new(@boss)
+        Sidekiq.server_middleware {|chain| chain.add Sidekiq::Failures::Middleware }
+        Sidekiq.redis = REDIS
+        Sidekiq.redis { |c| c.flushdb }
+        Sidekiq.instance_eval { @failures_default_mode = nil }
+      end
+
+      TestException = Class.new(Exception)
+      ShutdownException = Class.new(Sidekiq::Shutdown)
+
+      class MockWorker
+        include Sidekiq::Worker
+
+        def perform(args)
+          $invokes += 1
+          raise ShutdownException.new if args == "shutdown"
+          raise TestException.new("failed!")
+        end
+      end
+
+      it 'raises an error when failures_default_mode is configured incorrectly' do
+        assert_raises ArgumentError do
+          Sidekiq.failures_default_mode = 'exhaustion'
+        end
+      end
+
+      it 'defaults failures_default_mode to all' do
+        assert_equal :all, Sidekiq.failures_default_mode
+      end
+
+      it 'records all failures by default' do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'])
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 1, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it 'records all failures if explicitly told to' do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'failures' => true)
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 1, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it "doesn't record internal shutdown failure" do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['shutdown'], 'failures' => true)
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        @processor.process(msg)
+        @boss.verify
+
+        assert_equal 0, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it "doesn't record failure if failures disabled" do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'failures' => false)
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 0, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it "doesn't record failure if going to be retired again and configured to track exhaustion by default" do
+        Sidekiq.failures_default_mode = :exhausted
+
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry' => true )
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 0, failures_count
+        assert_equal 1, $invokes
+      end
+
+
+      it "doesn't record failure if going to be retired again and configured to track exhaustion" do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry' => true, 'failures' => 'exhausted')
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 0, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it "records failure if failing last retry and configured to track exhaustion" do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry' => true, 'retry_count' => 25, 'failures' => 'exhausted')
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 1, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it "records failure if retry disabled and configured to track exhaustion" do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry' => false, 'failures' => 'exhausted')
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 1, failures_count
+        assert_equal 1, $invokes
+       end
+
+      it "records failure if retry disabled and configured to track exhaustion by default" do
+        Sidekiq.failures_default_mode = 'exhausted'
+
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry' => false)
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 1, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it "records failure if failing last retry and configured to track exhaustion by default" do
+        Sidekiq.failures_default_mode = 'exhausted'
+
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry' => true, 'retry_count' => 25)
+
+        assert_equal 0, failures_count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        2.times { @boss.expect(:async, actor, []) }
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 1, failures_count
+        assert_equal 1, $invokes
+      end
+
+      it "removes old failures when failures_max_count has been reached" do
+        assert_equal 1000, Sidekiq.failures_max_count
+        Sidekiq.failures_max_count = 2
+
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'])
+
+        assert_equal 0, failures_count
+
+        3.times do
+          boss = MiniTest::Mock.new
+          processor = ::Sidekiq::Processor.new(boss)
+
+          actor = MiniTest::Mock.new
+          actor.expect(:processor_done, nil, [processor])
+          actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+          2.times { boss.expect(:async, actor, []) }
+
+          assert_raises TestException do
+            processor.process(msg)
+          end
+        end
+
+        assert_equal 2, failures_count
+
+        Sidekiq.failures_max_count = false
+        assert Sidekiq.failures_max_count == false
+
+        Sidekiq.failures_max_count = nil
+        assert_equal 1000, Sidekiq.failures_max_count
+      end
+
+      it 'returns the total number of failed jobs in the queue' do
+        msg = create_work('class' => MockWorker.to_s, 'args' => ['myarg'], 'failures' => true)
+
+        assert_equal 0, Sidekiq::Failures.count
+
+        actor = MiniTest::Mock.new
+        actor.expect(:processor_done, nil, [@processor])
+        actor.expect(:real_thread, nil, [nil, Celluloid::Thread])
+        @boss.expect(:async, actor, [])
+
+        assert_raises TestException do
+          @processor.process(msg)
+        end
+
+        assert_equal 1, Sidekiq::Failures.count
+      end
+
+      def failures_count
+        Sidekiq.redis { |conn| conn.zcard(LIST_KEY) }
+      end
+
+      def create_work(msg)
+        Sidekiq::BasicFetch::UnitOfWork.new('default', Sidekiq.dump_json(msg))
+      end
+    end
+  end
+end