sidekiq-expected_failures 0.0.1 → 0.2.0

data/.travis.yml

@@ -4,5 +4,8 @@ rvm:
   - 1.9.3
   - 2.0.0
 
+services:
+  - redis-server
+
 notifications:
   email: false

data/CHANGELOG.md

@@ -1,3 +1,37 @@
+## 0.2.0
+
+- [**breaking change**] ability to use Sidekiq's build-in `handle_exception`
+  method - in case you want to use airbrake or other exception notify service.
+  Since version `0.2.0` you need to provide  `expected_failures` in a form of
+  a hash, for example:
+
+``` ruby
+class CustomizedWorker
+  include ::Sidekiq::Worker
+  sidekiq_options expected_failures: {
+    NotImplementedError      => nil,          # notification disabled
+    VeryOwn::CustomException => [10, 20, 50], # notify on 10th, 20th, 50th failure
+    ZeroDivisionError        => 5             # notify on 5th failure
+  }
+end
+```
+
+- removed `sinatra-assetpack` dependency - js assets are now served inline
+  (it seemed like a overkill to include that gem just for just two files)
+
+- added option to configure exception handled by default (for all workers):
+
+``` ruby
+  Sidekiq.configure_server do |config|
+    config.expected_failures = { AlwaysHandledExceptionByDefault => 1000 }
+  end
+```
+
+Note: if you specify `expected_failure`s for given worker defaults will be
+discarded (for that worker).
+
+- small front-end adjustments
+
 ## 0.0.1
 
 - Initial release

data/README.md

@@ -2,6 +2,7 @@
 
 [![Code Climate](https://codeclimate.com/github/emq/sidekiq-expected_failures.png)](https://codeclimate.com/github/emq/sidekiq-expected_failures)
 [![Build Status](https://travis-ci.org/emq/sidekiq-expected_failures.png?branch=master)](https://travis-ci.org/emq/sidekiq-expected_failures)
+[![Coverage Status](https://coveralls.io/repos/emq/sidekiq-expected_failures/badge.png)](https://coveralls.io/r/emq/sidekiq-expected_failures)
 
 If you don't rely on standard sidekiq's retry behavior and you want to track exceptions, that will happen one way, or another - this thing is for you.
 
@@ -26,7 +27,7 @@ Let's say you do a lot of API requests to some not reliable reliable service. In
 ``` ruby
 class ApiCallWorker
   include ::Sidekiq::Worker
-  sidekiq_options expected_failures: [Timeout::Error]
+  sidekiq_options expected_failures: { Timeout::Error => nil } # handle that exception, but disable notification
 
   def perform(arguments)
     # do some work
@@ -42,12 +43,29 @@ class ApiCallWorker
   end
 ```
 
-You can pass array of exceptions to handle inside `sidekiq_options`. This is how web interface looks like:
+You can pass a hash of exceptions to handle inside `sidekiq_options`. Each key-value pair may consist of:
+- `exception => nil` - notifications disabled
+- `exception => integer` - fires exception notify when x-th exception happens (on daily basis)
+- `exception => [integer, integer]` - same as above but for each value
+
+sidekiq-expected_failures utilizes sidekiq's [ExceptionHandler module][1] - so you might want to set some same limits for your exceptions and use Airbrake (for example) as a notification service to inform you that something bad is probably happing.
+
+This is how web interface looks like:
 
 ![](http://i.imgur.com/7Fe8voD.jpg)
 
 It logs each failed jobs to to redis list (per day) and keep global counters (per exception class as a single redis hash).
 
+### Default expected failures
+
+You can configure defaults for all your workers (overridden completely by specifying `expected_failures` hash inside `sidekiq_options` - per worker).
+
+``` ruby
+Sidekiq.configure_server do |config|
+  config.expected_failures = { ExceptionHandledByDefault => [1000, 2000] } # with notification enabled
+end
+```
+
 ### Usage with sidekiq-failures
 
 Just be sure to load this one after `sidekiq-failures`, otherwise failed jobs will end up logged twice - and you probably don't want that.
@@ -69,3 +87,5 @@ This might change in the future to something more sane.
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+[1]: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/exception_handler.rb#L4

data/lib/sidekiq/expected_failures/middleware.rb

@@ -3,35 +3,47 @@ module Sidekiq
     class Middleware
       include Sidekiq::Util
 
+      attr_reader :handled_exceptions
+
       def call(worker, msg, queue)
-        exceptions = worker.class.get_sidekiq_options['expected_failures'].to_a
+        setup_exceptions(worker)
 
         yield
 
-        rescue *exceptions => e
+        rescue *handled_exceptions.keys => ex
           data = {
             failed_at: Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
             args:      msg['args'],
-            exception: e.class.to_s,
-            error:     e.message,
+            exception: ex.class.to_s,
+            error:     ex.message,
             worker:    msg['class'],
             processor: "#{hostname}:#{process_id}-#{Thread.current.object_id}",
             queue:     queue
           }
 
-          log_exception(data)
+          log_exception(data, ex, msg)
       end
 
       private
 
-        def log_exception(data)
-          Sidekiq.redis do |conn|
+        def setup_exceptions(worker)
+          @handled_exceptions = worker.class.get_sidekiq_options['expected_failures'] || Sidekiq.expected_failures
+        end
+
+        def exception_intervals(ex)
+          [handled_exceptions[ex.class]].flatten.compact
+        end
+
+        def log_exception(data, ex, msg)
+          result = Sidekiq.redis do |conn|
             conn.multi do |m|
               m.lpush("expected:#{today}", Sidekiq.dump_json(data))
               m.sadd("expected:dates", today)
               m.hincrby("expected:count", data[:exception], 1)
             end
           end
+
+          handle_exception(ex, msg) if exception_intervals(ex).include?(result[0])
         end
 
         def today

data/lib/sidekiq/expected_failures/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module ExpectedFailures
-    VERSION = "0.0.1"
+    VERSION = "0.2.0"
   end
 end

data/lib/sidekiq/expected_failures/web.rb

@@ -1,20 +1,9 @@
-require 'sinatra/assetpack'
-
 module Sidekiq
   module ExpectedFailures
     module Web
 
       def self.registered(app)
-        web_dir    = File.expand_path("../../../../web", __FILE__)
-        assets_dir = File.join(web_dir, "assets")
-
-        app.register Sinatra::AssetPack
-
-        app.assets do
-          serve '/js', from: assets_dir
-          js 'expected',  ['/js/expected.js']
-          js 'bootstrap', ['/js/bootstrap.js']
-        end
+        web_dir = File.expand_path("../../../../web", __FILE__)
 
         app.helpers do
           def link_to_details(job)
@@ -24,14 +13,9 @@ module Sidekiq
           end
         end
 
-        app.get "/expected_failures/clear/:what" do
-          case params[:what]
-          when 'old'
-            Sidekiq::ExpectedFailures.clear_old
-          when 'all'
-            Sidekiq::ExpectedFailures.clear_all
-          when 'counters'
-            Sidekiq::ExpectedFailures.clear_counters
+        app.post "/expected_failures/clear" do
+          if %w(old all counters).include?(params[:what])
+            Sidekiq::ExpectedFailures.send("clear_#{params[:what]}")
           end
 
           redirect "#{root_path}expected_failures"
@@ -48,6 +32,10 @@ module Sidekiq
             @counters = Sidekiq::ExpectedFailures.counters
           end
 
+          @javascript = %w(expected bootstrap).map do |file|
+            File.read(File.join(web_dir, "assets/#{file}.js"))
+          end.join
+
           erb File.read(File.join(web_dir, "views/expected_failures.erb"))
         end
       end

data/lib/sidekiq/expected_failures.rb

@@ -4,6 +4,15 @@ require "sidekiq/expected_failures/middleware"
 require "sidekiq/expected_failures/web"
 
 module Sidekiq
+
+  def self.expected_failures=(exceptions)
+    @expected_failures = exceptions
+  end
+
+  def self.expected_failures
+    @expected_failures || {}
+  end
+
   module ExpectedFailures
 
     def self.dates

data/sidekiq-expected_failures.gemspec

@@ -18,11 +18,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "sidekiq", ">= 2.15.0"
-  spec.add_dependency "sinatra-assetpack", "~> 0.3.1"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "sinatra"
   spec.add_development_dependency "rack-test"
   spec.add_development_dependency "timecop", "~> 0.6.3"
+  spec.add_development_dependency "mocha", "~> 0.14.0"
+  spec.add_development_dependency "coveralls", "~> 0.7.0"
 end

data/test/lib/middleware_test.rb

@@ -8,6 +8,7 @@ module Sidekiq
         Sidekiq.redis = REDIS
         Sidekiq.redis { |c| c.flushdb }
         Timecop.freeze(Time.local(2013, 1, 10))
+        Sidekiq.expected_failures = nil
       end
 
       after { Timecop.return }
@@ -23,44 +24,53 @@ module Sidekiq
         end
       end
 
+      it 'can can be configured to handle exceptions by default' do
+        Sidekiq.expected_failures =  { VeryOwn::CustomException => nil }
+
+        handler.call(RegularWorker.new, msg, 'default') do
+          raise VeryOwn::CustomException
+        end
+
+        assert_raises RuntimeError do
+          handler.call(RegularWorker.new, msg, 'default') do
+            raise "This is not handled by default"
+          end
+        end
+      end
+
       it 'respects build-in rescue and ensure blocks' do
         assert_equal 0, $invokes
-        assert_block do
-          handler.call(SingleExceptionWorker.new, msg, 'default') do
-            begin
-            raise ZeroDivisionError.new("We go a problem, sir")
-            rescue ZeroDivisionError => e
-               $invokes += 1
-               raise e # and now this should be caught by middleware
-            ensure
-              $invokes += 1
-            end
+
+        handler.call(SingleExceptionWorker.new, msg, 'default') do
+          begin
+          raise ZeroDivisionError.new("We go a problem, sir")
+          rescue ZeroDivisionError => e
+             $invokes += 1
+             raise e # and now this should be caught by middleware
+          ensure
+            $invokes += 1
           end
         end
+
         assert_equal 2, $invokes
       end
 
       it 'handles all specified exceptions' do
-        assert_block do
-          handler.call(MultipleExceptionWorker.new, msg, 'default') do
-            raise NotImplementedError
-          end
+        handler.call(MultipleExceptionWorker.new, msg, 'default') do
+          raise NotImplementedError
         end
 
-        assert_block do
-          handler.call(MultipleExceptionWorker.new, msg, 'default') do
-            raise VeryOwn::CustomException
-          end
+        handler.call(MultipleExceptionWorker.new, msg, 'default') do
+          raise VeryOwn::CustomException
         end
       end
 
       it 'logs exceptions' do
-        assert_block do
-          handler.call(SingleExceptionWorker.new, msg, 'default') do
-            raise ZeroDivisionError
-          end
+        handler.call(SingleExceptionWorker.new, msg, 'default') do
+          raise ZeroDivisionError
         end
 
+
         assert_equal(['2013-01-10'], redis("smembers", "expected:dates"))
         assert_match(/custom_argument/, redis("lrange", "expected:2013-01-10", 0, -1)[0])
       end
@@ -100,6 +110,40 @@ module Sidekiq
         assert_equal 5, redis("llen", "expected:2013-05-15")
         assert_equal(['2013-05-15', '2013-01-10'].sort, redis("smembers", "expected:dates").sort)
       end
+
+      describe 'exception notify' do
+
+        it 'can be configured to notify once' do
+          exception = ZeroDivisionError.new
+          handler.expects(:handle_exception).with(exception, msg).once.returns(true)
+
+          50.times do
+            handler.call(CustomizedWorker.new, msg, 'default') do
+              raise exception
+            end
+          end
+        end
+
+        it 'can be configured to notify multiple number of times' do
+          handler.expects(:handle_exception).times(3).returns(true)
+
+          60.times do
+            handler.call(CustomizedWorker.new, msg, 'default') do
+              raise VeryOwn::CustomException
+            end
+          end
+        end
+
+        it 'can be configured not to notify at all' do
+          handler.expects(:handle_exception).never
+
+          60.times do
+            handler.call(CustomizedWorker.new, msg, 'default') do
+              raise NotImplementedError
+            end
+          end
+        end
+      end
     end
   end
 end

data/test/lib/web_test.rb

@@ -9,10 +9,6 @@ module Sidekiq
       Sidekiq::Web
     end
 
-    def failed_count
-      Sidekiq.redis { |c| c.get("stat:failed") }
-    end
-
     def create_sample_counter
       redis("hset", "expected:count", "StandardError", 5)
       redis("hset", "expected:count", "Custom::Error", 10)
@@ -80,7 +76,7 @@ module Sidekiq
         get '/expected_failures'
         last_response.body.must_match(/HardWorker/)
 
-        get '/expected_failures/clear/all'
+        post '/expected_failures/clear', { what: 'all' }
         last_response.status.must_equal(302)
         last_response.location.must_match(/expected_failures$/)
 
@@ -93,7 +89,7 @@ module Sidekiq
         last_response.body.must_match(/2013-09-10/)
         last_response.body.must_match(/2013-09-09/)
 
-        get '/expected_failures/clear/old'
+        post '/expected_failures/clear', { what: 'old' }
         last_response.status.must_equal(302)
         last_response.location.must_match(/expected_failures$/)
 
@@ -111,7 +107,7 @@ module Sidekiq
           create_sample_failure
           get '/expected_failures'
           last_response.body.wont_match(/dl-horizontal/)
-          last_response.body.wont_match(/Clear counters/i)
+          last_response.body.wont_match(/All counters/i)
         end
       end
 
@@ -121,14 +117,14 @@ module Sidekiq
         it 'displays counters' do
           get '/expected_failures'
           last_response.body.must_match(/dl-horizontal/)
-          last_response.body.must_match(/Clear counters/i)
+          last_response.body.must_match(/All counters/i)
         end
 
         it 'can clear counters' do
           get '/expected_failures'
           last_response.body.must_match(/Custom::Error/)
 
-          get '/expected_failures/clear/counters'
+          post '/expected_failures/clear', { what: 'counters' }
           last_response.status.must_equal(302)
           last_response.location.must_match(/expected_failures$/)
 

data/test/test_helper.rb

@@ -1,9 +1,15 @@
+require 'coveralls'
+Coveralls.wear!
+
 Encoding.default_external = Encoding::UTF_8
 Encoding.default_internal = Encoding::UTF_8
 
 require "minitest/autorun"
 require "minitest/spec"
 require "minitest/mock"
+require "minitest/pride"
+require "mocha/setup"
+
 
 require "timecop"
 require "rack/test"

data/test/test_workers.rb

@@ -8,10 +8,19 @@ end
 
 class SingleExceptionWorker
   include ::Sidekiq::Worker
-  sidekiq_options expected_failures: [ZeroDivisionError]
+  sidekiq_options expected_failures: { ZeroDivisionError => nil }
 end
 
 class MultipleExceptionWorker
   include ::Sidekiq::Worker
-  sidekiq_options expected_failures: [NotImplementedError, VeryOwn::CustomException]
+  sidekiq_options expected_failures: { NotImplementedError => nil, VeryOwn::CustomException => nil }
+end
+
+class CustomizedWorker
+  include ::Sidekiq::Worker
+  sidekiq_options expected_failures: {
+    NotImplementedError      => nil,
+    VeryOwn::CustomException => [10, 20, 50],
+    ZeroDivisionError        => 5
+  }
 end

data/web/assets/expected.js

@@ -15,7 +15,7 @@
     e.preventDefault();
   });
 
-  $('#filter-jobs select').live('change', function(e){
+  $('#filter-jobs select, #clear-jobs select').live('change', function(e){
     $(this).parent('form').submit();
   });
  });

data/web/views/expected_failures.erb

@@ -1,5 +1,6 @@
-<%= js :expected %>
-<%= js :bootstrap %>
+<script type="text/javascript">
+  <%= @javascript %>
+</script>
 
 <h3>Expected failures log
   <% if @date %>
@@ -8,9 +9,7 @@
 </h3>
 
 <% unless @counters.empty? %>
-  <div class="well well-sm clearfix">
-    <a href="<%= root_path %>expected_failures/clear/counters" class="btn btn-default btn-sm pull-right">Clear counters</span></a>
-
+  <div class="well well-sm">
     <dl class="dl-horizontal" style="margin: 0">
       <% @counters.each do |exception, count| %>
         <dt><%= exception %></dt>
@@ -21,6 +20,26 @@
   </div>
 <% end %>
 
+<% if @jobs.any? || @counters.any? %>
+  <form id="clear-jobs" method="post" class="form-inline pull-right" action="<%= root_path %>expected_failures/clear">
+    <label>Clear:</label>
+    <select name="what">
+      <option selected>Choose...</option>
+      <% if @jobs.any? %>
+        <optgroup label="Jobs">
+          <option value="old">Older than today</option>
+          <option value="all">All failed</option>
+        </optgroup>
+      <% end %>
+      <% if @counters.any? %>
+        <optgroup label="Counters">
+          <option value="counters">All counters</option>
+        </optgroup>
+      <% end %>
+    </select>
+  </form>
+<% end %>
+
 <% if @jobs.any? %>
   <div class="modal fade" id="job-details">
     <div class="modal-dialog">
@@ -56,13 +75,10 @@
     </select>
   </form>
 
-  <div class="pull-right">
-    <a href="<%= root_path %>expected_failures/clear/old" class="btn btn-default btn-sm">Clear older than today</a>&nbsp;
-    <a href="<%= root_path %>expected_failures/clear/all" class="btn btn-default btn-sm">Clear all failed</a>
-  </div>
-
   <p class="clearfix"></p>
 
+  <%= erb :_paging, :locals => { :url => "#{root_path}expected_failures/#{@date}" } %>
+
   <table id="expected" class="queues table table-hover table-bordered table-striped table-white">
     <thead>
       <th>Datetime</th>
@@ -85,6 +101,7 @@
   <%= erb :_paging, :locals => { :url => "#{root_path}expected_failures/#{@date}" } %>
 
 <% else %>
+  <p class="clearfix"></p>
   <div class="alert alert-success">
     No failed jobs found.
   </div>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-expected_failures
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-29 00:00:00.000000000 Z
+date: 2013-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -27,22 +27,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.15.0
-- !ruby/object:Gem::Dependency
-  name: sinatra-assetpack
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.3.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.3.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -123,6 +107,38 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.3
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
 description: If you don't rely on sidekiq' retry behavior, you handle exceptions on
   your own and want to keep track of them - this thing is for you.
 email:
@@ -166,7 +182,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3544806089198755601
+      hash: 3929689485539629877
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -175,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3544806089198755601
+      hash: 3929689485539629877
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25