freekiqs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 638d7e5e400a98fe5dbde8e7c145c8caf279a145
+  data.tar.gz: 967ad2c2b650ed7a27f4573dfdaaf3ac61070e0c
+SHA512:
+  metadata.gz: 01e4a44ec9eebe9e8328f258c69c18ff42ce1e7c8cc50d0166b958dab857a95e5cf30e5c2bac0fcb1af5bcdd0a37817b14bf9d5df5437927a37446760bdf523e
+  data.tar.gz: 65a722fce7bd7eaf1dc575dcf227a3d33efaaf9fa81f05989b70e4349ac5d92eff246d672bf0897fd6c25955bbdc5bc808e368437bbbd2a1fd83bf4adb25ca73

data/.gitignore

@@ -0,0 +1,5 @@
+tmp
+gemfiles/*.lock
+Gemfile.lock
+.ruby-version
+pkg

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'rake'
+
+group :test do
+  gem 'rspec', '~> 3.1'
+end

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2016 BookBub
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Freekiqs
+
+Sidekiq middleware that allows capturing exceptions thrown
+by failed jobs and wrapping them with a `FreekiqException` exception class
+that can be filtered by monitoring tools such as New Relic and
+Honeybadger.
+
+Configuration example (in config/initializers/sidekiq.rb):
+``` ruby
+  Sidekiq.configure_server do |config|
+    config.server_middleware do |chain|
+      chain.insert_after Sidekiq::Middleware::Server::RetryJobs, Sidekiq::Middleware::Server::Freekiqs
+    end
+  end
+```
+
+Worker example:
+``` ruby
+  class MyWorker
+    include Sidekiq::Worker
+    sidekiq_options freekiqs: 3
+      #
+    def perform(param)
+      ...
+    end
+  end
+```
+
+Freekiqs is disabled by default. It can be enabled per-worker
+by setting `:freekiqs` on `sidekiq_options`. Or, it can be
+enabled globally by adding `:freekiqs` to the middleware
+registration.
+
+Example:
+``` ruby
+  Sidekiq.configure_server do |config|
+    config.server_middleware do |chain|
+      chain.insert_after Sidekiq::Middleware::Server::RetryJobs, Sidekiq::Middleware::Server::Freekiqs, freekiqs: 3
+    end
+  end
+```
+
+## Overview
+
+Up to the configured number of "freekiqs", catch exceptions thrown
+from job and wrap them with the `FreekiqException` exception (which is a
+`RuntimeError`). That exception type can be ignored by monitoring
+tools. If job fails more times than configured "freekiqs", thrown
+exception will not be wrapped. That should result in normal operation
+and the exception showing up in monitoring tools.
+
+Implementation Details:
+This relies on Sidekiq's built-in retry handling. Specifically, its
+`retry_count` value. When a job first fails, its `retry_count` value
+is nil (because it hasn't actually been retried yet). That exception,
+along with subsequent exceptions from retries, are caught and wrapped
+with the `FreekiqException` exception.
+Cases where Freekiqs does NOT wrap the exception:
+ - The `retry` option is false
+ - The `freekiqs` option is not set on the worker nor globally
+ - The `freekiqs` option is not set on worker and set to `false` globally
+ - The `freekiqs` option is set to `false` on the worker
+ - The job threw an exception that is not a StandardError (nor a subclass)
+ - The number of thrown exceptions is more than specified freekiqs

data/Rakefile

@@ -0,0 +1,7 @@
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError
+end
+
+task default: :spec

data/freekiqs.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name                  = 'freekiqs'
+  gem.version               = '0.1.0'
+  gem.authors               = ['Rob Lewis']
+  gem.email                 = ['rob@bookbub.com']
+  gem.summary               = 'Sidekiq middleware extending RetryJobs to allow silient errors.'
+  gem.description           = 'Sidekiq middleware extending RetryJobs to allow configuring ' \
+                              'how many exceptions a job can throw and be wrapped by a silenceable exception.'
+  gem.homepage              = 'https://github.com/BookBub/freekiqs'
+  gem.license               = 'MIT'
+  gem.executables           = []
+  gem.files                 = `git ls-files`.split("\n")
+  gem.test_files            = `git ls-files -- spec/*`.split("\n")
+  gem.require_paths         = ['lib']
+  gem.required_ruby_version = '>= 1.9.3'
+
+  gem.add_dependency             'sidekiq', '>= 1.0.0', '< 4.0.0'
+  gem.add_development_dependency 'rspec', '~> 2.14.1'
+end

data/lib/freekiqs.rb

@@ -0,0 +1 @@
+require 'sidekiq/middleware/server/freekiqs'

data/lib/sidekiq/middleware/server/freekiqs.rb

@@ -0,0 +1,46 @@
+require 'sidekiq'
+require 'sidekiq/util'
+
+module Sidekiq
+  class FreekiqException < RuntimeError; end
+
+  module Middleware
+    module Server
+      class Freekiqs
+        include Sidekiq::Util
+
+        def initialize(opts={})
+          @default_freekiqs = opts[:freekiqs]
+        end
+
+        def call(worker, msg, queue)
+          yield
+        rescue => ex
+          freekiqs = get_freekiqs_if_enabled(worker, msg)
+          if freekiqs
+            if msg['retry_count'].nil? || msg['retry_count'] < freekiqs-1
+              raise FreekiqException, ex.message
+            else
+              Sidekiq.logger.info { "Out of free kiqs for #{msg['class']} job #{msg['jid']}" }
+            end
+          end
+          raise ex
+        end
+
+        def get_freekiqs_if_enabled(worker, msg)
+          freekiqs = nil
+          if msg['retry']
+            if worker.class.sidekiq_options['freekiqs'] != false
+              if worker.class.sidekiq_options['freekiqs']
+                freekiqs = worker.class.sidekiq_options['freekiqs'].to_i
+              elsif @default_freekiqs
+                freekiqs = @default_freekiqs.to_i
+              end
+            end
+          end
+          freekiqs
+        end
+      end
+    end
+  end
+end

data/spec/freekiqs_spec.rb

@@ -0,0 +1,168 @@
+require 'spec_helper'
+require 'celluloid'  # Getting error without this required. Should remove once this supports Sidekiq 4.x
+require 'sidekiq/cli'
+require 'sidekiq/middleware/server/retry_jobs'
+
+describe Sidekiq::Middleware::Server::Freekiqs do
+  class DummyWorkerPlain
+    include Sidekiq::Worker
+  end
+  class DummyWorkerWithFreekiqsEnabled
+    include Sidekiq::Worker
+    sidekiq_options freekiqs: 2
+  end
+  class DummyWorkerWithFreekiqsDisabled
+    include Sidekiq::Worker
+    sidekiq_options freekiqs: false
+  end
+
+  def build_handler_chain(freekiq_options={}, retry_options={})
+    Sidekiq::Middleware::Chain.new do |chain|
+      chain.add Sidekiq::Middleware::Server::RetryJobs, retry_options
+      chain.add Sidekiq::Middleware::Server::Freekiqs, freekiq_options
+    end
+  end
+
+  def build_job(options={})
+    {'class' => 'FreekiqDummyWorker', 'args' => [], 'retry' => true}.merge(options)
+  end
+
+  def cleanup_redis
+    Sidekiq::RetrySet.new.select{|job| job.klass == 'FreekiqDummyWorker'}.each(&:delete)
+    Sidekiq::DeadSet.new.clear
+  end
+
+  before(:each) do
+    cleanup_redis
+  end
+
+  let(:worker_plain)                  { DummyWorkerPlain.new }
+  let(:worker_with_freekiqs_enabled)  { DummyWorkerWithFreekiqsEnabled.new }
+  let(:worker_with_freekiqs_disabled) { DummyWorkerWithFreekiqsDisabled.new }
+
+  it 'requires RetryJobs to update retry_count' do
+    handler = Sidekiq::Middleware::Server::RetryJobs.new
+    worker = worker_plain
+    job = build_job
+    expect {
+      handler.call(worker, job, 'default') do
+        raise 'Oops'
+      end
+    }.to raise_error(RuntimeError)
+    expect(job['retry_count']).to eq(0)
+    expect {
+      handler.call(worker, job, 'default') do
+        raise 'Oops'
+      end
+    }.to raise_error(RuntimeError)
+    expect(job['retry_count']).to eq(1)
+  end
+
+  it 'throws Freekiq exception for specified number of free kiqs' do
+    handler = build_handler_chain
+    worker = worker_with_freekiqs_enabled
+    job = build_job
+
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'overlooked'
+      end
+    }.to raise_error(Sidekiq::FreekiqException, 'overlooked')
+    expect(job['retry_count']).to eq(0)
+    expect(job['error_class']).to eq('Sidekiq::FreekiqException')
+    expect(job['error_message']).to eq('overlooked')
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'overlooked'
+      end
+    }.to raise_error(Sidekiq::FreekiqException, 'overlooked')
+    expect(job['retry_count']).to eq(1)
+    expect(job['error_class']).to eq('Sidekiq::FreekiqException')
+    expect(job['error_message']).to eq('overlooked')
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'not overlooked'
+      end
+    }.to raise_error(ArgumentError, 'not overlooked')
+    expect(job['retry_count']).to eq(2)
+    expect(job['error_class']).to eq('ArgumentError')
+    expect(job['error_message']).to eq('not overlooked')
+    expect(Sidekiq::RetrySet.new.size).to eq(3)
+    expect(Sidekiq::DeadSet.new.size).to eq(0)
+  end
+
+  it 'allows a freekiqs option in initializer' do
+    handler = build_handler_chain(freekiqs: 1)
+    worker = worker_plain
+    job = build_job
+
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'overlooked'
+      end
+    }.to raise_error(Sidekiq::FreekiqException, 'overlooked')
+    expect(job['retry_count']).to eq(0)
+    expect(job['error_class']).to eq('Sidekiq::FreekiqException')
+    expect(job['error_message']).to eq('overlooked')
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'not overlooked'
+      end
+    }.to raise_error(ArgumentError, 'not overlooked')
+    expect(job['retry_count']).to eq(1)
+    expect(job['error_class']).to eq('ArgumentError')
+    expect(job['error_message']).to eq('not overlooked')
+    expect(Sidekiq::RetrySet.new.size).to eq(2)
+    expect(Sidekiq::DeadSet.new.size).to eq(0)
+  end
+
+  it 'allows explicitly disabling freekiqs' do
+    handler = build_handler_chain(freekiqs: 3)
+    worker = worker_with_freekiqs_disabled
+    job = build_job
+
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'not overlooked'
+      end
+    }.to raise_error(ArgumentError, 'not overlooked')
+    expect(job['retry_count']).to eq(0)
+    expect(job['error_class']).to eq('ArgumentError')
+    expect(job['error_message']).to eq('not overlooked')
+    expect(Sidekiq::RetrySet.new.size).to eq(1)
+    expect(Sidekiq::DeadSet.new.size).to eq(0)
+  end
+
+  it 'does nothing if not explicitly enabled' do
+    handler = build_handler_chain
+    worker = worker_plain
+    job = build_job
+
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'not overlooked'
+      end
+    }.to raise_error(ArgumentError, 'not overlooked')
+    expect(job['retry_count']).to eq(0)
+    expect(job['error_class']).to eq('ArgumentError')
+    expect(job['error_message']).to eq('not overlooked')
+  end
+
+  it 'does nothing if retries disabled' do
+    handler = build_handler_chain
+    worker = worker_with_freekiqs_enabled
+    job = build_job('retry' => false)
+
+    expect {
+      handler.invoke(worker, job, 'default') do
+        raise ArgumentError, 'not overlooked'
+      end
+    }.to raise_error(ArgumentError, 'not overlooked')
+    expect(job['retry_count']).to be_nil
+    expect(job['error_class']).to be_nil
+    expect(job['error_message']).to be_nil
+    expect(job['error_message']).to be_nil
+    expect(Sidekiq::RetrySet.new.size).to eq(0)
+    expect(Sidekiq::DeadSet.new.size).to eq(0)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+require 'freekiqs'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: freekiqs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Rob Lewis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+description: Sidekiq middleware extending RetryJobs to allow configuring how many
+  exceptions a job can throw and be wrapped by a silenceable exception.
+email:
+- rob@bookbub.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- freekiqs.gemspec
+- lib/freekiqs.rb
+- lib/sidekiq/middleware/server/freekiqs.rb
+- spec/freekiqs_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/BookBub/freekiqs
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Sidekiq middleware extending RetryJobs to allow silient errors.
+test_files:
+- spec/freekiqs_spec.rb
+- spec/spec_helper.rb