sidekiq-failures-discourse 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c480f406d6925142f99a976501f2388b6d1e1e00
+  data.tar.gz: c82a6b0f9d58f2d02edd54f5ea329d2ed72ff58a
+SHA512:
+  metadata.gz: 065f2dca2eaaacf33997b7365b2a3a429b69108b68b2c1a88eeb717d78fb06ac4729d34a74b240a72b312668d17d5c00ed8627c45f5e2ac6053c7b6cfea5a0a2
+  data.tar.gz: 22efa3a06a64125f18678fb456b40c1b99653554a7238fec7463c512d2783007d249a5e16b0d81644e296c17f71772c21c121471c9b50bd8200e59e9aca32f7f

data/.gitignore

@@ -0,0 +1,16 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+services:
+  - redis-server
+rvm:
+  - 1.9.3
+  - jruby-19mode
+  - 2.0.0

data/CHANGELOG.md

@@ -0,0 +1,48 @@
+## Unreleased
+
+## 0.3.0
+  * Bump sidekiq dependency to sidekiq >= 2.14.0
+  * Remove slim templates and dependecy
+  * Escape exception info when outputing to html
+  * Add `Sidekiq::Failures.reset_failures` helper method
+  * Add `Sidekiq::Failures.count` helper method (@zanker)
+  * Adhere to sidekiq approach of showing UTC times
+  * Catch all exceptions, not just those that inherit from StandardError (@tylerkovacs)
+  * Fix private method call (@bwthomas)
+
+## 0.2.2
+  * Support ERB for sidekiq >= 2.14.0 (@tobiassvn)
+  * Bump sidekiq dep to >= 2.9.0
+  * Show newest failures first (@PlugIN73)
+  * Fix specs (@krasnoukhov)
+
+## 0.2.1
+  * Fix exhausted mode when retry is disabled (@wr0ngway)
+
+## 0.2.0
+  * Added processor identity to failure data (@krasnoukhov)
+  * Handle Sidekiq::Shutdown exceptions (@krasnoukhov)
+  * Add failures maximum count option (@mathieulaporte)
+  * User Expception#message instead of Exception#to_s (@supaspoida)
+  * Removed web depencies (@LongMan)
+  * Stop overloading find_template (@zquestz)
+
+## 0.1.0
+  * Allow per worker configuration of failure tracking mode. Thanks to
+    @kbaum for most of the work.
+  * Prevent sidekiq-failures from loading up sidekiq/processor (and thus
+    Celluloid actors) except for inside a Sidekiq server context (@cheald)
+  * Fix pagination bug
+  * Add failures default mode option (@kbaum)
+  * Add checkbox option to reset failed counter (@krasnoukhov)
+
+## 0.0.3
+
+  * Adequate layout to new sidekiq web ui design (@krasnoukhov)
+  * Improve backtrace view (@krasnoukhov)
+  * Remove unnecessary slash in redirect (@krasnoukhov)
+  * Invert order of failed jobs (@krasnoukhov)
+
+## 0.0.2
+
+  * Show backtrace similar to resque (Kunal Modi)

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-failures.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,51 @@
+PATH
+  remote: .
+  specs:
+    sidekiq-failures-discourse (0.3.0)
+      sidekiq (>= 2.14.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    connection_pool (2.0.0)
+    hike (1.2.1)
+    json (1.8.1)
+    multi_json (1.7.3)
+    rack (1.4.1)
+    rack-protection (1.2.0)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rake (0.9.2.2)
+    redis (3.0.7)
+    redis-namespace (1.4.1)
+      redis (~> 3.0.4)
+    sidekiq (3.0.0)
+      celluloid (>= 0.15.2)
+      connection_pool (>= 2.0.0)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    sinatra (1.3.3)
+      rack (~> 1.3, >= 1.3.6)
+      rack-protection (~> 1.2)
+      tilt (~> 1.3, >= 1.3.3)
+    sprockets (2.8.1)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    tilt (1.3.3)
+    timers (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rack-test
+  rake
+  sidekiq-failures-discourse!
+  sinatra
+  sprockets

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Marcelo Silveira
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,153 @@
+# Sidekiq::Failures [![Build Status](https://secure.travis-ci.org/mhfs/sidekiq-failures.png)](http://travis-ci.org/mhfs/sidekiq-failures)
+
+Keeps track of Sidekiq failed jobs and adds a tab to the Web UI to let you browse
+them. Makes use of Sidekiq's custom tabs and middleware chain.
+
+It mimics the way Resque keeps track of failures.
+
+WARNING: by default sidekiq-failures will keep up to 1000 failures. See [Maximum Tracked Failures](https://github.com/mhfs/sidekiq-failures#maximum-tracked-failures) below.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-failures'
+```
+
+## Usage
+
+Simply having the gem in your Gemfile is enough to get you started. Your failed
+jobs will be visible via a Failures tab in the Web UI.
+
+## Configuring
+
+### Maximum Tracked Failures
+
+Since each failed job/retry creates a new failure entry that will only be removed
+by you manually, your failures list might consume more resources than you have
+available.
+
+To avoid this sidekiq-failures adopts a default of 1000 maximum tracked failures.
+
+To change the maximum amount:
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.failures_max_count = 5000
+end
+```
+
+To disable the limit entirely:
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.failures_max_count = false
+end
+```
+
+### Failures Tracking Mode
+
+Sidekiq-failures offers three failures tracking options (per worker):
+
+
+#### :all (default)
+
+Tracks failures every time a background job fails. This mean a job with 25 retries
+enabled might generate up to 25 failure entries. If the worker has retry disabled
+only one failure will be tracked.
+
+This is the default behavior but can be made explicit with:
+
+```ruby
+class MyWorker
+  include Sidekiq::Worker
+
+  sidekiq_options :failures => true # or :all
+
+  def perform; end
+end
+```
+
+#### :exhausted
+
+Only track failures if the job exhausts all its retries (or doesn't have retries
+enabled).
+
+You can set this mode as follows:
+
+```ruby
+class MyWorker
+  include Sidekiq::Worker
+
+  sidekiq_options :failures => :exhausted
+
+  def perform; end
+end
+```
+
+#### :off
+
+You can also completely turn off failures tracking for a given worker as follows:
+
+```ruby
+class MyWorker
+  include Sidekiq::Worker
+
+  sidekiq_options :failures => false # or :off
+
+  def perform; end
+end
+```
+
+#### Change the default mode
+
+You can also change the default of all your workers at once by setting the following
+server config:
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.failures_default_mode = :off
+end
+```
+
+The valid modes are `:all`, `:exhausted` or `:off`.
+
+## Helper Methods
+
+### Failures Count
+
+Gives back the number of failed jobs currently stored in Sidekiq Failures. Notice that it's
+different from `Sidekiq` built in failed stat. Also, notice that this might be
+influenced by `failures_max_count`.
+
+```ruby
+Sidekiq::Failures.count
+```
+
+### Reset Failures
+
+Gives a convenient way of reseting Sidekiq Failure stored failed jobs programmatically.
+Takes an options hash and if the `counter` key is present also resets Sidekiq own failed stats.
+
+```ruby
+Sidekiq::Failures.reset_failures
+```
+
+## Dependencies
+
+Depends on Sidekiq >= 2.14.0
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## License
+
+Released under the MIT License. See the [LICENSE][license] file for further details.
+
+[license]: https://github.com/mhfs/sidekiq-failures/blob/master/LICENSE

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs << "lib"
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end

data/lib/sidekiq-failures.rb

@@ -0,0 +1 @@
+require "sidekiq/failures"

data/lib/sidekiq/failures.rb

@@ -0,0 +1,79 @@
+begin
+  require "sidekiq/web"
+rescue LoadError
+  # client-only usage
+end
+
+require "sidekiq/api"
+require "sidekiq/failures/version"
+require "sidekiq/failures/sorted_entry"
+require "sidekiq/failures/failure_set"
+require "sidekiq/failures/middleware"
+require "sidekiq/failures/web_extension"
+
+module Sidekiq
+
+  SIDEKIQ_FAILURES_MODES = [:all, :exhausted, :off].freeze
+
+  # Sets the default failure tracking mode.
+  #
+  # The value provided here will be the default behavior but can be overwritten
+  # per worker by using `sidekiq_options :failures => :mode`
+  #
+  # Defaults to :all
+  def self.failures_default_mode=(mode)
+    unless SIDEKIQ_FAILURES_MODES.include?(mode.to_sym)
+      raise ArgumentError, "Sidekiq#failures_default_mode valid options: #{SIDEKIQ_FAILURES_MODES}"
+    end
+
+    @failures_default_mode = mode.to_sym
+  end
+
+  # Fetches the default failure tracking mode.
+  def self.failures_default_mode
+    @failures_default_mode || :all
+  end
+
+  # Sets the maximum number of failures to track
+  #
+  # If the number of failures exceeds this number the list will be trimmed (oldest
+  # failures will be purged).
+  #
+  # Defaults to 1000
+  # Set to false to disable rotation
+  def self.failures_max_count=(value)
+    @failures_max_count = value
+  end
+
+  # Fetches the failures max count value
+  def self.failures_max_count
+    return 1000 if @failures_max_count.nil?
+
+    @failures_max_count
+  end
+
+  module Failures
+    LIST_KEY = :failed
+
+    def self.reset_failures
+      Sidekiq.redis { |c| c.set("stat:failed", 0) }
+    end
+
+    def self.count
+      Sidekiq.redis {|r| r.zcard(LIST_KEY) }
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.insert_before Sidekiq::Middleware::Server::RetryJobs,
+                        Sidekiq::Failures::Middleware
+  end
+end
+
+if defined?(Sidekiq::Web)
+  Sidekiq::Web.register Sidekiq::Failures::WebExtension
+  Sidekiq::Web.tabs["Failures"] = "failures"
+  Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "failures/locales")
+end

data/lib/sidekiq/failures/failure_set.rb

@@ -0,0 +1,22 @@
+module Sidekiq
+  module Failures
+    Superclass =
+      if defined?(Sidekiq::JobSet)
+        Sidekiq::JobSet
+      else
+        Sidekiq::SortedSet
+      end
+
+    class FailureSet < Superclass
+      def initialize
+        super LIST_KEY
+      end
+
+      def retry_all_failures
+        while size > 0
+          each(&:retry_failure)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/failures/locales/en.yml

@@ -0,0 +1,5 @@
+en:
+  FailedJobs: Failed Jobs
+  FailedAt: Failed At
+  ResetCounter: Reset Counter
+  NoFailedJobsFound: No failed jobs found

data/lib/sidekiq/failures/middleware.rb

@@ -0,0 +1,99 @@
+module Sidekiq
+  module Failures
+
+    class Middleware
+      attr_accessor :msg
+
+      def call(worker, msg, queue)
+        self.msg = msg
+        yield
+      rescue Sidekiq::Shutdown
+        raise
+      rescue Exception => e
+        raise e if skip_failure?
+
+        msg['error_message'] = e.message
+        msg['error_class'] = e.class.name
+        msg['processor'] = identity
+        msg['failed_at'] = Time.now.utc.to_f
+
+        if msg['backtrace'] == true
+          msg['error_backtrace'] = e.backtrace
+        elsif msg['backtrace'] == false
+          # do nothing
+        elsif msg['backtrace'].to_i != 0
+          msg['error_backtrace'] = e.backtrace[0..msg['backtrace'].to_i]
+        end
+
+        payload = Sidekiq.dump_json(msg)
+        Sidekiq.redis do |conn|
+          conn.zadd(LIST_KEY, Time.now.utc.to_f, payload)
+          unless Sidekiq.failures_max_count == false
+            conn.zremrangebyrank(LIST_KEY, 0, -(Sidekiq.failures_max_count + 1))
+          end
+        end
+
+        raise e
+      end
+
+      private
+
+      def failure_mode_off?
+        failure_mode == :off
+      end
+
+      def failure_mode_exhausted?
+        failure_mode == :exhausted
+      end
+
+      def skip_failure?
+        failure_mode_off? || failure_mode_exhausted? && !exhausted?
+      end
+
+      def failure_mode
+        case msg['failures'].to_s
+        when 'true', 'all'
+          :all
+        when 'false', 'off'
+          :off
+        when 'exhausted'
+          :exhausted
+        else
+          Sidekiq.failures_default_mode
+        end
+      end
+
+      def exhausted?
+        !retriable? || retry_count >= max_retries
+      end
+
+      def retriable?
+        msg['retry']
+      end
+
+      def retry_count
+        msg['retry_count'] || 0
+      end
+
+      def max_retries
+        retry_middleware.send(:retry_attempts_from, msg['retry'], default_max_retries)
+      end
+
+      def retry_middleware
+        @retry_middleware ||= Sidekiq::Middleware::Server::RetryJobs.new
+      end
+
+      def default_max_retries
+        Sidekiq::Middleware::Server::RetryJobs::DEFAULT_MAX_RETRY_ATTEMPTS
+      end
+
+      def hostname
+        Socket.gethostname
+      end
+
+      def identity
+        @@identity ||= "#{hostname}:#{$$}"
+      end
+    end
+  end
+end