attentive_sidekiq 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a5dd7d7255704b9f2a4c0a17a494457f9827aac
-  data.tar.gz: 989e2e842e2de465df0c23235861c66da5eaa8eb
+  metadata.gz: 710aa8d08d27aece645bef61d22be38bdf5424f5
+  data.tar.gz: e8dde883933212b7ea5659749d12cb810005bc2d
 SHA512:
-  metadata.gz: decd8b0dd325a36d57d74ed5635203b6ddc5da90ea6f3faa6dafa222ede9e6b6c2d1332802b4b9d904c5cdb1586d86919989660fe9f2b5a9a90086859ed8fdab
-  data.tar.gz: ef514362d5011d880cae7ab7ee70eda9f39738719e022c542549247033768f38e19faa354cb1f904528a60390f27b6384dff71936065c9aae854cafc553ae683
+  metadata.gz: fd6587c4312f1e2cf96cf2bc106f509c3464f871fcb8acd08faf031ad1dd207d85ddac21d9e5eacde83dcb7e8a96f162238801d497d88bab6b284977e8775e27
+  data.tar.gz: 629fb700fca7a68df8054624a648319db57feb9a0f154a4a7b38a6c5a2b87df3f09888f2e13c8474455d91ffdb2956c9935741ea54288968d9267a71821cbeff

data/Gemfile.lock

@@ -1,25 +1,23 @@
 PATH
   remote: .
   specs:
-    attentive_sidekiq (0.1.0)
+    attentive_sidekiq (0.3.0)
       activesupport
       concurrent-ruby (~> 1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.5)
+    activesupport (5.0.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
-      json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     byebug (9.0.6)
     coderay (1.1.1)
     concurrent-ruby (1.0.2)
     connection_pool (2.2.1)
     i18n (0.7.0)
-    json (1.8.3)
     method_source (0.8.2)
     minitest (5.9.1)
     minitest-stub_any_instance (1.0.1)

data/README.md

@@ -1,24 +1,21 @@
-# Attentive Sidekiq
+Attentive Sidekiq
+===========================
 
 [![Gem Version](https://badge.fury.io/rb/attentive_sidekiq.svg)](https://badge.fury.io/rb/attentive_sidekiq)
 [![Code Climate](https://codeclimate.com/github/twonegatives/attentive_sidekiq/badges/gpa.svg)](https://codeclimate.com/github/twonegatives/attentive_sidekiq)
 [![CircleCI](https://circleci.com/gh/twonegatives/attentive_sidekiq.svg?style=shield)](https://circleci.com/gh/twonegatives/attentive_sidekiq)
 
+Be aware of suddenly disappeared sidekiq jobs!
 
-### Motivation
-Bad things happen sometimes. Sidekiq process being killed in the middle of job processing may lead to job losage. In other words, that job will never be returned to the queue.
-This case was proved by [github issues](https://github.com/mperham/sidekiq/issues/1831), [stackoverflow questions](http://stackoverflow.com/questions/35555000/current-sidekiq-job-lost-when-deploying-to-heroku) and (sadly) personal experience, and means that you are not safe from losing critical user data.
-Sidekiq's author, Mike Perham, suggests purchasing Sidekiq Pro which uses another fetch mechanism. However, it [is reported](https://github.com/mperham/sidekiq/issues/2531) to still cause the same issue.
+![shortsummary](https://cloud.githubusercontent.com/assets/1937799/20489411/fe726e82-b023-11e6-9528-7df519fec7dd.gif)
 
-### About
-Attentive Sidekiq is a sidekiq plugin which makes one more step to guard your jobs from being lost (well, at least to be notified of this).
-It saves info about started jobs into additional redis hash and keeps them there till jobs are finished.
-In case there appears a job being started but not finished and not being processing at the moment, you will know something bad happened.
+### About 
+The case of disappearing sidekiq jobs was proved by [github issues](https://github.com/mperham/sidekiq/issues/1831), [stackoverflow questions](http://stackoverflow.com/questions/35555000/current-sidekiq-job-lost-when-deploying-to-heroku) and (sadly) personal experience. Attentive Sidekiq was made to protect your jobs and critical data from silent disappearing. In case there appears a job being started but not finished and not being processing at the moment, you will know this.
 
 ### Usage
 Attentive Sidekiq provides you with a couple of useful API methods.
 
-To get a hash containing all information about jobs marked as lost:
+To get a hash containing all information about jobs marked as disappeared:
 ```ruby
 AttentiveSidekiq::Disappeared.jobs
 ```
@@ -43,7 +40,7 @@ You may also watch info about disappeared jobs in a web UI.
 Simply make sure you have Sidekiq UI enabled, then head right to the Disappeared Jobs tab in the navbar.
 The Web UI uses the API exclusively: anything you can do in the UI can be scripted with the API.
 
-![Web UI](web.png)
+![webui](https://cloud.githubusercontent.com/assets/1937799/20490807/a01216d0-b028-11e6-96b7-c23fd67bdf89.png)
 
 ### Pre-requirements and notes
 
@@ -73,6 +70,24 @@ end
 
 After that you can use your jobs as usual.
 
+### Configuration
+
+Specify desired configuration inside of `sidekiq.yml` file:
+
+```YML
+attentive:
+  # Time in seconds between checks for disappeared jobs
+  :execution_interval: 300  # default: 600
+  # Time limit in seconds to perform disappeared jobs check
+  :timeout_interval: 25     # default: 60
+```
+
+By default, Attentive Sidekiq uses `Sidekiq.logger` to log its work. You may change it in your initializer:
+
+```ruby
+AttentiveSidekiq.logger = Logger.new("log/attentive_sidekiq.log")
+```
+
 ### Suggestions? Bugs?
 
 If you've got a question, feature suggestion or found a bug please add an [issue on GitHub](https://github.com/twonegatives/attentive_sidekiq/issues) or fork the project and send a pull request.

data/lib/attentive_sidekiq.rb

@@ -2,7 +2,6 @@ require 'set'
 require 'active_support/inflector'
 require 'active_support/core_ext/hash'
 require 'concurrent'
-require 'attentive_sidekiq/middleware'
 require 'attentive_sidekiq/api'
 require 'attentive_sidekiq/middleware/server/attentionist'
 require 'attentive_sidekiq/middleware/client/attentionist'
@@ -12,11 +11,36 @@ require 'sidekiq/web' unless defined?(Sidekiq::Web)
 require 'attentive_sidekiq/web'
 
 module AttentiveSidekiq
+  DEFAULTS = {
+    timeout_interval: 60,
+    execution_interval: 600,
+  }
+    
+  REDIS_SUSPICIOUS_KEY  = "attentive_observed_hash"
+  REDIS_DISAPPEARED_KEY = "attentive_disappeared_hash"
+
   class << self
-    attr_writer :logger
+    attr_writer :timeout_interval, :execution_interval, :logger
+
+    def timeout_interval
+      return @execution_timeout if @execution_timeout
+      @timeout_interval = options[:timeout_interval] || DEFAULTS[:timeout_interval]
+    end
+
+    def execution_interval
+      return @execution_interval if @execution_interval
+      @execution_interval = options[:execution_interval] || DEFAULTS[:execution_interval]
+    end
 
     def logger
       @logger ||= Sidekiq.logger
     end
+
+    def options
+      Sidekiq.options["attentive"] || {}
+    end
+
   end
 end
+
+AttentiveSidekiq::Manager.instance.start! if Sidekiq.server?

data/lib/attentive_sidekiq/api.rb

@@ -20,17 +20,10 @@ module AttentiveSidekiq
       def remove jid
         Sidekiq.redis{|conn| conn.hdel(hash_name, jid)}
       end
-
-      private
-
-      def hash_name
-        self.const_get(:HASH_NAME)
-      end
     end
   end
   
   class Disappeared < RedisBasedHash
-    HASH_NAME = AttentiveSidekiq::Middleware::REDIS_DISAPPEARED_KEY
     STATUS_DETECTED = 'detected'
     STATUS_REQUEUED = 'requeued'
 
@@ -47,11 +40,19 @@ module AttentiveSidekiq
         record['class'].constantize.perform_async(*record['args'])
         base_add(record.merge('status' => STATUS_REQUEUED))
       end
+
+      def hash_name
+        AttentiveSidekiq::REDIS_DISAPPEARED_KEY
+      end
     end
   end
   
   class Suspicious < RedisBasedHash
-    HASH_NAME = AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY
+    class << self
+      def hash_name
+        AttentiveSidekiq::REDIS_SUSPICIOUS_KEY
+      end
+    end
   end
 
   class Active

data/lib/attentive_sidekiq/manager.rb

@@ -29,12 +29,11 @@ module AttentiveSidekiq
     private
     
     def options
-      timeout  = 1*60
-      interval = 10*60
-      { execution_interval: interval, timeout_interval: timeout }
+      { 
+        execution_interval: AttentiveSidekiq.execution_interval,
+        timeout_interval: AttentiveSidekiq.timeout_interval
+      }
     end
     
   end
 end
-
-AttentiveSidekiq::Manager.instance.start! if Sidekiq.server?

data/lib/attentive_sidekiq/version.rb

@@ -1,3 +1,3 @@
 module AttentiveSidekiq
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/test/manager_test.rb

@@ -35,11 +35,11 @@ class ManagerTest < Minitest::Test
     end
 
     def suspicious_now
-      Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.map{|i| JSON.parse(i)}
+      Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.map{|i| JSON.parse(i)}
     end
 
     def disappeared_now
-      from_redis = Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_DISAPPEARED_KEY)}
+      from_redis = Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_DISAPPEARED_KEY)}
       from_redis.map{|i| JSON.parse(i).except('noticed_at', 'status')}
     end
   end

data/test/server_middleware_test.rb

@@ -78,21 +78,21 @@ class ServerMiddlewareTest < Minitest::Test
       assert_equal 0, DefaultQueue.instance.size
       HardWorker.perform_async(1)
       assert_equal 1, DefaultQueue.instance.size
-      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
+      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size
     end
 
     it "marks job as suspicious as soon as it is started" do
-      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
+      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size
       HardWorker.perform_async(2, 100_000)
       Thread.new{
         SidekiqEmulator.instance.process_jobs
       }
       sleep(1) # TODO: refactor this somehow
-      assert_equal 1, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
+      assert_equal 1, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size
     end
 
     it "removes suspicious mark as soon as job is finished succesfully" do
-      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
+      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size
       HardWorker.perform_async(2, 1)
       Thread.new{
         @mutex.synchronize{
@@ -101,11 +101,11 @@ class ServerMiddlewareTest < Minitest::Test
         }
       }
       @mutex.synchronize{ @stopper.wait(@mutex) }
-      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
+      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size
     end
 
     it "removes suspicious mark as soon as job failed" do
-      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
+      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size
       HardWorker.perform_async(2, -1)
       Thread.new{
         @mutex.synchronize{
@@ -114,7 +114,7 @@ class ServerMiddlewareTest < Minitest::Test
         }
       }
       @mutex.synchronize{ @stopper.wait(@mutex) }
-      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
+      assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: attentive_sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - twonegatives
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-20 00:00:00.000000000 Z
+date: 2016-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -164,10 +164,10 @@ files:
 - Rakefile
 - attentive_sidekiq.gemspec
 - circle.yml
+- dump.rdb
 - lib/attentive_sidekiq.rb
 - lib/attentive_sidekiq/api.rb
 - lib/attentive_sidekiq/manager.rb
-- lib/attentive_sidekiq/middleware.rb
 - lib/attentive_sidekiq/middleware/client/attentionist.rb
 - lib/attentive_sidekiq/middleware/server/attentionist.rb
 - lib/attentive_sidekiq/updater_observer.rb
@@ -181,7 +181,6 @@ files:
 - test/server_middleware_test.rb
 - test/test_helper.rb
 - test/web_test.rb
-- web.png
 homepage: http://rubygems.org/gems/attentive_sidekiq
 licenses:
 - MIT
@@ -202,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Make your sidekiq to be attentive to lost jobs

data/lib/attentive_sidekiq/middleware.rb

@@ -1,6 +0,0 @@
-module AttentiveSidekiq
-  module Middleware
-    REDIS_SUSPICIOUS_KEY  = "attentive_observed_hash"
-    REDIS_DISAPPEARED_KEY = "attentive_disappeared_hash"
-  end
-end