attentive_sidekiq 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 74d0c748009ee5ce4f34769f90fb071a375b6da9
-  data.tar.gz: bfa38425968b6178b3b7d74d68ee76f745fe0771
+  metadata.gz: 77e7b5ed662a75451ebd6432ed1897ed01cb6f55
+  data.tar.gz: d26bda1a3c256bf452a35394d6fcf125459fa5ae
 SHA512:
-  metadata.gz: f47d3137dc08eb4b83f8768e416fa925d238b13c51a193f87225f7b08fd9bb5bba69d913446a974a0512fc5cb9f506279a91de8190d71415cd49ae4603a1c7b9
-  data.tar.gz: fa70ba452df6d816d8a05b3ca6accdd00256ba4556085a69000416e6cfa3349956d706a237ee761f8fac1a03c2a8feacf14eaa924e31be369d47390ffe1ee7f7
+  metadata.gz: 0f0a0635e8fa0671808a90b0d9f98ab571560019eabc6b1e45f035317ce945955b789923b7f8b472f3bb65143ee76ec13dfd2fa4506b66d2b68064b3831dc288
+  data.tar.gz: e2e5e97b3cf9e206773de95ce5166280d043516b9b2c7e027356f0011364009264c7675ebf9624123c65a3570e7b170fa4f347b3b1e1fddf09c8c9fbfc7280b2

data/Gemfile.lock

@@ -2,20 +2,34 @@ PATH
   remote: .
   specs:
     attentive_sidekiq (0.1.0)
+      activesupport
       concurrent-ruby (~> 1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (4.2.5)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    byebug (9.0.6)
     coderay (1.1.1)
     concurrent-ruby (1.0.2)
     connection_pool (2.2.1)
+    i18n (0.7.0)
+    json (1.8.3)
     method_source (0.8.2)
     minitest (5.9.1)
+    minitest-stub_any_instance (1.0.1)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
+    pry-byebug (3.4.0)
+      byebug (~> 9.0)
+      pry (~> 0.10)
     rack (2.0.1)
     rack-protection (1.5.3)
       rack
@@ -31,6 +45,9 @@ GEM
       rack-protection (>= 1.5.0)
       redis (~> 3.2, >= 3.2.1)
     slop (3.6.0)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
 
 PLATFORMS
   ruby
@@ -38,7 +55,9 @@ PLATFORMS
 DEPENDENCIES
   attentive_sidekiq!
   minitest (~> 5.0)
+  minitest-stub_any_instance
   pry (~> 0.10)
+  pry-byebug
   rack-test (~> 0.6)
   rake (~> 11.3)
   redis-namespace (~> 1.5)

data/README.md

@@ -28,7 +28,12 @@ To get only JIDs of lost jobs:
 AttentiveSidekiq::Disappeared.job_ids
 ```
 
-To remove a job from disappeared hash (e.g. after manual relaunch):
+To place a disappeared job back into queue:
+```ruby
+AttentiveSidekiq::Disappeared.requeue(jid)
+```
+
+To remove a job from disappeared hash (e.g. after manual requeue):
 ```ruby
 AttentiveSidekiq::Disappeared.remove(jid)
 ```
@@ -36,10 +41,18 @@ AttentiveSidekiq::Disappeared.remove(jid)
 ### Sidekiq Web integration
 You may also watch info about disappeared jobs in a web UI.
 Simply make sure you have Sidekiq UI enabled, then head right to the Disappeared Jobs tab in the navbar.
+The Web UI uses the API exclusively: anything you can do in the UI can be scripted with the API.
 
 ![Web UI](web.png)
 
+### Pre-requirements and notes
+
+- Attentive Sidekiq assumes you've got Sidekiq installed already.
+- You should make sure sidekiq process is started in order for disappeared jobs updater to work properly.
+- It was tested with Sidekiq version 4. Seamless functionality with lower sidekiq versions is not guaranteed.
+
 ### Installation
+
 Add this line to your application's Gemfile:
     
     gem 'attentive_sidekiq'

data/attentive_sidekiq.gemspec

@@ -5,7 +5,6 @@ require 'attentive_sidekiq/version'
 Gem::Specification.new do |s|
   s.name        = 'attentive_sidekiq'
   s.version     = AttentiveSidekiq::VERSION
-  s.date        = '2016-10-31'
   s.summary     = "Make your sidekiq to be attentive to lost jobs"
   s.description = "This gem allows you to watch the jobs which suddenly dissappeared from redis without being completed by redis worker"
   s.authors     = ["twonegatives"]
@@ -19,9 +18,12 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'sidekiq', '~> 4.2'
   s.add_development_dependency 'rake', '~> 11.3'
   s.add_development_dependency 'minitest', '~> 5.0'
+  s.add_development_dependency 'minitest-stub_any_instance'
   s.add_development_dependency 'redis-namespace', '~> 1.5'
   s.add_development_dependency "rack-test", '~> 0.6'
   s.add_development_dependency 'pry', '~> 0.10'
+  s.add_development_dependency "pry-byebug"
 
+  s.add_dependency "activesupport"
   s.add_dependency 'concurrent-ruby', '~> 1.0'
 end

data/lib/attentive_sidekiq.rb

@@ -1,4 +1,6 @@
 require 'set'
+require 'active_support/inflector'
+require 'active_support/core_ext/hash'
 require 'concurrent'
 require 'attentive_sidekiq/middleware'
 require 'attentive_sidekiq/api'

data/lib/attentive_sidekiq/api.rb

@@ -9,6 +9,10 @@ module AttentiveSidekiq
         jobs.map{|i| i["jid"]}
       end
 
+      def get_job jid
+        JSON.parse(Sidekiq.redis{|conn| conn.hget(hash_name, jid)})
+      end
+
       def add item
         Sidekiq.redis{ |conn| conn.hset(hash_name, item['jid'], item.to_json) }
       end
@@ -27,6 +31,23 @@ module AttentiveSidekiq
   
   class Disappeared < RedisBasedHash
     HASH_NAME = AttentiveSidekiq::Middleware::REDIS_DISAPPEARED_KEY
+    STATUS_DETECTED = 'detected'
+    STATUS_REQUEUED = 'requeued'
+
+    class << self
+      alias_method :base_add, :add
+      
+      def add item
+        extended_item = {'noticed_at' => Time.now.to_i, 'status' => STATUS_DETECTED}.merge(item)
+        super extended_item
+      end
+
+      def requeue jid
+        record = get_job(jid)
+        record['class'].constantize.perform_async(*record['args'])
+        base_add(record.merge('status' => STATUS_REQUEUED))
+      end
+    end
   end
   
   class Suspicious < RedisBasedHash

data/lib/attentive_sidekiq/version.rb

@@ -1,3 +1,3 @@
 module AttentiveSidekiq
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/attentive_sidekiq/web.rb

@@ -3,10 +3,21 @@ module AttentiveSidekiq
     VIEW_PATH = File.expand_path("../web/views", __FILE__)
 
     def self.registered(app)
-      app.get('/disappeared-jobs') do
-        @suspicious_jobs  = AttentiveSidekiq::Disappeared.jobs
+      app.get("/disappeared-jobs") do
+        @disappeared_jobs = AttentiveSidekiq::Disappeared.jobs
         erb File.read(File.join(VIEW_PATH, 'disappeared-list.erb'))
-      end	
+      end
+
+      app.post("/disappeared-jobs/:jid/delete") do
+        AttentiveSidekiq::Disappeared.remove(params['jid'])
+        redirect "#{root_path}disappeared-jobs"
+      end
+
+      app.post("/disappeared-jobs/:jid/requeue") do
+        AttentiveSidekiq::Disappeared.requeue(params['jid'])
+        redirect "#{root_path}disappeared-jobs"
+      end
+
     end
   end
 end

data/lib/attentive_sidekiq/web/locales/en.yml

@@ -4,4 +4,14 @@ en:
   class: Class name
   arguments: Arguments
   disappeared_jobs: Disappeared jobs
-  time: When
+  created_at: Created at
+  noticed_at: Noticed disappeared at
+  noticed_unknown: time unknown
+  actions: Actions
+  delete: Delete
+  delete_confirmation: Are you sure you want to delete this record?
+  status: Status
+  detected: Detected
+  requeued: Requeued
+  requeue: Requeue
+  requeue_confirmation: Are you sure you want to requeue this job?

data/lib/attentive_sidekiq/web/locales/ru.yml

@@ -4,4 +4,14 @@ ru:
   class: Задача
   arguments: Аргументы
   disappeared_jobs: Пропавшие задачи
-  time: Время
+  created_at: Создана
+  noticed_at: Пропажа замечена
+  noticed_unknown: время неизвестно
+  actions: Действия
+  delete: Удалить
+  delete_confirmation: Вы уверены, что хотите удалить эту запись? 
+  status: Статус
+  detected: Обнаружена
+  requeued: Поставлена в очередь
+  requeue: Перезапустить
+  requeue_confirmation: Вы уверены, что хотите перезапустить задачу?

data/lib/attentive_sidekiq/web/views/disappeared-list.erb

@@ -8,18 +8,35 @@
         <th><%= t('queue') %></th>
         <th><%= t('class') %></th>
         <th><%= t('arguments') %></th>
-        <th><%= t('time') %></th>
+        <th><%= t('created_at') %></th>
+        <th><%= t('noticed_at') %></th>
+        <th><%= t('status') %></th>
+        <th><%= t('actions') %></th>
       </tr>
     </thead>
 
     <tbody>
-      <% @suspicious_jobs.each do |job| %>
+      <% @disappeared_jobs.each do |job| %>
         <tr>
           <td><%= job['jid'] %></td>
           <td><%= job['queue'] %></td>
           <td><%= job['class'] %></td>
           <td><%= job['args'] %></td>
           <td><%= Time.at(job['created_at']).strftime("%H:%M:%S %d.%m.%Y %z") %></td>
+          <td><%= job['noticed_at'] ? Time.at(job['noticed_at']).strftime("%H:%M:%S %d.%m.%Y %z") : t("noticed_unknown") %></td>
+          <th><%= job['status'] ? t(job['status']) : t('detected') %></th>
+          <td>
+            <form action="<%= "#{root_path}disappeared-jobs/#{job['jid']}/delete" %>" method="post">
+              <%= csrf_tag %>
+              <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('delete') %>" style="width:100%" data-confirm="<%= t('delete_confirmation') %>" />
+            </form>
+            <% if job['status'] != AttentiveSidekiq::Disappeared::STATUS_REQUEUED %>
+              <form action="<%= "#{root_path}disappeared-jobs/#{job['jid']}/requeue" %>" method="post">
+                <%= csrf_tag %>
+                <input class="btn btn-xs" type="submit" name="requeue" value="<%= t('requeue') %>" style="width:100%" data-confirm="<%= t('requeue_confirmation') %>" />
+              </form>
+            <% end %>
+          </td>
         </tr>
       <% end %>
     </tbody>

data/test/api_test.rb

@@ -0,0 +1,73 @@
+require "test_helper"
+
+class ApiTest < Minitest::Test
+  describe "with real redis" do
+    before do
+      Sidekiq.redis = REDIS
+      Sidekiq.redis{ |c| c.flushdb }
+
+      simple_hash             = {'class' => 'ApiTest::SimpleWorker', 'args' => [], 'created_at' => Time.now.to_i}
+      args_hash               = {'args' => [1, "boo", "woo"], 'class' => 'ApiTest::WorkerWithArgs', 'created_at' => Time.now.to_i}
+
+      @item_in_progress       = {'jid' => "REDA-257513", 'queue' => 'red_queue'}.merge!(simple_hash)
+      @disappeared_wo_args    = {'jid' => "YE11OW5-247", 'queue' => 'yellow_queue'}.merge!(simple_hash)
+      @disappeared_with_args  = {'jid' => "B1UE-441", 'queue' => 'blue_queue'}.merge!(args_hash)
+
+      AttentiveSidekiq::Suspicious.add(@item_in_progress)
+
+      AttentiveSidekiq::Disappeared.add(@disappeared_wo_args)
+      AttentiveSidekiq::Disappeared.add(@disappeared_with_args)
+    end
+
+    class BaseWorker
+      include Sidekiq::Worker
+    end
+
+    class SimpleWorker < BaseWorker
+      def perform
+      end
+    end
+
+    class WorkerWithArgs < BaseWorker
+      def perform a, b, c
+      end
+    end
+
+    describe "requeue" do
+      it "adds jobs without args to sidekiq queue" do
+        item = @disappeared_wo_args
+        assert_equal 0, Sidekiq::Queue.new.size
+        String.stub_any_instance("constantize", SimpleWorker) do
+          AttentiveSidekiq::Disappeared.requeue(item['jid'])
+        end
+        assert_equal 1, Sidekiq::Queue.new.size
+        element = Sidekiq::Queue.new.to_a.first
+        assert_equal item['class'], element['class']
+        assert_equal item['args'], element['args']
+      end
+
+      it "adds job with args to sidekiq queue" do
+        item = @disappeared_with_args
+        String.stub_any_instance("constantize", WorkerWithArgs) do
+          AttentiveSidekiq::Disappeared.requeue(item['jid'])
+        end
+        element = Sidekiq::Queue.new.to_a.first
+        assert_equal item['class'], element['class']
+        assert_equal item['args'], element['args']
+      end
+
+      it "marks jobs as requeued" do
+        item = @disappeared_wo_args
+        assert_equal job_status(item), AttentiveSidekiq::Disappeared::STATUS_DETECTED
+        String.stub_any_instance("constantize", SimpleWorker) do
+          AttentiveSidekiq::Disappeared.requeue(item['jid'])
+        end
+        assert_equal job_status(item), AttentiveSidekiq::Disappeared::STATUS_REQUEUED
+      end
+
+      def job_status(item)
+        AttentiveSidekiq::Disappeared.get_job(item['jid'])['status']
+      end
+    end
+  end
+end

data/test/manager_test.rb

@@ -39,7 +39,8 @@ class ManagerTest < Minitest::Test
     end
 
     def disappeared_now
-      Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_DISAPPEARED_KEY)}.map{|i| JSON.parse(i)}
+      from_redis = Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_DISAPPEARED_KEY)}
+      from_redis.map{|i| JSON.parse(i).except('noticed_at', 'status')}
     end
   end
 end

data/test/test_helper.rb

@@ -12,6 +12,7 @@ require 'redis-namespace'
 require 'attentive_sidekiq'
 require 'minitest/autorun'
 require 'minitest/pride'
+require 'minitest/stub_any_instance'
 
 REDIS_URL = ENV["REDIS_URL"] || "redis://localhost:15"
 REDIS_NAMESPACE = ENV["REDIS_NAMESPACE"] || 'testy'
@@ -22,3 +23,5 @@ Sidekiq.configure_server do |config|
     chain.add AttentiveSidekiq::Middleware::Server::Attentionist
   end
 end
+
+Sidekiq.logger = nil

data/test/web_test.rb

@@ -13,7 +13,6 @@ class WebTest < Minitest::Test
     @item_in_progress = {'jid' => "REDA-257513", 'queue' => 'red_queue'}.merge!(common_hash)
 
     AttentiveSidekiq::Suspicious.add(@item_in_progress)
-    AttentiveSidekiq::Suspicious.add(@item_disappeared)
     AttentiveSidekiq::Disappeared.add(@item_disappeared)
   end
 
@@ -29,6 +28,22 @@ class WebTest < Minitest::Test
     refute_match @item_in_progress['jid'], last_response.body
   end
 
+  def test_delete_route_functions_fine
+    AttentiveSidekiq::Disappeared.stub(:remove, nil) do
+      post "/disappeared-jobs/#{@item_disappeared['jid']}/delete"
+      follow_redirect!
+      assert_equal 200, last_response.status
+    end
+  end
+
+  def test_requeue_route_functions_fine
+    AttentiveSidekiq::Disappeared.stub(:requeue, nil) do
+      post "/disappeared-jobs/#{@item_disappeared['jid']}/requeue"
+      follow_redirect!
+      assert_equal 200, last_response.status
+    end
+  end
+
   private
 
   def app

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: attentive_sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - twonegatives
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-31 00:00:00.000000000 Z
+date: 2016-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: minitest-stub_any_instance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +108,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -133,6 +175,7 @@ files:
 - lib/attentive_sidekiq/web/locales/en.yml
 - lib/attentive_sidekiq/web/locales/ru.yml
 - lib/attentive_sidekiq/web/views/disappeared-list.erb
+- test/api_test.rb
 - test/manager_test.rb
 - test/server_middleware_test.rb
 - test/test_helper.rb
@@ -163,6 +206,7 @@ signing_key:
 specification_version: 4
 summary: Make your sidekiq to be attentive to lost jobs
 test_files:
+- test/api_test.rb
 - test/manager_test.rb
 - test/server_middleware_test.rb
 - test/test_helper.rb