attentive_sidekiq 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/Gemfile.lock +48 -0
- data/README.md +44 -28
- data/Rakefile +9 -0
- data/attentive_sidekiq.gemspec +15 -1
- data/circle.yml +9 -0
- data/lib/attentive_sidekiq.rb +16 -1
- data/lib/attentive_sidekiq/api.rb +47 -0
- data/lib/attentive_sidekiq/manager.rb +40 -0
- data/lib/attentive_sidekiq/middleware.rb +3 -2
- data/lib/attentive_sidekiq/middleware/client/attentionist.rb +2 -14
- data/lib/attentive_sidekiq/middleware/server/attentionist.rb +3 -12
- data/lib/attentive_sidekiq/updater_observer.rb +13 -0
- data/lib/attentive_sidekiq/version.rb +3 -0
- data/lib/attentive_sidekiq/web.rb +7 -15
- data/lib/attentive_sidekiq/web/locales/en.yml +1 -1
- data/lib/attentive_sidekiq/web/locales/ru.yml +1 -1
- data/lib/attentive_sidekiq/web/views/disappeared-list.erb +1 -1
- data/test/manager_test.rb +45 -0
- data/test/server_middleware_test.rb +120 -0
- data/test/test_helper.rb +24 -0
- data/test/web_test.rb +37 -0
- data/web.png +0 -0
- metadata +119 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 74d0c748009ee5ce4f34769f90fb071a375b6da9
|
4
|
+
data.tar.gz: bfa38425968b6178b3b7d74d68ee76f745fe0771
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f47d3137dc08eb4b83f8768e416fa925d238b13c51a193f87225f7b08fd9bb5bba69d913446a974a0512fc5cb9f506279a91de8190d71415cd49ae4603a1c7b9
|
7
|
+
data.tar.gz: fa70ba452df6d816d8a05b3ca6accdd00256ba4556085a69000416e6cfa3349956d706a237ee761f8fac1a03c2a8feacf14eaa924e31be369d47390ffe1ee7f7
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
attentive_sidekiq (0.1.0)
|
5
|
+
concurrent-ruby (~> 1.0)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
coderay (1.1.1)
|
11
|
+
concurrent-ruby (1.0.2)
|
12
|
+
connection_pool (2.2.1)
|
13
|
+
method_source (0.8.2)
|
14
|
+
minitest (5.9.1)
|
15
|
+
pry (0.10.4)
|
16
|
+
coderay (~> 1.1.0)
|
17
|
+
method_source (~> 0.8.1)
|
18
|
+
slop (~> 3.4)
|
19
|
+
rack (2.0.1)
|
20
|
+
rack-protection (1.5.3)
|
21
|
+
rack
|
22
|
+
rack-test (0.6.3)
|
23
|
+
rack (>= 1.0)
|
24
|
+
rake (11.3.0)
|
25
|
+
redis (3.3.1)
|
26
|
+
redis-namespace (1.5.2)
|
27
|
+
redis (~> 3.0, >= 3.0.4)
|
28
|
+
sidekiq (4.2.5)
|
29
|
+
concurrent-ruby (~> 1.0)
|
30
|
+
connection_pool (~> 2.2, >= 2.2.0)
|
31
|
+
rack-protection (>= 1.5.0)
|
32
|
+
redis (~> 3.2, >= 3.2.1)
|
33
|
+
slop (3.6.0)
|
34
|
+
|
35
|
+
PLATFORMS
|
36
|
+
ruby
|
37
|
+
|
38
|
+
DEPENDENCIES
|
39
|
+
attentive_sidekiq!
|
40
|
+
minitest (~> 5.0)
|
41
|
+
pry (~> 0.10)
|
42
|
+
rack-test (~> 0.6)
|
43
|
+
rake (~> 11.3)
|
44
|
+
redis-namespace (~> 1.5)
|
45
|
+
sidekiq (~> 4.2)
|
46
|
+
|
47
|
+
BUNDLED WITH
|
48
|
+
1.13.6
|
data/README.md
CHANGED
@@ -1,15 +1,43 @@
|
|
1
1
|
# Attentive Sidekiq
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
However, it is reported by some users to still to cause the same issue.
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/attentive_sidekiq.svg)](https://badge.fury.io/rb/attentive_sidekiq)
|
4
|
+
[![Code Climate](https://codeclimate.com/github/twonegatives/attentive_sidekiq/badges/gpa.svg)](https://codeclimate.com/github/twonegatives/attentive_sidekiq)
|
5
|
+
[![CircleCI](https://circleci.com/gh/twonegatives/attentive_sidekiq.svg?style=shield)](https://circleci.com/gh/twonegatives/attentive_sidekiq)
|
6
|
+
|
8
7
|
|
8
|
+
### Motivation
|
9
|
+
Bad things happen sometimes. Sidekiq process being killed in the middle of job processing may lead to job losage. In other words, that job will never be returned to the queue.
|
10
|
+
This case was proved by [github issues](https://github.com/mperham/sidekiq/issues/1831), [stackoverflow questions](http://stackoverflow.com/questions/35555000/current-sidekiq-job-lost-when-deploying-to-heroku) and (sadly) personal experience, and means that you are not safe from losing critical user data.
|
11
|
+
Sidekiq's author, Mike Perham, suggests purchasing Sidekiq Pro which uses another fetch mechanism. However, it [is reported](https://github.com/mperham/sidekiq/issues/2531) to still cause the same issue.
|
9
12
|
|
10
13
|
### About
|
11
|
-
Attentive Sidekiq is a sidekiq plugin which
|
12
|
-
It saves started jobs
|
14
|
+
Attentive Sidekiq is a sidekiq plugin which makes one more step to guard your jobs from being lost (well, at least to be notified of this).
|
15
|
+
It saves info about started jobs into additional redis hash and keeps them there till jobs are finished.
|
16
|
+
In case there appears a job being started but not finished and not being processing at the moment, you will know something bad happened.
|
17
|
+
|
18
|
+
### Usage
|
19
|
+
Attentive Sidekiq provides you with a couple of useful API methods.
|
20
|
+
|
21
|
+
To get a hash containing all information about jobs marked as lost:
|
22
|
+
```ruby
|
23
|
+
AttentiveSidekiq::Disappeared.jobs
|
24
|
+
```
|
25
|
+
|
26
|
+
To get only JIDs of lost jobs:
|
27
|
+
```ruby
|
28
|
+
AttentiveSidekiq::Disappeared.job_ids
|
29
|
+
```
|
30
|
+
|
31
|
+
To remove a job from disappeared hash (e.g. after manual relaunch):
|
32
|
+
```ruby
|
33
|
+
AttentiveSidekiq::Disappeared.remove(jid)
|
34
|
+
```
|
35
|
+
|
36
|
+
### Sidekiq Web integration
|
37
|
+
You may also watch info about disappeared jobs in a web UI.
|
38
|
+
Simply make sure you have Sidekiq UI enabled, then head right to the Disappeared Jobs tab in the navbar.
|
39
|
+
|
40
|
+
![Web UI](web.png)
|
13
41
|
|
14
42
|
### Installation
|
15
43
|
Add this line to your application's Gemfile:
|
@@ -20,26 +48,14 @@ And then execute:
|
|
20
48
|
|
21
49
|
$ bundle
|
22
50
|
|
23
|
-
|
24
|
-
Configure your middleware chains, lookup Middleware usage on Sidekiq wiki for more info.
|
25
|
-
|
26
|
-
Sidekiq.configure_server do |config|
|
27
|
-
config.server_middleware do |chain|
|
28
|
-
chain.add AttentiveSidekiq::Middleware::Server::Attentionist
|
29
|
-
end
|
30
|
-
config.client_middleware do |chain|
|
31
|
-
chain.add AttentiveSidekiq::Middleware::Client::Attentionist
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
Sidekiq.configure_client do |config|
|
36
|
-
config.client_middleware do |chain|
|
37
|
-
chain.add AttentiveSidekiq::Middleware::Client::Attentionist
|
38
|
-
end
|
39
|
-
end
|
51
|
+
Configure your middleware chains, lookup [Middleware usage](https://github.com/mperham/sidekiq/wiki/Middleware) on Sidekiq wiki for more info.
|
40
52
|
|
41
|
-
|
53
|
+
```ruby
|
54
|
+
Sidekiq.configure_server do |config|
|
55
|
+
config.server_middleware do |chain|
|
56
|
+
chain.add AttentiveSidekiq::Middleware::Server::Attentionist
|
57
|
+
end
|
58
|
+
end
|
59
|
+
```
|
42
60
|
|
43
|
-
|
44
|
-
Attentive Sidekiq provides an extension to the Sidekiq web interface that adds a Disappeared Jobs page.
|
45
|
-
To use it, head to your sidekiq dashboard and click the link at the tabs section.
|
61
|
+
After that you can use your jobs as usual.
|
data/Rakefile
ADDED
data/attentive_sidekiq.gemspec
CHANGED
@@ -1,13 +1,27 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'attentive_sidekiq/version'
|
4
|
+
|
1
5
|
Gem::Specification.new do |s|
|
2
6
|
s.name = 'attentive_sidekiq'
|
3
|
-
s.version =
|
7
|
+
s.version = AttentiveSidekiq::VERSION
|
4
8
|
s.date = '2016-10-31'
|
5
9
|
s.summary = "Make your sidekiq to be attentive to lost jobs"
|
6
10
|
s.description = "This gem allows you to watch the jobs which suddenly dissappeared from redis without being completed by redis worker"
|
7
11
|
s.authors = ["twonegatives"]
|
8
12
|
s.email = 'whitewhiteheaven@gmail.com'
|
9
13
|
s.files = Dir['**/*'].keep_if{ |file| File.file?(file) }
|
14
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
10
15
|
s.homepage =
|
11
16
|
'http://rubygems.org/gems/attentive_sidekiq'
|
12
17
|
s.license = 'MIT'
|
18
|
+
|
19
|
+
s.add_development_dependency 'sidekiq', '~> 4.2'
|
20
|
+
s.add_development_dependency 'rake', '~> 11.3'
|
21
|
+
s.add_development_dependency 'minitest', '~> 5.0'
|
22
|
+
s.add_development_dependency 'redis-namespace', '~> 1.5'
|
23
|
+
s.add_development_dependency "rack-test", '~> 0.6'
|
24
|
+
s.add_development_dependency 'pry', '~> 0.10'
|
25
|
+
|
26
|
+
s.add_dependency 'concurrent-ruby', '~> 1.0'
|
13
27
|
end
|
data/circle.yml
ADDED
data/lib/attentive_sidekiq.rb
CHANGED
@@ -1,5 +1,20 @@
|
|
1
|
+
require 'set'
|
2
|
+
require 'concurrent'
|
1
3
|
require 'attentive_sidekiq/middleware'
|
4
|
+
require 'attentive_sidekiq/api'
|
2
5
|
require 'attentive_sidekiq/middleware/server/attentionist'
|
3
6
|
require 'attentive_sidekiq/middleware/client/attentionist'
|
7
|
+
require 'attentive_sidekiq/updater_observer'
|
8
|
+
require 'attentive_sidekiq/manager'
|
4
9
|
require 'sidekiq/web' unless defined?(Sidekiq::Web)
|
5
|
-
require 'attentive_sidekiq/web'
|
10
|
+
require 'attentive_sidekiq/web'
|
11
|
+
|
12
|
+
module AttentiveSidekiq
|
13
|
+
class << self
|
14
|
+
attr_writer :logger
|
15
|
+
|
16
|
+
def logger
|
17
|
+
@logger ||= Sidekiq.logger
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module AttentiveSidekiq
|
2
|
+
class RedisBasedHash
|
3
|
+
class << self
|
4
|
+
def jobs
|
5
|
+
Sidekiq.redis{|conn| conn.hvals(hash_name)}.map{|i| JSON.parse(i)}
|
6
|
+
end
|
7
|
+
|
8
|
+
def job_ids
|
9
|
+
jobs.map{|i| i["jid"]}
|
10
|
+
end
|
11
|
+
|
12
|
+
def add item
|
13
|
+
Sidekiq.redis{ |conn| conn.hset(hash_name, item['jid'], item.to_json) }
|
14
|
+
end
|
15
|
+
|
16
|
+
def remove jid
|
17
|
+
Sidekiq.redis{|conn| conn.hdel(hash_name, jid)}
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def hash_name
|
23
|
+
self.const_get(:HASH_NAME)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class Disappeared < RedisBasedHash
|
29
|
+
HASH_NAME = AttentiveSidekiq::Middleware::REDIS_DISAPPEARED_KEY
|
30
|
+
end
|
31
|
+
|
32
|
+
class Suspicious < RedisBasedHash
|
33
|
+
HASH_NAME = AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY
|
34
|
+
end
|
35
|
+
|
36
|
+
class Active
|
37
|
+
class << self
|
38
|
+
def jobs
|
39
|
+
Sidekiq::Workers.new.to_a.map{|i| i[2]["payload"]}
|
40
|
+
end
|
41
|
+
|
42
|
+
def job_ids
|
43
|
+
Set.new(jobs.map{|i| i["jid"]})
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module AttentiveSidekiq
|
2
|
+
class Manager
|
3
|
+
@@instance = AttentiveSidekiq::Manager.new
|
4
|
+
|
5
|
+
def self.instance
|
6
|
+
@@instance
|
7
|
+
end
|
8
|
+
|
9
|
+
def start!
|
10
|
+
task = Concurrent::TimerTask.new(options) do
|
11
|
+
AttentiveSidekiq::Manager.instance.update_disappeared_jobs
|
12
|
+
end
|
13
|
+
task.add_observer(AttentiveSidekiq::UpdaterObserver.new)
|
14
|
+
task.execute
|
15
|
+
end
|
16
|
+
|
17
|
+
def update_disappeared_jobs
|
18
|
+
suspicious = AttentiveSidekiq::Suspicious.jobs
|
19
|
+
active_ids = AttentiveSidekiq::Active.job_ids
|
20
|
+
those_lost = suspicious.delete_if{|i| active_ids.include?(i["jid"])}
|
21
|
+
those_lost.each do |job|
|
22
|
+
Disappeared.add(job)
|
23
|
+
Suspicious.remove(job['jid'])
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private_class_method :new
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def options
|
32
|
+
timeout = 1*60
|
33
|
+
interval = 1*60
|
34
|
+
{ execution_interval: interval, timeout_interval: timeout }
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
AttentiveSidekiq::Manager.instance.start! if Sidekiq.server?
|
@@ -2,23 +2,11 @@ module AttentiveSidekiq
|
|
2
2
|
module Middleware
|
3
3
|
module Client
|
4
4
|
class Attentionist
|
5
|
-
|
6
5
|
def call(worker_class, item, queue, redis_pool = nil)
|
6
|
+
# TODO: we could backup job info here aswell
|
7
|
+
# this would lead us to the need of more complex records filtering
|
7
8
|
yield
|
8
9
|
end
|
9
|
-
|
10
|
-
#def call(worker_class, item, queue, redis_pool = nil)
|
11
|
-
# yield
|
12
|
-
# if rerun?(item["jid"])
|
13
|
-
# mark_as_not_lost(item["jid"])
|
14
|
-
# else
|
15
|
-
# add_to_observed_list(item)
|
16
|
-
# end
|
17
|
-
#end
|
18
|
-
|
19
|
-
#def rerun?(jid)
|
20
|
-
# Sidekiq.redis{|conn| conn.hexists(AttentiveSidekiq::Middleware::REDIS_KEY, jid) } == 1
|
21
|
-
#end
|
22
10
|
end
|
23
11
|
end
|
24
12
|
end
|
@@ -1,21 +1,12 @@
|
|
1
1
|
module AttentiveSidekiq
|
2
2
|
module Middleware
|
3
|
-
|
3
|
+
module Server
|
4
4
|
class Attentionist
|
5
|
-
|
6
5
|
def call(worker_instance, item, queue)
|
7
|
-
|
6
|
+
Suspicious.add(item)
|
8
7
|
yield
|
9
8
|
ensure
|
10
|
-
|
11
|
-
end
|
12
|
-
|
13
|
-
def mark_as_not_lost(jid)
|
14
|
-
Sidekiq.redis{|conn| conn.hdel(AttentiveSidekiq::Middleware::REDIS_KEY, jid)}
|
15
|
-
end
|
16
|
-
|
17
|
-
def add_to_observed_list(item)
|
18
|
-
Sidekiq.redis{ |conn| conn.hset(AttentiveSidekiq::Middleware::REDIS_KEY, item['jid'], item.to_json) }
|
9
|
+
Suspicious.remove(item['jid'])
|
19
10
|
end
|
20
11
|
end
|
21
12
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module AttentiveSidekiq
|
2
|
+
class UpdaterObserver
|
3
|
+
def update time, result, ex
|
4
|
+
if result
|
5
|
+
AttentiveSidekiq.logger.info("#{time} [AttentiveSidekiq] Finished updating with result #{result}")
|
6
|
+
elsif ex.is_a?(Concurrent::TimeoutError)
|
7
|
+
AttentiveSidekiq.logger.error("#{time} [AttentiveSidekiq] Execution timed out")
|
8
|
+
else
|
9
|
+
AttentiveSidekiq.logger.error("#{time } [AttentiveSidekiq] Execution failed with error #{ex}\n")
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -2,23 +2,15 @@ module AttentiveSidekiq
|
|
2
2
|
module Web
|
3
3
|
VIEW_PATH = File.expand_path("../web/views", __FILE__)
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
end
|
12
|
-
|
13
|
-
app.get('/disappeared-jobs') do
|
14
|
-
jobs_active_now = Set.new(Sidekiq::Workers.new.to_a.map{|i| i[2]["payload"]["jid"]})
|
15
|
-
@suspicious_jobs = jobs_not_finished.delete_if{|i| jobs_active_now.include?(i["jid"])}
|
16
|
-
erb File.read(File.join(VIEW_PATH, 'disappeared-list.erb'))
|
17
|
-
end
|
18
|
-
end
|
5
|
+
def self.registered(app)
|
6
|
+
app.get('/disappeared-jobs') do
|
7
|
+
@suspicious_jobs = AttentiveSidekiq::Disappeared.jobs
|
8
|
+
erb File.read(File.join(VIEW_PATH, 'disappeared-list.erb'))
|
9
|
+
end
|
10
|
+
end
|
19
11
|
end
|
20
12
|
end
|
21
13
|
|
22
14
|
Sidekiq::Web.register AttentiveSidekiq::Web
|
23
15
|
Sidekiq::Web.locales << File.expand_path(File.dirname(__FILE__) + "/web/locales")
|
24
|
-
Sidekiq::Web.tabs['disappeared_jobs'] = 'disappeared-jobs'
|
16
|
+
Sidekiq::Web.tabs['disappeared_jobs'] = 'disappeared-jobs'
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
|
3
|
+
class ManagerTest < Minitest::Test
|
4
|
+
describe "with real redis" do
|
5
|
+
before do
|
6
|
+
Sidekiq.redis = REDIS
|
7
|
+
Sidekiq.redis{ |c| c.flushdb }
|
8
|
+
|
9
|
+
common_hash = {'class' => 'UnexistantWorker', 'args' => [], 'created_at' => Time.now.to_i}
|
10
|
+
@item_in_progress = {'jid' => "REDA-257513", 'queue' => 'red_queue'}.merge!(common_hash)
|
11
|
+
@item_disappeared = {'jid' => "YE11OW5-247", 'queue' => 'yellow_queue'}.merge!(common_hash)
|
12
|
+
|
13
|
+
AttentiveSidekiq::Suspicious.add @item_in_progress
|
14
|
+
AttentiveSidekiq::Suspicious.add @item_disappeared
|
15
|
+
|
16
|
+
@active_job_ids = [@item_in_progress['jid']]
|
17
|
+
end
|
18
|
+
|
19
|
+
it "removes lone job from suspicious and adds to disappeared" do
|
20
|
+
AttentiveSidekiq::Active.stub(:job_ids, @active_job_ids) do
|
21
|
+
AttentiveSidekiq::Manager.instance.update_disappeared_jobs
|
22
|
+
|
23
|
+
assert_includes disappeared_now, @item_disappeared
|
24
|
+
refute_includes suspicious_now, @item_disappeared
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
it "leaves jobs which are being currently processed in suspicious" do
|
29
|
+
AttentiveSidekiq::Active.stub(:job_ids, @active_job_ids) do
|
30
|
+
AttentiveSidekiq::Manager.instance.update_disappeared_jobs
|
31
|
+
|
32
|
+
assert_includes suspicious_now, @item_in_progress
|
33
|
+
refute_includes disappeared_now, @item_in_progress
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def suspicious_now
|
38
|
+
Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.map{|i| JSON.parse(i)}
|
39
|
+
end
|
40
|
+
|
41
|
+
def disappeared_now
|
42
|
+
Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_DISAPPEARED_KEY)}.map{|i| JSON.parse(i)}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
|
3
|
+
class ServerMiddlewareTest < Minitest::Test
|
4
|
+
describe "with real redis" do
|
5
|
+
before do
|
6
|
+
Sidekiq.redis = REDIS
|
7
|
+
Sidekiq.redis{ |c| c.flushdb }
|
8
|
+
|
9
|
+
@mutex = Mutex.new
|
10
|
+
@stopper = ConditionVariable.new
|
11
|
+
end
|
12
|
+
|
13
|
+
class HardWorker
|
14
|
+
include Sidekiq::Worker
|
15
|
+
|
16
|
+
def perform(seed, work_amount = 10)
|
17
|
+
raise "wrong amount of work" if work_amount <= 0
|
18
|
+
1.upto(work_amount) do |i|
|
19
|
+
1.upto(work_amount) do |j|
|
20
|
+
1.upto(work_amount) do |k|
|
21
|
+
i*j*k
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class SidekiqEmulator
|
29
|
+
@@instance = SidekiqEmulator.new
|
30
|
+
|
31
|
+
def self.instance
|
32
|
+
@@instance
|
33
|
+
end
|
34
|
+
|
35
|
+
def process_jobs
|
36
|
+
processor.send(:process, work_unit)
|
37
|
+
end
|
38
|
+
|
39
|
+
private_class_method :new
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def processor
|
44
|
+
::Sidekiq::Processor.new(manager)
|
45
|
+
end
|
46
|
+
|
47
|
+
def manager
|
48
|
+
options = { :concurrency => 1, :queues => ['default'] }
|
49
|
+
Sidekiq::Manager.new(options)
|
50
|
+
end
|
51
|
+
|
52
|
+
def work_unit
|
53
|
+
fetch = Sidekiq::BasicFetch.new(:queues => ['default'])
|
54
|
+
fetch.retrieve_work
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
class DefaultQueue
|
60
|
+
@@instance = DefaultQueue.new
|
61
|
+
|
62
|
+
def self.instance
|
63
|
+
@@instance
|
64
|
+
end
|
65
|
+
|
66
|
+
def size
|
67
|
+
queue.size rescue 0
|
68
|
+
end
|
69
|
+
|
70
|
+
def queue
|
71
|
+
::Sidekiq::Queue.new
|
72
|
+
end
|
73
|
+
|
74
|
+
private_class_method :new
|
75
|
+
end
|
76
|
+
|
77
|
+
it "does not mark job as suspicious while its queued" do
|
78
|
+
assert_equal 0, DefaultQueue.instance.size
|
79
|
+
HardWorker.perform_async(1)
|
80
|
+
assert_equal 1, DefaultQueue.instance.size
|
81
|
+
assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
|
82
|
+
end
|
83
|
+
|
84
|
+
it "marks job as suspicious as soon as it is started" do
|
85
|
+
assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
|
86
|
+
HardWorker.perform_async(2, 100_000)
|
87
|
+
Thread.new{
|
88
|
+
SidekiqEmulator.instance.process_jobs
|
89
|
+
}
|
90
|
+
sleep(1) # TODO: refactor this somehow
|
91
|
+
assert_equal 1, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
|
92
|
+
end
|
93
|
+
|
94
|
+
it "removes suspicious mark as soon as job is finished succesfully" do
|
95
|
+
assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
|
96
|
+
HardWorker.perform_async(2, 1)
|
97
|
+
Thread.new{
|
98
|
+
@mutex.synchronize{
|
99
|
+
SidekiqEmulator.instance.process_jobs
|
100
|
+
@stopper.signal
|
101
|
+
}
|
102
|
+
}
|
103
|
+
@mutex.synchronize{ @stopper.wait(@mutex) }
|
104
|
+
assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
|
105
|
+
end
|
106
|
+
|
107
|
+
it "removes suspicious mark as soon as job failed" do
|
108
|
+
assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
|
109
|
+
HardWorker.perform_async(2, -1)
|
110
|
+
Thread.new{
|
111
|
+
@mutex.synchronize{
|
112
|
+
SidekiqEmulator.instance.process_jobs rescue nil
|
113
|
+
@stopper.signal
|
114
|
+
}
|
115
|
+
}
|
116
|
+
@mutex.synchronize{ @stopper.wait(@mutex) }
|
117
|
+
assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::Middleware::REDIS_SUSPICIOUS_KEY)}.size
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
|
2
|
+
|
3
|
+
require 'pry'
|
4
|
+
require 'sidekiq'
|
5
|
+
require 'sidekiq/api'
|
6
|
+
require 'sidekiq/cli'
|
7
|
+
require 'sidekiq/processor'
|
8
|
+
require 'sidekiq/manager'
|
9
|
+
require 'sidekiq/util'
|
10
|
+
require 'sidekiq/redis_connection'
|
11
|
+
require 'redis-namespace'
|
12
|
+
require 'attentive_sidekiq'
|
13
|
+
require 'minitest/autorun'
|
14
|
+
require 'minitest/pride'
|
15
|
+
|
16
|
+
REDIS_URL = ENV["REDIS_URL"] || "redis://localhost:15"
|
17
|
+
REDIS_NAMESPACE = ENV["REDIS_NAMESPACE"] || 'testy'
|
18
|
+
REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => REDIS_NAMESPACE)
|
19
|
+
|
20
|
+
Sidekiq.configure_server do |config|
|
21
|
+
config.server_middleware do |chain|
|
22
|
+
chain.add AttentiveSidekiq::Middleware::Server::Attentionist
|
23
|
+
end
|
24
|
+
end
|
data/test/web_test.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
require "rack/test"
|
3
|
+
|
4
|
+
class WebTest < Minitest::Test
|
5
|
+
include Rack::Test::Methods
|
6
|
+
|
7
|
+
def setup
|
8
|
+
Sidekiq.redis = REDIS
|
9
|
+
Sidekiq.redis{ |c| c.flushdb }
|
10
|
+
|
11
|
+
common_hash = {'class' => 'UnexistantWorker', 'args' => [], 'created_at' => Time.now.to_i}
|
12
|
+
@item_disappeared = {'jid' => "YE11OW5-247", 'queue' => 'yellow_queue'}.merge!(common_hash)
|
13
|
+
@item_in_progress = {'jid' => "REDA-257513", 'queue' => 'red_queue'}.merge!(common_hash)
|
14
|
+
|
15
|
+
AttentiveSidekiq::Suspicious.add(@item_in_progress)
|
16
|
+
AttentiveSidekiq::Suspicious.add(@item_disappeared)
|
17
|
+
AttentiveSidekiq::Disappeared.add(@item_disappeared)
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_displays_jobs_in_disappeared_hash
|
21
|
+
get '/disappeared-jobs'
|
22
|
+
assert_equal 200, last_response.status
|
23
|
+
assert_match @item_disappeared['jid'], last_response.body
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_does_not_display_jobs_not_in_disappeared_hash
|
27
|
+
get '/disappeared-jobs'
|
28
|
+
assert_equal 200, last_response.status
|
29
|
+
refute_match @item_in_progress['jid'], last_response.body
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def app
|
35
|
+
Sidekiq::Web
|
36
|
+
end
|
37
|
+
end
|
data/web.png
ADDED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: attentive_sidekiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- twonegatives
|
@@ -9,7 +9,105 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
date: 2016-10-31 00:00:00.000000000 Z
|
12
|
-
dependencies:
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: sidekiq
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '4.2'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '4.2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '11.3'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '11.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: minitest
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '5.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '5.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: redis-namespace
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.5'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.5'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rack-test
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.6'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.6'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pry
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0.10'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0.10'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: concurrent-ruby
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1.0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '1.0'
|
13
111
|
description: This gem allows you to watch the jobs which suddenly dissappeared from
|
14
112
|
redis without being completed by redis worker
|
15
113
|
email: whitewhiteheaven@gmail.com
|
@@ -17,16 +115,29 @@ executables: []
|
|
17
115
|
extensions: []
|
18
116
|
extra_rdoc_files: []
|
19
117
|
files:
|
118
|
+
- Gemfile
|
119
|
+
- Gemfile.lock
|
20
120
|
- README.md
|
121
|
+
- Rakefile
|
21
122
|
- attentive_sidekiq.gemspec
|
123
|
+
- circle.yml
|
22
124
|
- lib/attentive_sidekiq.rb
|
125
|
+
- lib/attentive_sidekiq/api.rb
|
126
|
+
- lib/attentive_sidekiq/manager.rb
|
23
127
|
- lib/attentive_sidekiq/middleware.rb
|
24
128
|
- lib/attentive_sidekiq/middleware/client/attentionist.rb
|
25
129
|
- lib/attentive_sidekiq/middleware/server/attentionist.rb
|
130
|
+
- lib/attentive_sidekiq/updater_observer.rb
|
131
|
+
- lib/attentive_sidekiq/version.rb
|
26
132
|
- lib/attentive_sidekiq/web.rb
|
27
133
|
- lib/attentive_sidekiq/web/locales/en.yml
|
28
134
|
- lib/attentive_sidekiq/web/locales/ru.yml
|
29
135
|
- lib/attentive_sidekiq/web/views/disappeared-list.erb
|
136
|
+
- test/manager_test.rb
|
137
|
+
- test/server_middleware_test.rb
|
138
|
+
- test/test_helper.rb
|
139
|
+
- test/web_test.rb
|
140
|
+
- web.png
|
30
141
|
homepage: http://rubygems.org/gems/attentive_sidekiq
|
31
142
|
licenses:
|
32
143
|
- MIT
|
@@ -47,8 +158,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
47
158
|
version: '0'
|
48
159
|
requirements: []
|
49
160
|
rubyforge_project:
|
50
|
-
rubygems_version: 2.
|
161
|
+
rubygems_version: 2.2.2
|
51
162
|
signing_key:
|
52
163
|
specification_version: 4
|
53
164
|
summary: Make your sidekiq to be attentive to lost jobs
|
54
|
-
test_files:
|
165
|
+
test_files:
|
166
|
+
- test/manager_test.rb
|
167
|
+
- test/server_middleware_test.rb
|
168
|
+
- test/test_helper.rb
|
169
|
+
- test/web_test.rb
|