resque_stuck_queue 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +27 -28
- data/THOUGHTS +7 -0
- data/lib/resque_stuck_queue.rb +70 -43
- data/lib/resque_stuck_queue/config.rb +60 -0
- data/lib/resque_stuck_queue/version.rb +1 -1
- data/test/{test_logger.rb → test_config.rb} +12 -1
- data/test/test_helper.rb +9 -1
- data/test/test_integration.rb +48 -42
- data/test/test_lagtime.rb +34 -0
- data/test/test_named_queues.rb +4 -4
- data/test/test_resque_stuck_queue.rb +3 -3
- metadata +7 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
SHA512:
|
|
3
|
-
metadata.gz: df4e25ba257df91726f7c92cfe1a92f1a04b2afecb0106a4a80d0da12b494c67672fd5c40a981c813b4e18ede7d18b5e2aba3c8a445692e910acc17b9adc360e
|
|
4
|
-
data.tar.gz: 22f4307be4f19c7e3b138c4d18013ac8fefc3a7a4d53a4071c6a6f8c1f32ec9dce2f187db9476e12d4f871a332ba6d9842fd6eb6d2ac6bf576dad79522eae783
|
|
5
2
|
SHA1:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5d18f8d4224a9e8a9d30495ae0c7b6f3f0901f63
|
|
4
|
+
data.tar.gz: fd796adf7d09bb92e76c4806df3f725106d04d7a
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 476f6beaff11089bcd25c08428bdf6e7355af941ca5500c7e941c70addc5ed542e61a2f9d7d5df09970270a7401a880a0e3e58939139f00d3c5879222bc3d4f9
|
|
7
|
+
data.tar.gz: 6325fd7e73422455a97ea5479c82d36ef131ad5ba05a982db92d106bf088cd4804dd1126391c377bba7c8a538b56059164a303596123ab3f742c3c719ea71b87
|
data/README.md
CHANGED
|
@@ -20,42 +20,41 @@ It will trigger a pre-defined proc (see below) if the last time the hearbeat job
|
|
|
20
20
|
|
|
21
21
|
## Usage
|
|
22
22
|
|
|
23
|
-
Configure it first
|
|
23
|
+
Configure it first. Optional settings are below. You'll most likely at the least want to tune `:handler`,`:heartbeat` and `:trigger_timeout` settings.
|
|
24
24
|
|
|
25
25
|
<pre>
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
handler:
|
|
27
|
+
set to what gets triggered when resque-stuck-queue will detect the latest heartbeat is older than the trigger_timeout time setting.
|
|
28
|
+
Example:
|
|
29
|
+
Resque::StuckQueue.config[:handler] = proc { |queue_name, lagtime| send_email('queue #{queue_name} isnt working, aaah the daemons') }
|
|
28
30
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
Resque::StuckQueue.config[:trigger_timeout] = 10.hours
|
|
31
|
+
heartbeat:
|
|
32
|
+
set to how often to push that 'heartbeat' job to refresh the latest time it worked.
|
|
33
|
+
Example:
|
|
34
|
+
Resque::StuckQueue.config[:heartbeat] = 5.minutes
|
|
34
35
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
Resque::StuckQueue.config[:handler] = proc { |queue_name| send_email("queue #{queue_name} isnt working, aaah the daemons") }
|
|
36
|
+
trigger_timeout:
|
|
37
|
+
set to how much of a resque work lag you are willing to accept before being notified. note: take the :heartbeat setting into account when setting this timeout.
|
|
38
|
+
Example:
|
|
39
|
+
Resque::StuckQueue.config[:trigger_timeout] = 55.minutes
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
Resque::StuckQueue.config[:global_key] = "name-the-refresh-key-as-you-please"
|
|
41
|
+
redis:
|
|
42
|
+
set the Redis instance StuckQueue will use
|
|
44
43
|
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
heartbeat_key:
|
|
45
|
+
optional, name of keys to keep track of the last good resque heartbeat time
|
|
47
46
|
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
logger:
|
|
48
|
+
optional, pass a Logger. Default a ruby logger will be instantiated. Needs to respond to that interface.
|
|
50
49
|
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
queues:
|
|
51
|
+
optional, monitor specific queues you want to send a heartbeat/monitor to. default is :app
|
|
53
52
|
|
|
54
|
-
|
|
55
|
-
|
|
53
|
+
abort_on_exception:
|
|
54
|
+
optional, if you want the resque-stuck-queue threads to explicitly raise, default is false
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
|
|
56
|
+
refresh_job:
|
|
57
|
+
optional, your own custom refreshing job. if you are using something other than resque
|
|
59
58
|
</pre>
|
|
60
59
|
|
|
61
60
|
Then start it:
|
|
@@ -110,7 +109,7 @@ $ bundle exec rake --trace resque:stuck_queue
|
|
|
110
109
|
|
|
111
110
|
## Sidekiq/Other redis-based job queues
|
|
112
111
|
|
|
113
|
-
If you have trouble with other queues you can use this lib by setting your own custom refresh job (aka, the job that refreshes your queue specific
|
|
112
|
+
If you have trouble with other queues you can use this lib by setting your own custom refresh job (aka, the job that refreshes your queue specific heartbeat_key). The one thing you need to take care of is ensure whatever and however you enque your own custom job, it sets the heartbeat_key to Time.now:
|
|
114
113
|
|
|
115
114
|
<pre>
|
|
116
115
|
|
|
@@ -118,7 +117,7 @@ class CustomJob
|
|
|
118
117
|
include Sidekiq::Worker
|
|
119
118
|
def perform
|
|
120
119
|
# ensure you're setting the key in the redis the job queue is using
|
|
121
|
-
$redis.set(Resque::StuckQueue.
|
|
120
|
+
$redis.set(Resque::StuckQueue.heartbeat_key_for(queue_name), Time.now.to_i)
|
|
122
121
|
end
|
|
123
122
|
end
|
|
124
123
|
|
data/THOUGHTS
CHANGED
|
@@ -14,3 +14,10 @@ add a trap{} to force_stop. ok for overwriting process's trap handlers? use conf
|
|
|
14
14
|
|
|
15
15
|
fix skeleton recipe https://github.com/shaiguitar/resque_stuck_queue/blame/master/README.md#L103
|
|
16
16
|
raise appname, => :environment, log path
|
|
17
|
+
|
|
18
|
+
set a redis key instead of doing a manual refresh and have a recover handler. also pass the lag time into the handlers/procs (same semantics)
|
|
19
|
+
investigate: why is temple getting triggered? how often? is the enqueing/checking taking too much time?
|
|
20
|
+
also, if one queue is bad, does it trigger other queue's handlers? write some tests, asshole.
|
|
21
|
+
woes of redis namespace, in regards to Awsm.redis != Resque.redis etc. (which is important for setting the key through @redis)
|
|
22
|
+
|
|
23
|
+
with lag time, it will continue to trigger, for every heartbeat time it's supposed to tick, find some way to do that, and then maybe add some resolved handler/proc?
|
data/lib/resque_stuck_queue.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "resque_stuck_queue/version"
|
|
2
|
+
require "resque_stuck_queue/config"
|
|
2
3
|
|
|
3
4
|
# TODO move this require into a configurable?
|
|
4
5
|
require 'resque'
|
|
@@ -11,35 +12,12 @@ require 'logger'
|
|
|
11
12
|
module Resque
|
|
12
13
|
module StuckQueue
|
|
13
14
|
|
|
14
|
-
GLOBAL_KEY = "resque-stuck-queue"
|
|
15
|
-
HEARTBEAT = 60 * 60 # check/refresh every hour
|
|
16
|
-
TRIGGER_TIMEOUT = 5 * 60 * 60 # warn/trigger 5 hours
|
|
17
|
-
HANDLER = proc { |queue_name| $stdout.puts("Shit gone bad with them queues...on #{queue_name}.") }
|
|
18
|
-
|
|
19
15
|
class << self
|
|
20
16
|
|
|
21
17
|
attr_accessor :config
|
|
22
18
|
|
|
23
|
-
# # how often we refresh the key
|
|
24
|
-
# :heartbeat = 5 * 60
|
|
25
|
-
#
|
|
26
|
-
# # this could just be :heartbeat but it's possible there's an acceptable lag/bottleneck
|
|
27
|
-
# # in the queue that we want to allow to be before we think it's bad.
|
|
28
|
-
# :trigger_timeout = 10 * 60
|
|
29
|
-
#
|
|
30
|
-
# # The global key that will be used to check the latest time
|
|
31
|
-
# :global_key = "resque-stuck-queue"
|
|
32
|
-
#
|
|
33
|
-
# # for threads involved here. default is false
|
|
34
|
-
# :abort_on_exception
|
|
35
|
-
#
|
|
36
|
-
# # default handler
|
|
37
|
-
# config[:handler] = proc { |queue_name| send_mail }
|
|
38
|
-
#
|
|
39
|
-
# # explicit redis
|
|
40
|
-
# config[:redis] = Redis.new
|
|
41
19
|
def config
|
|
42
|
-
@config ||=
|
|
20
|
+
@config ||= Config.new
|
|
43
21
|
end
|
|
44
22
|
|
|
45
23
|
def logger
|
|
@@ -56,12 +34,16 @@ module Resque
|
|
|
56
34
|
Resque.redis = @redis
|
|
57
35
|
end
|
|
58
36
|
|
|
59
|
-
def
|
|
60
|
-
"#{
|
|
37
|
+
def heartbeat_key_for(queue)
|
|
38
|
+
"#{queue}:#{config[:heartbeat_key] || HEARTBEAT_KEY}"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def triggered_key_for(queue)
|
|
42
|
+
"#{queue}:#{config[:triggered_key] || TRIGGERED_KEY}"
|
|
61
43
|
end
|
|
62
44
|
|
|
63
|
-
def
|
|
64
|
-
queues.map{|q|
|
|
45
|
+
def heartbeat_keys
|
|
46
|
+
queues.map{|q| heartbeat_key_for(q) }
|
|
65
47
|
end
|
|
66
48
|
|
|
67
49
|
def queues
|
|
@@ -130,7 +112,7 @@ module Resque
|
|
|
130
112
|
while @running
|
|
131
113
|
# we want to go through resque jobs, because that's what we're trying to test here:
|
|
132
114
|
# ensure that jobs get executed and the time is updated!
|
|
133
|
-
logger.info("Sending refresh
|
|
115
|
+
logger.info("Sending refresh jobs")
|
|
134
116
|
enqueue_jobs
|
|
135
117
|
wait_for_it
|
|
136
118
|
end
|
|
@@ -143,7 +125,7 @@ module Resque
|
|
|
143
125
|
config[:refresh_job].call
|
|
144
126
|
else
|
|
145
127
|
queues.each do |queue_name|
|
|
146
|
-
Resque.enqueue_to(queue_name, RefreshLatestTimestamp, [
|
|
128
|
+
Resque.enqueue_to(queue_name, RefreshLatestTimestamp, [heartbeat_key_for(queue_name), redis.client.host, redis.client.port])
|
|
147
129
|
end
|
|
148
130
|
end
|
|
149
131
|
end
|
|
@@ -157,8 +139,11 @@ module Resque
|
|
|
157
139
|
if mutex.lock
|
|
158
140
|
begin
|
|
159
141
|
queues.each do |queue_name|
|
|
160
|
-
if
|
|
161
|
-
|
|
142
|
+
logger.info("Checking if queue #{queue_name} is lagging")
|
|
143
|
+
# else TODO recovered?
|
|
144
|
+
if should_trigger?(queue_name)
|
|
145
|
+
logger.info("Triggering handler for #{queue_name} at #{Time.now}.")
|
|
146
|
+
logger.info("Lag time for #{queue_name} is #{lag_time(queue_name)} seconds.")
|
|
162
147
|
trigger_handler(queue_name)
|
|
163
148
|
end
|
|
164
149
|
end
|
|
@@ -171,35 +156,77 @@ module Resque
|
|
|
171
156
|
end
|
|
172
157
|
end
|
|
173
158
|
|
|
174
|
-
def
|
|
175
|
-
|
|
159
|
+
#def recovered?
|
|
160
|
+
# already triggered once (last_trigger is not nil), but lag time is ok.
|
|
161
|
+
# then over here we'll rm last_triggered
|
|
162
|
+
# and fire a recovered handler. by rm the last_triggered the next time
|
|
163
|
+
# there is a problem, should_trigger? should return true
|
|
164
|
+
#end
|
|
165
|
+
|
|
166
|
+
def last_successful_heartbeat(queue_name)
|
|
167
|
+
time_set = read_from_redis(heartbeat_key_for(queue_name))
|
|
176
168
|
if time_set
|
|
177
169
|
time_set
|
|
178
170
|
else
|
|
171
|
+
# the first time this is being used, key wont be there
|
|
172
|
+
# so just start now.
|
|
179
173
|
manual_refresh(queue_name)
|
|
180
174
|
end.to_i
|
|
181
175
|
end
|
|
182
176
|
|
|
183
|
-
def manual_refresh(queue_name)
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
177
|
+
def manual_refresh(queue_name, type = :first_time)
|
|
178
|
+
if type == :triggered
|
|
179
|
+
time = Time.now.to_i
|
|
180
|
+
redis.set(triggered_key_for(queue_name), time)
|
|
181
|
+
time
|
|
182
|
+
elsif type == :first_time
|
|
183
|
+
time = Time.now.to_i
|
|
184
|
+
redis.set(heartbeat_key_for(queue_name), time)
|
|
185
|
+
time
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def lag_time(queue_name)
|
|
190
|
+
Time.now.to_i - last_successful_heartbeat(queue_name)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def last_triggered(queue_name)
|
|
194
|
+
time_set = read_from_redis(triggered_key_for(queue_name))
|
|
195
|
+
if !time_set.nil?
|
|
196
|
+
Time.now.to_i - time_set.to_i
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def should_trigger?(queue_name)
|
|
201
|
+
if lag_time(queue_name) > max_wait_time
|
|
202
|
+
last_trigger = last_triggered(queue_name)
|
|
203
|
+
|
|
204
|
+
if last_trigger.nil?
|
|
205
|
+
return true
|
|
206
|
+
elsif last_trigger > max_wait_time
|
|
207
|
+
return true
|
|
208
|
+
else
|
|
209
|
+
# if we've already triggered, the next trigger should be on the next iteration of max_wait_time.
|
|
210
|
+
return false
|
|
211
|
+
end
|
|
212
|
+
end
|
|
187
213
|
end
|
|
188
214
|
|
|
189
215
|
def trigger_handler(queue_name)
|
|
190
|
-
(config[:handler] || HANDLER).call(queue_name)
|
|
191
|
-
manual_refresh(queue_name)
|
|
216
|
+
(config[:handler] || HANDLER).call(queue_name, lag_time(queue_name))
|
|
217
|
+
manual_refresh(queue_name, :triggered)
|
|
192
218
|
rescue => e
|
|
193
219
|
logger.info("handler for #{queue_name} crashed: #{e.inspect}")
|
|
220
|
+
logger.info("\n#{e.backtrace.join("\n")}")
|
|
194
221
|
force_stop!
|
|
195
222
|
end
|
|
196
223
|
|
|
197
|
-
def read_from_redis(
|
|
198
|
-
redis.get(
|
|
224
|
+
def read_from_redis(keyname)
|
|
225
|
+
redis.get(keyname)
|
|
199
226
|
end
|
|
200
227
|
|
|
201
228
|
def wait_for_it
|
|
202
|
-
sleep config[:heartbeat] ||
|
|
229
|
+
sleep config[:heartbeat] || HEARTBEAT_TIMEOUT
|
|
203
230
|
end
|
|
204
231
|
|
|
205
232
|
def max_wait_time
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
module Resque
|
|
2
|
+
module StuckQueue
|
|
3
|
+
|
|
4
|
+
# defaults
|
|
5
|
+
HEARTBEAT_KEY = "resque-stuck-queue"
|
|
6
|
+
TRIGGERED_KEY = "resque-stuck-queue-last-triggered"
|
|
7
|
+
HEARTBEAT_TIMEOUT = 20 * 60 # check/refresh every 20 mins.
|
|
8
|
+
TRIGGER_TIMEOUT = 40 * 60 # warn/trigger after an hour (with 20 min heartbeat time).
|
|
9
|
+
HANDLER = proc { |queue_name, lag| $stdout.puts("Shit gone bad with them queues...on #{queue_name}. Lag time is #{lag}") }
|
|
10
|
+
|
|
11
|
+
class Config < Hash
|
|
12
|
+
|
|
13
|
+
OPTIONS_DESCRIPTIONS = {
|
|
14
|
+
:handler => "set to what gets triggered when resque-stuck-queue will detect the latest heartbeat is older than the trigger_timeout time setting.\n\tExample:\n\tResque::StuckQueue.config[:handler] = proc { |queue_name, lagtime| send_email('queue \#{queue_name} isnt working, aaah the daemons') }",
|
|
15
|
+
:heartbeat => "set to how often to push that 'heartbeat' job to refresh the latest time it worked.\n\tExample:\n\tResque::StuckQueue.config[:heartbeat] = 5.minutes",
|
|
16
|
+
:trigger_timeout => "set to how much of a resque work lag you are willing to accept before being notified. note: take the :heartbeat setting into account when setting this timeout.\n\tExample:\n\tResque::StuckQueue.config[:trigger_timeout] = 55.minutes",
|
|
17
|
+
:redis => "set the Redis instance StuckQueue will use",
|
|
18
|
+
:heartbeat_key => "optional, name of keys to keep track of the last good resque heartbeat time",
|
|
19
|
+
:triggered_key => "optional, name of keys to keep track of the last trigger time",
|
|
20
|
+
:logger => "optional, pass a Logger. Default a ruby logger will be instantiated. Needs to respond to that interface.",
|
|
21
|
+
:queues => "optional, monitor specific queues you want to send a heartbeat/monitor to. default is :app",
|
|
22
|
+
:abort_on_exception => "optional, if you want the resque-stuck-queue threads to explicitly raise, default is false",
|
|
23
|
+
:refresh_job => "optional, your own custom refreshing job. if you are using something other than resque",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
OPTIONS = OPTIONS_DESCRIPTIONS.keys
|
|
27
|
+
|
|
28
|
+
def []=(k,v)
|
|
29
|
+
validate_key_exists!(k)
|
|
30
|
+
super(k,v)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def [](k)
|
|
34
|
+
validate_key_exists!(k)
|
|
35
|
+
super(k)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
class NoConfigError < StandardError; end
|
|
39
|
+
|
|
40
|
+
def validate_key_exists!(k)
|
|
41
|
+
if ! OPTIONS.include?(k)
|
|
42
|
+
raise NoConfigError, "no such config key exists!"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def description_for(k)
|
|
47
|
+
OPTIONS_DESCRIPTIONS[k.to_sym]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def pretty_descriptions
|
|
51
|
+
out = "\n"
|
|
52
|
+
OPTIONS_DESCRIPTIONS.map{|key,msg|
|
|
53
|
+
out << "#{key}:\n\t#{msg}\n\n"
|
|
54
|
+
}
|
|
55
|
+
out
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
|
|
2
2
|
|
|
3
|
-
class
|
|
3
|
+
class TestConfig < Minitest::Test
|
|
4
4
|
|
|
5
5
|
include TestHelper
|
|
6
6
|
|
|
@@ -14,6 +14,17 @@ class TestResqueStuckQueue < Minitest::Test
|
|
|
14
14
|
Resque::StuckQueue.reset!
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
+
def test_config_has_descriptions
|
|
18
|
+
c = Resque::StuckQueue::Config.new
|
|
19
|
+
assert c.description_for(:logger) =~ /Logger/, "has descriptions"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def test_outputs_all_config_options
|
|
23
|
+
c = Resque::StuckQueue::Config.new
|
|
24
|
+
puts c.pretty_descriptions
|
|
25
|
+
assert true
|
|
26
|
+
end
|
|
27
|
+
|
|
17
28
|
def test_has_logger
|
|
18
29
|
puts "#{__method__}"
|
|
19
30
|
begin
|
data/test/test_helper.rb
CHANGED
|
@@ -11,12 +11,20 @@ require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "refresh_l
|
|
|
11
11
|
|
|
12
12
|
module TestHelper
|
|
13
13
|
|
|
14
|
+
extend self
|
|
15
|
+
|
|
14
16
|
def run_resque(queue_name = "*")
|
|
15
|
-
pid = fork { exec("QUEUE=#{queue_name} bundle exec rake --trace resque:work") }
|
|
17
|
+
pid = fork { exec("export QUEUE=#{queue_name}; bundle exec rake --trace resque:work") }
|
|
16
18
|
sleep 3 # wait for resque to boot up
|
|
17
19
|
pid
|
|
18
20
|
end
|
|
19
21
|
|
|
22
|
+
def hax_kill_resque
|
|
23
|
+
# ugly, FIXME how to get pid of forked forked process. run_resque pid is incorrect.
|
|
24
|
+
`ps aux |grep resque |awk '{print $2}' |xargs kill`
|
|
25
|
+
sleep 2 # wait for shutdown
|
|
26
|
+
end
|
|
27
|
+
|
|
20
28
|
def start_and_stop_loops_after(secs)
|
|
21
29
|
ops = []
|
|
22
30
|
ops << Thread.new { Resque::StuckQueue.start }
|
data/test/test_integration.rb
CHANGED
|
@@ -12,84 +12,91 @@ require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
|
|
|
12
12
|
class TestIntegration < Minitest::Test
|
|
13
13
|
|
|
14
14
|
include TestHelper
|
|
15
|
-
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
15
|
+
|
|
16
|
+
# UBER HAXING no after(:all) or before(:all)
|
|
17
|
+
# if adding a test here, add
|
|
18
|
+
# self.class.tests_ran += 1
|
|
19
|
+
class << self
|
|
20
|
+
def tests_running?
|
|
21
|
+
test_count = public_instance_methods.select{|m| m.to_s.match(/^test_/)}.size
|
|
22
|
+
true if tests_ran != test_count
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def tests_done?
|
|
26
|
+
!tests_running?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
attr_accessor :tests_ran, :resque_pid
|
|
30
|
+
def tests_ran
|
|
31
|
+
@tests_ran ||= 0
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def run_resque_before_all
|
|
35
|
+
return if @running_resque
|
|
36
|
+
@running_resque = true
|
|
37
|
+
|
|
38
|
+
@resque_pid = TestHelper.run_resque
|
|
39
|
+
end
|
|
40
|
+
end
|
|
26
41
|
|
|
27
42
|
def setup
|
|
28
43
|
Resque::StuckQueue.redis = Redis.new
|
|
29
44
|
Resque::StuckQueue.redis.flushall
|
|
30
45
|
Resque::StuckQueue.config[:abort_on_exception] = true
|
|
46
|
+
self.class.run_resque_before_all
|
|
31
47
|
end
|
|
32
48
|
|
|
33
49
|
def teardown
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
50
|
+
if self.class.tests_done?
|
|
51
|
+
hax_kill_resque
|
|
52
|
+
Process.waitall
|
|
53
|
+
return
|
|
54
|
+
end
|
|
37
55
|
end
|
|
38
56
|
|
|
39
57
|
def test_resque_enqueues_a_job_does_not_trigger
|
|
40
58
|
puts "#{__method__}"
|
|
41
|
-
|
|
42
|
-
|
|
59
|
+
self.class.tests_ran += 1
|
|
60
|
+
|
|
61
|
+
Resque::StuckQueue.config[:trigger_timeout] = 10
|
|
62
|
+
Resque::StuckQueue.config[:heartbeat] = 1
|
|
43
63
|
@triggered = false
|
|
44
64
|
Resque::StuckQueue.config[:handler] = proc { @triggered = true }
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
# job gets enqueued successfully
|
|
48
|
-
@resque_pid = run_resque
|
|
65
|
+
start_and_stop_loops_after(5)
|
|
49
66
|
Resque::StuckQueue.redis.del(SetRedisKey::NAME)
|
|
50
67
|
Resque.enqueue_to(:app, SetRedisKey)
|
|
51
|
-
sleep
|
|
52
|
-
assert_equal Resque::StuckQueue.redis.get(SetRedisKey::NAME), "1"
|
|
53
|
-
|
|
54
|
-
# check handler did not get called
|
|
68
|
+
sleep 3
|
|
69
|
+
assert_equal Resque::StuckQueue.redis.get(SetRedisKey::NAME), "1"
|
|
70
|
+
# job ran successfully, so don't trigger
|
|
55
71
|
assert_equal @triggered, false
|
|
56
72
|
end
|
|
57
73
|
|
|
58
74
|
def test_resque_does_not_enqueues_a_job_does_trigger
|
|
59
75
|
puts "#{__method__}"
|
|
60
|
-
|
|
76
|
+
self.class.tests_ran += 1
|
|
77
|
+
|
|
78
|
+
Resque::StuckQueue.config[:trigger_timeout] = 0
|
|
61
79
|
Resque::StuckQueue.config[:heartbeat] = 1
|
|
62
80
|
@triggered = false
|
|
63
81
|
Resque::StuckQueue.config[:handler] = proc { @triggered = true }
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
# job gets enqueued successfully
|
|
67
|
-
@resque_pid = run_resque
|
|
68
|
-
Resque::StuckQueue.redis.del(SetRedisKey::NAME)
|
|
69
|
-
Process.kill("SIGSTOP", @resque_pid) # jic, do not process jobs so we definitely trigger
|
|
70
|
-
Resque.enqueue(SetRedisKey)
|
|
71
|
-
assert_equal Resque::StuckQueue.redis.get(SetRedisKey::NAME), nil
|
|
72
|
-
sleep 2 # allow timeout to trigger
|
|
73
|
-
|
|
82
|
+
start_and_stop_loops_after(2)
|
|
74
83
|
# check handler did get called
|
|
75
84
|
assert_equal @triggered, true
|
|
76
|
-
|
|
77
|
-
# unstick the process so we can kill it in teardown
|
|
78
|
-
Process.kill("SIGCONT", @resque_pid)
|
|
79
85
|
end
|
|
80
86
|
|
|
81
87
|
def test_has_settable_custom_hearbeat_job
|
|
82
88
|
puts "#{__method__}"
|
|
89
|
+
self.class.tests_ran += 1
|
|
90
|
+
|
|
83
91
|
Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
|
|
84
92
|
Resque::StuckQueue.config[:heartbeat] = 1
|
|
85
93
|
|
|
86
94
|
begin
|
|
87
|
-
Resque::StuckQueue.config[:refresh_job] = proc { Resque.enqueue(RefreshLatestTimestamp, Resque::StuckQueue.
|
|
95
|
+
Resque::StuckQueue.config[:refresh_job] = proc { Resque.enqueue(RefreshLatestTimestamp, Resque::StuckQueue.heartbeat_key_for(:app)) }
|
|
88
96
|
@triggered = false
|
|
89
97
|
Resque::StuckQueue.config[:handler] = proc { @triggered = true }
|
|
90
|
-
|
|
98
|
+
start_and_stop_loops_after(4)
|
|
91
99
|
|
|
92
|
-
@resque_pid = run_resque
|
|
93
100
|
sleep 3 # allow trigger
|
|
94
101
|
assert true, "should not have raised"
|
|
95
102
|
assert @triggered, "should have triggered"
|
|
@@ -98,5 +105,4 @@ class TestIntegration < Minitest::Test
|
|
|
98
105
|
end
|
|
99
106
|
end
|
|
100
107
|
|
|
101
|
-
|
|
102
108
|
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
require 'minitest'
|
|
2
|
+
require "minitest/autorun"
|
|
3
|
+
require 'pry'
|
|
4
|
+
|
|
5
|
+
$:.unshift(".")
|
|
6
|
+
require 'resque_stuck_queue'
|
|
7
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "set_redis_key")
|
|
8
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "refresh_latest_timestamp")
|
|
9
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
|
|
10
|
+
|
|
11
|
+
class TestLagTime < Minitest::Test
|
|
12
|
+
|
|
13
|
+
include TestHelper
|
|
14
|
+
|
|
15
|
+
def setup
|
|
16
|
+
Resque::StuckQueue.redis = Redis.new
|
|
17
|
+
Resque::StuckQueue.redis.flushall
|
|
18
|
+
Resque::StuckQueue.config[:abort_on_exception] = true
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def test_triggers_handler_with_lagtime
|
|
22
|
+
Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
|
|
23
|
+
Resque::StuckQueue.config[:heartbeat] = 1
|
|
24
|
+
@lagtime = 0
|
|
25
|
+
Resque::StuckQueue.config[:handler] = proc { |queue_name, lagtime| @lagtime = lagtime }
|
|
26
|
+
start_and_stop_loops_after(5)
|
|
27
|
+
|
|
28
|
+
# check handler did get called
|
|
29
|
+
assert @lagtime > 0, "lagtime shoudl be set"
|
|
30
|
+
assert @lagtime < 5, "lagtime shoudl be set"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
end
|
data/test/test_named_queues.rb
CHANGED
|
@@ -13,7 +13,7 @@ class TestNamedQueues < Minitest::Test
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
def teardown
|
|
16
|
-
|
|
16
|
+
hax_kill_resque
|
|
17
17
|
Resque::StuckQueue.force_stop!
|
|
18
18
|
Process.waitpid(@resque_pid) if @resque_pid
|
|
19
19
|
end
|
|
@@ -22,14 +22,14 @@ class TestNamedQueues < Minitest::Test
|
|
|
22
22
|
puts "#{__method__}"
|
|
23
23
|
Resque::StuckQueue.config[:queues] = nil
|
|
24
24
|
start_and_stop_loops_after(2)
|
|
25
|
-
assert Resque::StuckQueue.
|
|
25
|
+
assert Resque::StuckQueue.heartbeat_keys.include?("app:resque-stuck-queue"), 'has global keys'
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def test_has_custom_queues
|
|
29
29
|
puts "#{__method__}"
|
|
30
30
|
Resque::StuckQueue.config[:queues] = [:foo,:bar]
|
|
31
31
|
start_and_stop_loops_after(2)
|
|
32
|
-
assert Resque::StuckQueue.
|
|
32
|
+
assert Resque::StuckQueue.heartbeat_keys.include?("foo:resque-stuck-queue"), 'has global keys'
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
def test_resque_enqueues_a_job_with_resqueue_running_but_on_that_queue_does_trigger
|
|
@@ -54,7 +54,7 @@ class TestNamedQueues < Minitest::Test
|
|
|
54
54
|
Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
|
|
55
55
|
Resque::StuckQueue.config[:heartbeat] = 1
|
|
56
56
|
Resque::StuckQueue.config[:queues] = [:custom_queue_name, :diff_one]
|
|
57
|
-
assert Resque::StuckQueue.
|
|
57
|
+
assert Resque::StuckQueue.heartbeat_keys.include?("custom_queue_name:resque-stuck-queue"), 'has global keys'
|
|
58
58
|
@triggered = false
|
|
59
59
|
Resque::StuckQueue.config[:handler] = proc { @triggered = true }
|
|
60
60
|
@resque_pid = run_resque("custom_queue_name")
|
|
@@ -19,11 +19,11 @@ class TestResqueStuckQueue < Minitest::Test
|
|
|
19
19
|
Resque::StuckQueue.config[:abort_on_exception] = true
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
def
|
|
22
|
+
def test_configure_heartbeat_key
|
|
23
23
|
puts "#{__method__}"
|
|
24
24
|
assert_nil Resque::StuckQueue.redis.get("it-is-configurable"), "global key should not be set"
|
|
25
|
-
Resque::StuckQueue.config[:
|
|
26
|
-
start_and_stop_loops_after(
|
|
25
|
+
Resque::StuckQueue.config[:heartbeat_key] = "it-is-configurable"
|
|
26
|
+
start_and_stop_loops_after(3)
|
|
27
27
|
refute_nil Resque::StuckQueue.redis.get("app:it-is-configurable"), "global key should be set"
|
|
28
28
|
end
|
|
29
29
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: resque_stuck_queue
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.13
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shai Rosenfeld
|
|
@@ -9,7 +9,7 @@ autorequire:
|
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
11
|
|
|
12
|
-
date: 2014-01-
|
|
12
|
+
date: 2014-01-26 00:00:00 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: redis-mutex
|
|
@@ -58,14 +58,16 @@ files:
|
|
|
58
58
|
- THOUGHTS
|
|
59
59
|
- lib/resque/stuck_queue.rb
|
|
60
60
|
- lib/resque_stuck_queue.rb
|
|
61
|
+
- lib/resque_stuck_queue/config.rb
|
|
61
62
|
- lib/resque_stuck_queue/version.rb
|
|
62
63
|
- resque_stuck_queue.gemspec
|
|
63
64
|
- test/resque/refresh_latest_timestamp.rb
|
|
64
65
|
- test/resque/set_redis_key.rb
|
|
65
66
|
- test/test_collision.rb
|
|
67
|
+
- test/test_config.rb
|
|
66
68
|
- test/test_helper.rb
|
|
67
69
|
- test/test_integration.rb
|
|
68
|
-
- test/
|
|
70
|
+
- test/test_lagtime.rb
|
|
69
71
|
- test/test_named_queues.rb
|
|
70
72
|
- test/test_resque_2.rb
|
|
71
73
|
- test/test_resque_stuck_queue.rb
|
|
@@ -97,9 +99,10 @@ test_files:
|
|
|
97
99
|
- test/resque/refresh_latest_timestamp.rb
|
|
98
100
|
- test/resque/set_redis_key.rb
|
|
99
101
|
- test/test_collision.rb
|
|
102
|
+
- test/test_config.rb
|
|
100
103
|
- test/test_helper.rb
|
|
101
104
|
- test/test_integration.rb
|
|
102
|
-
- test/
|
|
105
|
+
- test/test_lagtime.rb
|
|
103
106
|
- test/test_named_queues.rb
|
|
104
107
|
- test/test_resque_2.rb
|
|
105
108
|
- test/test_resque_stuck_queue.rb
|