resque-batched-job 1.3.0 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
data/README.md
CHANGED
@@ -0,0 +1,72 @@
|
|
1
|
+
module Resque
|
2
|
+
module Plugins
|
3
|
+
module BatchedJobSupport
|
4
|
+
|
5
|
+
# Checks the size of the batched job list and returns true if the list is
|
6
|
+
# empty or if the key does not exist.
|
7
|
+
#
|
8
|
+
# @param batch_id (see Resque::Plugins::BatchedJob#batch)
|
9
|
+
def batch_complete?(batch_id)
|
10
|
+
mutex(batch_id) do |bid|
|
11
|
+
redis.llen(bid) == 0
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
# Check to see if the Redis key exists.
|
16
|
+
#
|
17
|
+
# @param batch_id (see Resque::Plugins::BatchedJob#batch)
|
18
|
+
def batch_exist?(batch_id)
|
19
|
+
mutex(batch_id) do |bid|
|
20
|
+
redis.exists(bid)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# Lock a batch key before executing Redis commands. This will ensure
|
27
|
+
# no race conditions occur when modifying batch information. Here is
|
28
|
+
# an example of how this works. See http://redis.io/commands/setnx for
|
29
|
+
# more information. (fixes #4) (closes #5)
|
30
|
+
#
|
31
|
+
# * Job2 sends SETNX batch:123:lock in order to aquire a lock.
|
32
|
+
# * Job1 still has the key locked, so Job2 continues into the loop.
|
33
|
+
# * Job2 sends GET to aquire the lock timestamp.
|
34
|
+
# * If the timestamp does not exist (Job1 released the lock), Job2
|
35
|
+
# attemps to start from the beginning again.
|
36
|
+
# * If the timestamp exists and has not expired, Job2 sleeps for a
|
37
|
+
# moment and then retries from the start.
|
38
|
+
# * If the timestamp exists and has expired, Job2 sends GETSET to aquire
|
39
|
+
# a lock. This returns the previous value of the lock.
|
40
|
+
# * If the previous timestamp has not expired, another process was faster
|
41
|
+
# and aquired the lock. This means Job2 has to start from the beginnig.
|
42
|
+
# * If the previous timestamp is still expired the lock has been set and
|
43
|
+
# processing can continue safely
|
44
|
+
#
|
45
|
+
# @param id (see Resque::Plugins::BatchedJob#batch)
|
46
|
+
# @yield [bid] Yields the current batch id.
|
47
|
+
# @yieldparam [String] The current batch id.
|
48
|
+
def mutex(id, &block)
|
49
|
+
is_expired = lambda do |locked_at|
|
50
|
+
locked_at.to_f < Time.now.to_f
|
51
|
+
end
|
52
|
+
bid = batch(id)
|
53
|
+
_key_ = "#{bid}:lock"
|
54
|
+
|
55
|
+
until redis.setnx(_key_, Time.now.to_f + 0.5)
|
56
|
+
next unless timestamp = redis.get(_key_)
|
57
|
+
|
58
|
+
unless is_expired.call(timestamp)
|
59
|
+
sleep(0.1)
|
60
|
+
next
|
61
|
+
end
|
62
|
+
|
63
|
+
break unless timestamp = redis.getset(_key_, Time.now.to_f + 0.5)
|
64
|
+
break if is_expired.call(timestamp)
|
65
|
+
end
|
66
|
+
yield(bid)
|
67
|
+
ensure
|
68
|
+
redis.del(_key_)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -45,9 +45,7 @@ module Resque
|
|
45
45
|
#
|
46
46
|
# @param id (see Resque::Plugins::BatchedJob#after_enqueue_batch)
|
47
47
|
def after_perform_batch(id, *args)
|
48
|
-
remove_batched_job(id, *args)
|
49
|
-
|
50
|
-
if batch_complete?(id)
|
48
|
+
if remove_batched_job(id, *args) == 0
|
51
49
|
after_batch_hooks = Resque::Plugin.after_batch_hooks(self)
|
52
50
|
after_batch_hooks.each do |hook|
|
53
51
|
send(hook, id, *args)
|
@@ -60,7 +58,7 @@ module Resque
|
|
60
58
|
#
|
61
59
|
# @param id (see Resque::Plugins::BatchedJob#batch)
|
62
60
|
def batch_complete?(id)
|
63
|
-
mutex(id) do |bid|
|
61
|
+
mutex(id) do |bid|
|
64
62
|
redis.llen(bid) == 0
|
65
63
|
end
|
66
64
|
end
|
@@ -69,7 +67,7 @@ module Resque
|
|
69
67
|
#
|
70
68
|
# @param id (see Resque::Plugins::BatchedJob#batch)
|
71
69
|
def batch_exist?(id)
|
72
|
-
mutex(id) do |bid|
|
70
|
+
mutex(id) do |bid|
|
73
71
|
redis.exists(bid)
|
74
72
|
end
|
75
73
|
end
|
@@ -80,6 +78,7 @@ module Resque
|
|
80
78
|
def remove_batched_job(id, *args)
|
81
79
|
mutex(id) do |bid|
|
82
80
|
redis.lrem(bid, 1, encode(:class => self.name, :args => args))
|
81
|
+
redis.llen(bid)
|
83
82
|
end
|
84
83
|
end
|
85
84
|
|
@@ -96,13 +95,13 @@ module Resque
|
|
96
95
|
# no race conditions occur when modifying batch information. Here is
|
97
96
|
# an example of how this works. See http://redis.io/commands/setnx for
|
98
97
|
# more information. (fixes #4) (closes #5)
|
99
|
-
#
|
98
|
+
#
|
100
99
|
# * Job2 sends SETNX batch:123:lock in order to aquire a lock.
|
101
100
|
# * Job1 still has the key locked, so Job2 continues into the loop.
|
102
101
|
# * Job2 sends GET to aquire the lock timestamp.
|
103
|
-
# * If the timestamp does not exist (Job1 released the lock), Job2
|
102
|
+
# * If the timestamp does not exist (Job1 released the lock), Job2
|
104
103
|
# attemps to start from the beginning again.
|
105
|
-
# * If the timestamp exists and has not expired, Job2 sleeps for a
|
104
|
+
# * If the timestamp exists and has not expired, Job2 sleeps for a
|
106
105
|
# moment and then retries from the start.
|
107
106
|
# * If the timestamp exists and has expired, Job2 sends GETSET to aquire
|
108
107
|
# a lock. This returns the previous value of the lock.
|
data/test/test_batched_job.rb
CHANGED
@@ -46,7 +46,7 @@ class BatchedJobTest < Test::Unit::TestCase
|
|
46
46
|
# Make sure the after_batch hook is fired
|
47
47
|
def test_batch_hook
|
48
48
|
assert_equal(false, Job.batch_exist?(@batch_id))
|
49
|
-
|
49
|
+
|
50
50
|
assert_nothing_raised do
|
51
51
|
5.times { Resque.enqueue(Job, @batch_id, "arg#{rand(100)}") }
|
52
52
|
end
|
@@ -129,6 +129,11 @@ class BatchedJobTest < Test::Unit::TestCase
|
|
129
129
|
end
|
130
130
|
|
131
131
|
def test_remove_batched_job
|
132
|
+
Resque.enqueue(JobWithoutArgs, @batch_id)
|
133
|
+
Resque.enqueue(JobWithoutArgs, @batch_id)
|
134
|
+
assert_equal(1, JobWithoutArgs.remove_batched_job(@batch_id))
|
135
|
+
assert_equal(0, JobWithoutArgs.remove_batched_job(@batch_id))
|
136
|
+
|
132
137
|
Resque.enqueue(JobWithoutArgs, @batch_id)
|
133
138
|
|
134
139
|
assert_nothing_raised do
|
@@ -159,4 +164,4 @@ class BatchedJobTest < Test::Unit::TestCase
|
|
159
164
|
Resque.redis
|
160
165
|
end
|
161
166
|
|
162
|
-
end
|
167
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-batched-job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-03-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: resque
|
16
|
-
requirement: &
|
16
|
+
requirement: &70106096870560 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,7 +21,7 @@ dependencies:
|
|
21
21
|
version: 1.10.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70106096870560
|
25
25
|
description: ! " Resque plugin for batching jobs. When a batch/group of jobs are
|
26
26
|
complete, \nadditional work can be performed usings batch hooks.\n"
|
27
27
|
email: dan@dj-agiledev.com
|
@@ -33,6 +33,7 @@ files:
|
|
33
33
|
- Rakefile
|
34
34
|
- README.md
|
35
35
|
- lib/resque/batched_job.rb
|
36
|
+
- lib/resque/plugins/batched_job/support.rb
|
36
37
|
- lib/resque/plugins/batched_job/tasks.rb
|
37
38
|
- lib/resque/plugins/batched_job/version.rb
|
38
39
|
- lib/resque/plugins/batched_job.rb
|