resque-batched-job 0.6.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +21 -6
- data/lib/resque/plugins/batched_job/version.rb +1 -1
- data/lib/resque/plugins/batched_job.rb +61 -4
- data/test/test_batched_job.rb +96 -30
- data/test/test_helper.rb +32 -0
- metadata +6 -4
data/Rakefile
CHANGED
@@ -2,6 +2,8 @@ require 'rubygems'
|
|
2
2
|
require 'rake'
|
3
3
|
require 'rake/testtask'
|
4
4
|
|
5
|
+
require File.dirname(__FILE__) + '/lib/resque/plugins/batched_job/version'
|
6
|
+
|
5
7
|
task :default => :test
|
6
8
|
|
7
9
|
Rake::TestTask.new do |task|
|
@@ -9,16 +11,29 @@ Rake::TestTask.new do |task|
|
|
9
11
|
task.verbose = true
|
10
12
|
end
|
11
13
|
|
12
|
-
desc "Publish
|
13
|
-
task :publish => :build do
|
14
|
-
require File.dirname(__FILE__) + '/lib/resque/plugins/batched_job/version'
|
15
|
-
sh "gem push resque-batched-job-#{Resque::Plugins::BatchedJob::VERSION}.gem"
|
16
|
-
sh "git tag v#{Resque::Plugins::BatchedJob::VERSION}"
|
14
|
+
desc "Publish RubyGem and source."
|
15
|
+
task :publish => [:build, :tag] do
|
17
16
|
sh "git push origin v#{Resque::Plugins::BatchedJob::VERSION}"
|
18
17
|
sh "git push origin master"
|
18
|
+
sh "gem push resque-batched-job-#{Resque::Plugins::BatchedJob::VERSION}.gem"
|
19
|
+
end
|
20
|
+
|
21
|
+
desc "Tag project with current version."
|
22
|
+
task :tag do
|
23
|
+
sh "git tag v#{Resque::Plugins::BatchedJob::VERSION}"
|
19
24
|
end
|
20
25
|
|
21
|
-
desc "Build
|
26
|
+
desc "Build RubyGem."
|
22
27
|
task :build do
|
23
28
|
sh "gem build resque-batched-job.gemspec"
|
24
29
|
end
|
30
|
+
|
31
|
+
desc "View changelog"
|
32
|
+
task :changelog do
|
33
|
+
tags = `git tag`.split("\n").reverse
|
34
|
+
|
35
|
+
tags.each_slice(2) do |tags|
|
36
|
+
puts "========== #{tags[1]}..#{tags[0]} =========="
|
37
|
+
sh "git log --pretty=format:'%h : %s' --graph #{tags[1]}..#{tags[0]}"
|
38
|
+
end
|
39
|
+
end
|
@@ -22,14 +22,17 @@ module Resque
|
|
22
22
|
|
23
23
|
# Batch the job. The first argument of a batched job, is the batch id.
|
24
24
|
def after_enqueue_batch(id, *args)
|
25
|
-
|
25
|
+
mutex(id) do |bid|
|
26
|
+
redis.rpush(bid, encode(:class => self, :args => args))
|
27
|
+
end
|
26
28
|
end
|
27
29
|
|
28
30
|
# After the job is performed, remove it from the batched job list. If the
|
29
31
|
# current job is the last in the batch to be performed, invoke the after_batch
|
30
32
|
# hooks.
|
31
33
|
def after_perform_batch(id, *args)
|
32
|
-
|
34
|
+
remove_batched_job(id, *args)
|
35
|
+
|
33
36
|
if batch_complete?(id)
|
34
37
|
after_batch_hooks = Resque::Plugin.after_batch_hooks(self)
|
35
38
|
after_batch_hooks.each do |hook|
|
@@ -41,13 +44,67 @@ module Resque
|
|
41
44
|
# Checks the size of the batched job list and returns true if the list is
|
42
45
|
# empty or if the key does not exist.
|
43
46
|
def batch_complete?(id)
|
44
|
-
|
47
|
+
mutex(id) do |bid|
|
48
|
+
redis.llen(bid) == 0
|
49
|
+
end
|
45
50
|
end
|
46
51
|
|
47
52
|
def batch_exist?(id)
|
48
|
-
|
53
|
+
mutex(id) do |bid|
|
54
|
+
redis.exists(bid)
|
55
|
+
end
|
49
56
|
end
|
50
57
|
|
58
|
+
# Remove a job from the batch list.
|
59
|
+
def remove_batched_job(id, *args)
|
60
|
+
mutex(id) do |bid|
|
61
|
+
redis.lrem(bid, 1, encode(:class => self, :args => args))
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
# Lock a batch key before executing Redis commands. This will ensure
|
68
|
+
# no race conditions occur when modifying batch information. Here is
|
69
|
+
# an example of how this works. See http://redis.io/commands/setnx for
|
70
|
+
# more information.
|
71
|
+
#
|
72
|
+
# * Job2 sends SETNX batch:123:lock in order to aquire a lock.
|
73
|
+
# * Job1 still has the key locked, so Job2 continues into the loop.
|
74
|
+
# * Job2 sends GET to aquire the lock timestamp.
|
75
|
+
# * If the timestamp does not exist (Job1 released the lock), Job2
|
76
|
+
# attemps to start from the beginning again.
|
77
|
+
# * If the timestamp exists and has not expired, Job2 sleeps for a
|
78
|
+
# moment and then retries from the start.
|
79
|
+
# * If the timestamp exists and has expired, Job2 sends GETSET to aquire
|
80
|
+
# a lock. This returns the previous value of the lock.
|
81
|
+
# * If the previous timestamp has not expired, another process was faster
|
82
|
+
# and aquired the lock. This means Job2 has to start from the beginnig.
|
83
|
+
# * If the previous timestamp is still expired the lock has been set and
|
84
|
+
# processing can continue safely
|
85
|
+
def mutex(id, &block)
|
86
|
+
is_expired = lambda do |locked_at|
|
87
|
+
locked_at.to_f < Time.now.to_f
|
88
|
+
end
|
89
|
+
bid = batch(id)
|
90
|
+
_key_ = "#{bid}:lock"
|
91
|
+
|
92
|
+
until redis.setnx(_key_, Time.now.to_f + 0.5)
|
93
|
+
next unless timestamp = redis.get(_key_)
|
94
|
+
|
95
|
+
unless is_expired.call(timestamp)
|
96
|
+
sleep(0.1)
|
97
|
+
next
|
98
|
+
end
|
99
|
+
|
100
|
+
break unless timestamp = redis.getset(_key_, Time.now.to_f + 0.5)
|
101
|
+
break if is_expired.call(timestamp)
|
102
|
+
end
|
103
|
+
yield(bid)
|
104
|
+
ensure
|
105
|
+
redis.del(_key_)
|
106
|
+
end
|
107
|
+
|
51
108
|
end
|
52
109
|
|
53
110
|
end
|
data/test/test_batched_job.rb
CHANGED
@@ -1,57 +1,123 @@
|
|
1
|
-
require '
|
2
|
-
require 'test/unit'
|
3
|
-
require 'resque'
|
4
|
-
require 'resque-batched-job'
|
5
|
-
|
6
|
-
class Job
|
7
|
-
extend Resque::Plugins::BatchedJob
|
8
|
-
@queue = :test
|
9
|
-
|
10
|
-
def self.perform(batch_id, arg)
|
11
|
-
end
|
12
|
-
|
13
|
-
def self.after_batch_hook(batch_id, arg)
|
14
|
-
$batch_complete = true
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
1
|
+
require File.dirname(__FILE__) + '/test_helper'
|
18
2
|
|
19
3
|
class BatchedJobTest < Test::Unit::TestCase
|
20
|
-
|
4
|
+
|
21
5
|
def setup
|
22
6
|
$batch_complete = false
|
23
7
|
@batch_id = :foo
|
24
8
|
@batch = "batch:#{@batch_id}"
|
25
|
-
@cnt = 5
|
26
|
-
@cnt.times { Resque.enqueue(Job, @batch_id, "arg#{rand(100)}") }
|
27
9
|
end
|
28
|
-
|
10
|
+
|
29
11
|
def teardown
|
30
12
|
redis.del(@batch)
|
31
13
|
redis.del("queue:test")
|
14
|
+
redis.del("#{@batch}:lock")
|
32
15
|
end
|
33
|
-
|
16
|
+
|
34
17
|
def test_list
|
35
18
|
assert_nothing_raised do
|
36
19
|
Resque::Plugin.lint(Resque::Plugins::BatchedJob)
|
37
20
|
end
|
38
21
|
end
|
39
|
-
|
22
|
+
|
40
23
|
def test_batch_key
|
24
|
+
assert_nothing_raised do
|
25
|
+
Resque.enqueue(Job, @batch_id, 'foobar')
|
26
|
+
end
|
41
27
|
assert_equal(@batch, Job.batch(@batch_id))
|
42
28
|
end
|
43
|
-
|
29
|
+
|
30
|
+
# Ensure the length of the Redis list matches the number of jobs we enqueue.
|
44
31
|
def test_batch_size
|
45
|
-
|
46
|
-
|
32
|
+
assert_nothing_raised do
|
33
|
+
5.times { Resque.enqueue(Job, @batch_id, "arg#{rand(100)}") }
|
34
|
+
end
|
35
|
+
assert_equal(5, redis.llen(@batch))
|
47
36
|
end
|
48
|
-
|
37
|
+
|
38
|
+
# Make sure the after_batch hook is fired
|
49
39
|
def test_batch_hook
|
40
|
+
assert_equal(false, Job.batch_exist?(@batch_id))
|
41
|
+
|
42
|
+
assert_nothing_raised do
|
43
|
+
5.times { Resque.enqueue(Job, @batch_id, "arg#{rand(100)}") }
|
44
|
+
end
|
45
|
+
|
46
|
+
assert_equal(false, $batch_complete)
|
47
|
+
assert_equal(false, Job.batch_complete?(@batch_id))
|
48
|
+
assert(Job.batch_exist?(@batch_id))
|
49
|
+
|
50
|
+
assert_nothing_raised do
|
51
|
+
4.times { Resque.reserve(:test).perform }
|
52
|
+
end
|
53
|
+
|
54
|
+
assert_equal(false, $batch_complete)
|
55
|
+
assert_equal(false, Job.batch_complete?(@batch_id))
|
56
|
+
assert(Job.batch_exist?(@batch_id))
|
57
|
+
|
58
|
+
assert_nothing_raised do
|
59
|
+
Resque.reserve(:test).perform
|
60
|
+
end
|
61
|
+
|
62
|
+
assert($batch_complete)
|
63
|
+
assert(Job.batch_complete?(@batch_id))
|
64
|
+
assert_equal(false, Job.batch_exist?(@batch_id))
|
65
|
+
end
|
66
|
+
|
67
|
+
# Test that jobs with identical args behave properly.
|
68
|
+
def test_duplicate_args
|
69
|
+
assert_equal(false, Job.batch_exist?(@batch_id))
|
70
|
+
|
71
|
+
assert_nothing_raised do
|
72
|
+
5.times { Resque.enqueue(JobWithoutArgs, @batch_id) }
|
73
|
+
end
|
74
|
+
|
75
|
+
assert_equal(false, $batch_complete)
|
76
|
+
assert_equal(false, Job.batch_complete?(@batch_id))
|
77
|
+
assert(Job.batch_exist?(@batch_id))
|
78
|
+
|
79
|
+
assert_nothing_raised do
|
80
|
+
2.times { Resque.reserve(:test).perform }
|
81
|
+
end
|
82
|
+
|
83
|
+
assert_equal(false, $batch_complete)
|
84
|
+
assert_equal(false, Job.batch_complete?(@batch_id))
|
85
|
+
assert(Job.batch_exist?(@batch_id))
|
86
|
+
|
50
87
|
assert_nothing_raised do
|
51
|
-
|
88
|
+
3.times { Resque.reserve(:test).perform }
|
52
89
|
end
|
90
|
+
|
53
91
|
assert($batch_complete)
|
54
|
-
|
92
|
+
assert(Job.batch_complete?(@batch_id))
|
93
|
+
assert_equal(false, Job.batch_exist?(@batch_id))
|
94
|
+
end
|
95
|
+
|
96
|
+
# Make sure the block is executed and the lock is removed.
|
97
|
+
def test_mutex
|
98
|
+
Job.send(:mutex, @batch_id) do
|
99
|
+
assert(true)
|
100
|
+
end
|
101
|
+
assert_equal(false, redis.exists("#{@batch}:lock"))
|
102
|
+
end
|
103
|
+
|
104
|
+
# Make sure no race conditions occur.
|
105
|
+
def test_locking
|
106
|
+
threads = []
|
107
|
+
x, y = 10, 5
|
108
|
+
|
109
|
+
x.times do
|
110
|
+
threads << Thread.new do
|
111
|
+
y.times do
|
112
|
+
Job.send(:mutex, @batch_id) do
|
113
|
+
redis.incr(@batch)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
threads.each { |t| t.join }
|
119
|
+
|
120
|
+
assert_equal(x * y, Integer(redis.get(@batch)))
|
55
121
|
end
|
56
122
|
|
57
123
|
private
|
@@ -59,5 +125,5 @@ class BatchedJobTest < Test::Unit::TestCase
|
|
59
125
|
def redis
|
60
126
|
Resque.redis
|
61
127
|
end
|
62
|
-
|
128
|
+
|
63
129
|
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'test/unit'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
require 'resque'
|
6
|
+
require File.dirname(__FILE__) + '/../lib/resque-batched-job'
|
7
|
+
|
8
|
+
class Job
|
9
|
+
extend Resque::Plugins::BatchedJob
|
10
|
+
@queue = :test
|
11
|
+
|
12
|
+
def self.perform(batch_id, arg)
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.after_batch_hook(batch_id, arg)
|
16
|
+
$batch_complete = true
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
|
21
|
+
class JobWithoutArgs
|
22
|
+
extend Resque::Plugins::BatchedJob
|
23
|
+
@queue = :test
|
24
|
+
|
25
|
+
def self.perform(id)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.after_batch_hook(id)
|
29
|
+
$batch_complete = true
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-batched-job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-12-
|
12
|
+
date: 2011-12-11 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: resque
|
16
|
-
requirement: &
|
16
|
+
requirement: &70295155687280 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,7 +21,7 @@ dependencies:
|
|
21
21
|
version: 1.10.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70295155687280
|
25
25
|
description: ! " Resque plugin for batching jobs. When a batch/group of jobs are
|
26
26
|
complete, \nadditional work can be performed usings batch hooks.\n"
|
27
27
|
email: dan@dj-agiledev.com
|
@@ -36,6 +36,7 @@ files:
|
|
36
36
|
- lib/resque/plugins/batched_job.rb
|
37
37
|
- lib/resque-batched-job.rb
|
38
38
|
- test/test_batched_job.rb
|
39
|
+
- test/test_helper.rb
|
39
40
|
homepage: https://github.com/drfeelngood/resque-batched-job
|
40
41
|
licenses: []
|
41
42
|
post_install_message:
|
@@ -62,3 +63,4 @@ specification_version: 3
|
|
62
63
|
summary: Resque plugin
|
63
64
|
test_files:
|
64
65
|
- test/test_batched_job.rb
|
66
|
+
- test/test_helper.rb
|