resque-serial-queues 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -4
- data/lib/resque-serial-queues/version.rb +1 -1
- data/lib/resque/plugins/serial_queues/resque_extension.rb +11 -2
- data/test/cases/lint_test.rb +14 -0
- data/test/cases/locking_test.rb +4 -4
- data/test/minitest_helper.rb +4 -4
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cd531373558ed3d2a84f9e421a3233c9ab905d1f
|
4
|
+
data.tar.gz: 9de3eb0f08eb657bd08c24aea3631e63da08373c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8fff5b737059347db4d065cfcdc542199a7bde5cd41b3debecadaa1b115de2ce2f1d7080dc3558a87730fde750ff94012b39d5d41e5d9d51740988045e0955be
|
7
|
+
data.tar.gz: cfc9029142c28796b5b500a533eaaf06eb39ee659517724daddd4edbee16d47cdfd024b2b36c502f5d7e5911b762bd0ee04057b6702bcf2781c9cac1d63eb693
|
data/README.md
CHANGED
@@ -1,6 +1,13 @@
|
|
1
1
|
# Resque::Plugins::SerialQueues
|
2
2
|
|
3
|
-
|
3
|
+
This gem is tested on the latest version from Resque 1.x.
|
4
|
+
|
5
|
+
Declaring resque queues as serial. Jobs from a serial queue won't be processed more than one at a time event if you have multiple workers / servers. There are similar solutions but none of them worked for me. To lock jobs from running in parallel I used `Redis#setnx`. This implementation has a few sensitive things:
|
6
|
+
- `Resque::Job.reserve` is overriden (here the queue locking is done)
|
7
|
+
- Added `Object#after_perform_unlock_queue_if_serial` (here the queue is unlocked)
|
8
|
+
- Added `Object#on_failure_unlock_queue_if_serial` (here the queue is unlocked)
|
9
|
+
|
10
|
+
I think serial queues are not a scalable. You shouldn't have to have serial queues. This should be a temporary drop-in solution without changing the deployment process. If you really need serial job processing you should start a worker for each serial queue that will listen for jobs only on that queue or migrate to a background processing / messaging tool that provides serial processing out of the box.
|
4
11
|
|
5
12
|
## Installation
|
6
13
|
|
@@ -16,15 +23,20 @@ Or install it yourself as:
|
|
16
23
|
|
17
24
|
$ gem install resque-serial-queues
|
18
25
|
|
19
|
-
## Usage
|
26
|
+
## Usage (Rails)
|
20
27
|
|
21
|
-
|
28
|
+
Create a `.rb` file in config/initializers
|
22
29
|
|
30
|
+
```ruby
|
23
31
|
Resque::Plugins::SerialQueues.configure do |config|
|
24
32
|
config.serial_queues = [:serial_jobs]
|
25
33
|
config.lock_timeout = 120 #default 3600 seconds
|
26
34
|
end
|
35
|
+
```
|
36
|
+
|
37
|
+
and use the declared serial queues in your jobs:
|
27
38
|
|
39
|
+
```ruby
|
28
40
|
class MyJob
|
29
41
|
|
30
42
|
@queue = :serial_jobs
|
@@ -33,7 +45,6 @@ class MyJob
|
|
33
45
|
end
|
34
46
|
|
35
47
|
end
|
36
|
-
|
37
48
|
```
|
38
49
|
|
39
50
|
## Contributing
|
@@ -3,8 +3,9 @@ require 'resque/job'
|
|
3
3
|
|
4
4
|
class Resque::Job
|
5
5
|
def self.reserve(queue)
|
6
|
-
return if is_serial_queue?(queue) and
|
6
|
+
return if is_serial_queue?(queue) and is_queue_locked?(queue)
|
7
7
|
return unless payload = Resque.pop(queue)
|
8
|
+
lock_queue(queue)
|
8
9
|
new(queue, payload)
|
9
10
|
end
|
10
11
|
|
@@ -13,13 +14,21 @@ class Resque::Job
|
|
13
14
|
Resque::Plugins::SerialQueues.is_queue_serial?(queue)
|
14
15
|
end
|
15
16
|
|
17
|
+
def self.is_queue_locked?(queue)
|
18
|
+
Resque::Plugins::SerialQueues.is_queue_locked?(queue)
|
19
|
+
end
|
20
|
+
|
16
21
|
def self.lock_queue(queue)
|
17
22
|
Resque::Plugins::SerialQueues.lock_queue(queue)
|
18
23
|
end
|
19
24
|
end
|
20
25
|
|
21
26
|
class Object
|
22
|
-
def self.
|
27
|
+
def self.after_perform_unlock_queue_if_serial(*)
|
28
|
+
Resque::Plugins::SerialQueues.unlock_queue(@queue) if Resque::Plugins::SerialQueues.is_queue_serial?(@queue)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.on_failure_unlock_queue_if_serial(*)
|
23
32
|
Resque::Plugins::SerialQueues.unlock_queue(@queue) if Resque::Plugins::SerialQueues.is_queue_serial?(@queue)
|
24
33
|
end
|
25
34
|
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'minitest_helper'
|
2
|
+
require 'resque/plugin'
|
3
|
+
class LintTest < Minitest::Test
|
4
|
+
|
5
|
+
def test_resque_plugin_lint
|
6
|
+
begin
|
7
|
+
Resque::Plugin.lint(Resque::Plugins::SerialQueues)
|
8
|
+
rescue Exception => e
|
9
|
+
assert false, "Resque::Plugin.lint raised #{e.inspect}"
|
10
|
+
end
|
11
|
+
assert true
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
data/test/cases/locking_test.rb
CHANGED
@@ -14,7 +14,7 @@ class LockingTest < ResqueSerialQueuesTest
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_locking_a_queue
|
17
|
-
assert Resque::Plugins::SerialQueues.lock_queue(:serial_jobs)
|
17
|
+
assert Resque::Plugins::SerialQueues.lock_queue(:serial_jobs), "Should be able to lock a queue but couldn't"
|
18
18
|
assert_queue_locked :serial_jobs
|
19
19
|
end
|
20
20
|
|
@@ -51,9 +51,9 @@ class LockingTest < ResqueSerialQueuesTest
|
|
51
51
|
end
|
52
52
|
|
53
53
|
def test_running_many_concurrent_jobs
|
54
|
-
jobs =
|
54
|
+
jobs = 10
|
55
55
|
max_sleep = 0.05
|
56
|
-
workers =
|
56
|
+
workers = 2
|
57
57
|
self.class.start_redis_workers workers
|
58
58
|
jobs.times do
|
59
59
|
Resque.enqueue BenchmarkJob, max_sleep
|
@@ -61,7 +61,7 @@ class LockingTest < ResqueSerialQueuesTest
|
|
61
61
|
assert_queue_locked :serial_jobs
|
62
62
|
while Resque.info[:pending]>0
|
63
63
|
sleep 0.1
|
64
|
-
assert Resque.info[:working]<=1
|
64
|
+
assert Resque.info[:working]<=1, "Should not have more than one working worker"
|
65
65
|
end
|
66
66
|
assert_queue_unlocked :serial_jobs
|
67
67
|
results = []
|
data/test/minitest_helper.rb
CHANGED
@@ -28,19 +28,19 @@ class ResqueSerialQueuesTest < Minitest::Test
|
|
28
28
|
end
|
29
29
|
|
30
30
|
def assert_queue_locked(queue)
|
31
|
-
Resque::Plugins::SerialQueues.is_queue_locked?(queue)
|
31
|
+
assert Resque::Plugins::SerialQueues.is_queue_locked?(queue), "Queue #{queue} should be locked but it's unlocked"
|
32
32
|
end
|
33
33
|
|
34
34
|
def assert_queue_unlocked(queue)
|
35
|
-
!Resque::Plugins::SerialQueues.is_queue_locked?(queue)
|
35
|
+
assert !Resque::Plugins::SerialQueues.is_queue_locked?(queue), "Queue #{queue} should be unlocked but it's locked"
|
36
36
|
end
|
37
37
|
|
38
38
|
def assert_serial_queue(queue)
|
39
|
-
Resque::Plugins::SerialQueues.is_queue_serial?(queue)
|
39
|
+
assert Resque::Plugins::SerialQueues.is_queue_serial?(queue), "Queue #{queue} should be serial but it's concurrent"
|
40
40
|
end
|
41
41
|
|
42
42
|
def assert_concurrent_queue(queue)
|
43
|
-
!
|
43
|
+
assert !Resque::Plugins::SerialQueues.is_queue_serial?(queue), "Queue #{queue} should be concurrent but it's serial"
|
44
44
|
end
|
45
45
|
|
46
46
|
def log(message)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-serial-queues
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Cristian Bica
|
@@ -114,6 +114,7 @@ files:
|
|
114
114
|
- lib/resque/plugins/serial_queues/config.rb
|
115
115
|
- lib/resque/plugins/serial_queues/resque_extension.rb
|
116
116
|
- resque-serial-queues.gemspec
|
117
|
+
- test/cases/lint_test.rb
|
117
118
|
- test/cases/locking_test.rb
|
118
119
|
- test/jobs/benchmark_job.rb
|
119
120
|
- test/jobs/delayed_failing_job.rb
|
@@ -144,6 +145,7 @@ signing_key:
|
|
144
145
|
specification_version: 4
|
145
146
|
summary: Declare resque queues serial
|
146
147
|
test_files:
|
148
|
+
- test/cases/lint_test.rb
|
147
149
|
- test/cases/locking_test.rb
|
148
150
|
- test/jobs/benchmark_job.rb
|
149
151
|
- test/jobs/delayed_failing_job.rb
|