worker_roulette 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +3 -2
- data/lib/worker_roulette/a_foreman.rb +22 -0
- data/lib/worker_roulette/a_tradesman.rb +52 -0
- data/lib/worker_roulette/foreman.rb +3 -3
- data/lib/worker_roulette/tradesman.rb +4 -4
- data/lib/worker_roulette/version.rb +1 -1
- data/lib/worker_roulette.rb +32 -17
- data/spec/benchmark/perf_test.rb +2 -2
- data/spec/integration/evented_worker_roulette_spec.rb +222 -0
- data/spec/integration/worker_roulette_spec.rb +4 -12
- data/spec/spec_helper.rb +3 -25
- data/worker_roulette.gemspec +3 -1
- metadata +43 -7
data/README.md
CHANGED
@@ -57,18 +57,19 @@ bad_foreman = WorkerRoulette.foreman('foreman', 'bad_channel')
|
|
57
57
|
|
58
58
|
publish = -> do
|
59
59
|
good_foreman.enqueue_work_order('some old fashion work')
|
60
|
-
bad_foreman.enqueue_work_order('evil biddings you should not carry out')
|
60
|
+
bad_foreman.enqueue_work_order('evil biddings you should not carry out')
|
61
61
|
end
|
62
62
|
|
63
63
|
tradesman.wait_for_work_orders(publish) do |work|
|
64
64
|
work.to_s.should match("some old fashion work") #only got the work from the good foreman
|
65
|
+
work.to_s.should_not match("evil") #channels let us ignore the other's evil orders
|
65
66
|
tradesman.unsubscribe
|
66
67
|
end
|
67
68
|
|
68
69
|
```
|
69
70
|
|
70
71
|
##Caveat Emptor
|
71
|
-
While WorkerRoulette does promise to keep the messages of each consumer processed in order by competing consumers, it does NOT guarantee the order in which the queues themselves will be processed. In general, work is processed in a FIFO order, but for performance reasons this has been left a loose FIFO. For example, if Abdul
|
72
|
+
While WorkerRoulette does promise to keep the messages of each consumer processed in order by competing consumers, it does NOT guarantee the order in which the queues themselves will be processed. In general, work is processed in a FIFO order, but for performance reasons this has been left a loose FIFO. For example, if Abdul enqueues some ordered messages ('1', '2', and '3') and then so do Mark and Wanda, Mark's messages may be processed first, then it would likely be Abdul's, and then Wanda's. However, even though Mark jumped the line, Abdul's messages will still be processed the order he enqueued them ('1', '2', then '3').
|
72
73
|
|
73
74
|
## Installation
|
74
75
|
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative './foreman'
|
2
|
+
module WorkerRoulette
|
3
|
+
class AForeman < Foreman
|
4
|
+
def enqueue_work_order_without_headers(work_order, &callback)
|
5
|
+
#Caveat Emptor: There is a race condition here, but it not serious;
|
6
|
+
#the count may be incremented again by another process before the sender
|
7
|
+
#is added to the job_queue. This is not a big deal bc it just means that
|
8
|
+
#the sender's queue will be processed one slot behind it's rightful place.
|
9
|
+
#This does not effect work_order ordering.
|
10
|
+
@redis_pool.with do |redis|
|
11
|
+
redis.incr(COUNTER_KEY) do |count|
|
12
|
+
@count = count ||= 1
|
13
|
+
redis.multi
|
14
|
+
redis.zadd(job_board_key, @count, @sender)
|
15
|
+
redis.rpush(sender_key, Oj.dump(work_order))
|
16
|
+
redis.publish(@channel, WorkerRoulette::JOB_NOTIFICATIONS)
|
17
|
+
redis.exec.callback &callback
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require_relative './tradesman'
|
2
|
+
module WorkerRoulette
|
3
|
+
class ATradesman < Tradesman
|
4
|
+
def wait_for_work_orders(on_subscribe_callback = nil, &on_message_callback)
|
5
|
+
@redis_pubsub ||= WorkerRoulette.new_redis_pubsub #cannot use connection pool bc redis expects each obj to own its own pubsub connection for the life of the subscription
|
6
|
+
@redis_pubsub.on(:subscribe) {|channel, subscription_count| on_subscribe_callback.call(channel, subscription_count) if on_subscribe_callback}
|
7
|
+
@redis_pubsub.on(:message) {|channel, message| work_orders! {|work_orders| on_message_callback.call(work_orders, message, channel)} if on_message_callback}
|
8
|
+
@redis_pubsub.subscribe(@channel)
|
9
|
+
end
|
10
|
+
|
11
|
+
def work_orders!(&callback)
|
12
|
+
@client_pool.with do |redis|
|
13
|
+
get_sender_for_next_job(redis) do |sender_results|
|
14
|
+
@sender = (sender_results || []).first.to_s
|
15
|
+
redis.multi
|
16
|
+
redis.lrange(sender_key, 0, -1)
|
17
|
+
redis.del(sender_key)
|
18
|
+
redis.zrem(job_board_key, sender_key)
|
19
|
+
redis.exec do |work_orders|
|
20
|
+
callback.call ((work_orders || []).first || []).map {|work_order| Oj.load(work_order)} if callback
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def get_lock(redis, sender, timeout, on_failure = nil, &on_success)
|
27
|
+
@lock = EM::Hiredis::Lock.new(redis, sender, timeout)
|
28
|
+
@lock.callback &on_success
|
29
|
+
@lock.errback &(on_failure || proc {})
|
30
|
+
@lock
|
31
|
+
end
|
32
|
+
|
33
|
+
def unsubscribe(&callback)
|
34
|
+
deferable = @redis_pubsub.unsubscribe(@channel)
|
35
|
+
deferable.callback do
|
36
|
+
@redis_pubsub.close_connection
|
37
|
+
@redis_pubsub = nil
|
38
|
+
callback.call
|
39
|
+
end
|
40
|
+
deferable.errback do
|
41
|
+
@redis_pubsub.close_connection
|
42
|
+
@redis_pubsub = nil
|
43
|
+
callback.call
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
def get_sender_for_next_job(redis, &callback)
|
49
|
+
redis.zrange(job_board_key, 0, 0).callback &callback
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -22,7 +22,7 @@ module WorkerRoulette
|
|
22
22
|
@counter_key ||= "#{@namespace + ':' if @namespace}#{COUNTER_KEY}"
|
23
23
|
end
|
24
24
|
|
25
|
-
def enqueue_work_order_without_headers(work_order)
|
25
|
+
def enqueue_work_order_without_headers(work_order, &callback)
|
26
26
|
#Caveat Emptor: There is a race condition here, but it not serious;
|
27
27
|
#the count may be incremented again by another process before the sender
|
28
28
|
#is added to the job_queue. This is not a big deal bc it just means that
|
@@ -38,9 +38,9 @@ module WorkerRoulette
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
-
def enqueue_work_order(work_order, headers = {})
|
41
|
+
def enqueue_work_order(work_order, headers = {}, &callback)
|
42
42
|
work_order = {'headers' => default_headers.merge(headers), 'payload' => work_order}
|
43
|
-
enqueue_work_order_without_headers(work_order)
|
43
|
+
enqueue_work_order_without_headers(work_order, &callback)
|
44
44
|
end
|
45
45
|
|
46
46
|
def default_headers
|
@@ -17,16 +17,16 @@ module WorkerRoulette
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def wait_for_work_orders(on_subscribe_callback = nil, &block)
|
20
|
-
@pubsub_pool.with
|
20
|
+
@pubsub_pool.with do |redis|
|
21
21
|
redis.subscribe(@channel) do |on|
|
22
22
|
on.subscribe {on_subscribe_callback.call if on_subscribe_callback}
|
23
|
-
on.message {block.call(work_orders!) if block}
|
23
|
+
on.message {self.unsubscribe; block.call(work_orders!) if block}
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
28
|
def work_orders!
|
29
|
-
@client_pool.with
|
29
|
+
@client_pool.with do |redis|
|
30
30
|
get_sender_for_next_job(redis)
|
31
31
|
results = redis.multi do
|
32
32
|
redis.lrange(sender_key, 0, -1)
|
@@ -38,7 +38,7 @@ module WorkerRoulette
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def unsubscribe
|
41
|
-
@pubsub_pool.with
|
41
|
+
@pubsub_pool.with {|redis| redis.unsubscribe(@channel)}
|
42
42
|
end
|
43
43
|
|
44
44
|
private
|
data/lib/worker_roulette.rb
CHANGED
@@ -2,24 +2,23 @@ require "worker_roulette/version"
|
|
2
2
|
require 'oj'
|
3
3
|
require 'redis'
|
4
4
|
require 'hiredis'
|
5
|
-
require 'em-
|
5
|
+
require 'em-hiredis'
|
6
|
+
require 'connection_pool'
|
6
7
|
|
7
8
|
Dir[File.join(File.dirname(__FILE__),'worker_roulette','**','*.rb')].sort.each { |file| require file.gsub(".rb", "")}
|
8
9
|
|
9
|
-
class EventMachine::Synchrony::ConnectionPool
|
10
|
-
alias_method :with, :execute
|
11
|
-
end
|
12
|
-
|
13
10
|
module WorkerRoulette
|
14
11
|
JOB_BOARD = "job_board"
|
15
12
|
JOB_NOTIFICATIONS = "new_job_ready"
|
16
13
|
|
17
14
|
def self.start(config = {})
|
18
|
-
@redis_config = {host: 'localhost', db: 14, driver: :hiredis, timeout: 5, pool_size: 10}.merge(config)
|
15
|
+
@redis_config = {host: 'localhost', port: 6379, db: 14, driver: :hiredis, timeout: 5, evented: false, pool_size: 10}.merge(config)
|
19
16
|
@pool_config = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
|
20
|
-
@
|
21
|
-
|
22
|
-
@
|
17
|
+
@evented = @redis_config.delete(:evented)
|
18
|
+
|
19
|
+
@foreman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
|
20
|
+
@tradesman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
|
21
|
+
@pubsub_connection_pool = ConnectionPool.new(@pool_config) {new_redis_pubsub}
|
23
22
|
end
|
24
23
|
|
25
24
|
def self.foreman(sender, channel = nil)
|
@@ -32,6 +31,16 @@ module WorkerRoulette
|
|
32
31
|
Tradesman.new(@tradesman_connection_pool, @pubsub_connection_pool, channel)
|
33
32
|
end
|
34
33
|
|
34
|
+
def self.a_foreman(sender, channel = nil)
|
35
|
+
raise "WorkerRoulette not Started" unless @foreman_connection_pool
|
36
|
+
AForeman.new(sender, @foreman_connection_pool, channel)
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.a_tradesman(channel = nil)
|
40
|
+
raise "WorkerRoulette not Started" unless @tradesman_connection_pool
|
41
|
+
ATradesman.new(@tradesman_connection_pool, @pubsub_connection_pool, channel)
|
42
|
+
end
|
43
|
+
|
35
44
|
def self.tradesman_connection_pool
|
36
45
|
@tradesman_connection_pool
|
37
46
|
end
|
@@ -49,15 +58,21 @@ module WorkerRoulette
|
|
49
58
|
end
|
50
59
|
|
51
60
|
private
|
52
|
-
def self.
|
53
|
-
if @
|
54
|
-
require '
|
55
|
-
|
56
|
-
|
61
|
+
def self.new_redis
|
62
|
+
if @evented
|
63
|
+
require 'eventmachine'
|
64
|
+
redis = EM::Hiredis::Client.new(@redis_config[:host], @redis_config[:port], @redis_config[:password], @redis_config[:db])
|
65
|
+
redis.connect
|
66
|
+
else
|
67
|
+
Redis.new(@redis_config)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.new_redis_pubsub
|
72
|
+
if @evented
|
73
|
+
new_redis.pubsub
|
57
74
|
else
|
58
|
-
|
59
|
-
require 'connection_pool'
|
60
|
-
ConnectionPool
|
75
|
+
new_redis
|
61
76
|
end
|
62
77
|
end
|
63
78
|
end
|
data/spec/benchmark/perf_test.rb
CHANGED
@@ -7,7 +7,7 @@ ITERATIONS = 10_000
|
|
7
7
|
|
8
8
|
work_order = {'ding dong' => "hello_foreman_" * 100}
|
9
9
|
|
10
|
-
WorkerRoulette.start(REDIS_CONNECTION_POOL_SIZE)#{driver: :synchrony}
|
10
|
+
WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE)#{driver: :synchrony}
|
11
11
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
12
12
|
|
13
13
|
puts "Redis Connection Pool Size: #{REDIS_CONNECTION_POOL_SIZE}"
|
@@ -39,7 +39,7 @@ Benchmark.bmbm do |x|
|
|
39
39
|
foreman.enqueue_work_order(work_order)
|
40
40
|
end
|
41
41
|
tradesman = WorkerRoulette.tradesman
|
42
|
-
tradesman.wait_for_work_orders(p) {|m| m}
|
42
|
+
tradesman.wait_for_work_orders(p) {|m| m; tradesman.unsubscribe}
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
@@ -0,0 +1,222 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe WorkerRoulette do
|
4
|
+
include EventedSpec::EMSpec
|
5
|
+
|
6
|
+
let(:sender) {'katie_80'}
|
7
|
+
let(:work_orders) {["hello", "foreman"]}
|
8
|
+
let(:default_headers) {Hash['headers' => {'sender' => sender}]}
|
9
|
+
let(:hello_work_order) {Hash['payload' => "hello"]}
|
10
|
+
let(:foreman_work_order) {Hash['payload' => "foreman"]}
|
11
|
+
let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
|
12
|
+
let(:jsonized_work_orders_with_headers) {[Oj.dump(work_orders_with_headers)]}
|
13
|
+
|
14
|
+
let(:redis) {Redis.new(WorkerRoulette.redis_config)}
|
15
|
+
|
16
|
+
em_before do
|
17
|
+
WorkerRoulette.start(evented: true)
|
18
|
+
end
|
19
|
+
|
20
|
+
context Foreman do
|
21
|
+
let(:subject) {WorkerRoulette.a_foreman(sender)}
|
22
|
+
|
23
|
+
it "should enqueue work" do
|
24
|
+
called = false
|
25
|
+
foreman = WorkerRoulette.a_foreman('foreman')
|
26
|
+
foreman.enqueue_work_order('some old fashion work') do |redis_response|
|
27
|
+
called = true
|
28
|
+
redis_response.should == [1, 1, 0]
|
29
|
+
end
|
30
|
+
done(0.1) {called.should == true}
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should enqueue_work_order two work_orders in the sender's slot in the job board" do
|
34
|
+
subject.enqueue_work_order(work_orders.first) do
|
35
|
+
subject.enqueue_work_order(work_orders.last) do
|
36
|
+
redis.lrange(sender, 0, -1).should == work_orders.map {|m| Oj.dump(default_headers.merge({'payload' => m})) }
|
37
|
+
done
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should enqueue_work_order an array of work_orders without headers in the sender's slot in the job board" do
|
43
|
+
subject.enqueue_work_order_without_headers(work_orders) do
|
44
|
+
redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders)]
|
45
|
+
done
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should enqueue_work_order an array of work_orders with default headers in the sender's slot in the job board" do
|
50
|
+
subject.enqueue_work_order(work_orders) do
|
51
|
+
redis.lrange(sender, 0, -1).should == jsonized_work_orders_with_headers
|
52
|
+
done
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should enqueue_work_order an array of work_orders with additional headers in the sender's slot in the job board" do
|
57
|
+
extra_headers = {'foo' => 'bars'}
|
58
|
+
subject.enqueue_work_order(work_orders, extra_headers) do
|
59
|
+
work_orders_with_headers['headers'].merge!(extra_headers)
|
60
|
+
redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders_with_headers)]
|
61
|
+
done
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should post the sender's id to the job board with an order number" do
|
66
|
+
subject.enqueue_work_order(work_orders.first) do
|
67
|
+
subject.enqueue_work_order(work_orders.last) do
|
68
|
+
redis.zrange(subject.job_board_key, 0, -1, with_scores: true).should == [[sender.to_s, work_orders.length.to_f]]
|
69
|
+
done
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should post the sender_id and work_orders transactionally" do
|
75
|
+
EM::Hiredis::Client.any_instance.should_receive(:multi).and_call_original
|
76
|
+
subject.enqueue_work_order(work_orders.first) do
|
77
|
+
done
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should generate sequential order numbers" do
|
82
|
+
redis.get(subject.counter_key).should == nil
|
83
|
+
subject.enqueue_work_order(work_orders.first) do
|
84
|
+
redis.get(subject.counter_key).should == "1"
|
85
|
+
subject.enqueue_work_order(work_orders.last) do
|
86
|
+
redis.get(subject.counter_key).should == "2"
|
87
|
+
done
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
it "should publish a notification that a new job is ready" do
|
93
|
+
result = nil
|
94
|
+
subscriber = WorkerRoulette.new_redis_pubsub
|
95
|
+
subscriber.subscribe(WorkerRoulette::JOB_NOTIFICATIONS) do |message|
|
96
|
+
subscriber.unsubscribe(WorkerRoulette::JOB_NOTIFICATIONS)
|
97
|
+
message.should == WorkerRoulette::JOB_NOTIFICATIONS
|
98
|
+
done
|
99
|
+
end.callback { subject.enqueue_work_order(work_orders) }
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
context Tradesman do
|
104
|
+
let(:foreman) {WorkerRoulette.a_foreman(sender)}
|
105
|
+
let(:subject) {WorkerRoulette.a_tradesman}
|
106
|
+
|
107
|
+
it "should be working on behalf of a sender" do
|
108
|
+
foreman.enqueue_work_order(work_orders) do
|
109
|
+
subject.work_orders! do
|
110
|
+
subject.sender.should == sender
|
111
|
+
done
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
|
117
|
+
it "should drain one set of work_orders from the sender's slot in the job board" do
|
118
|
+
foreman.enqueue_work_order(work_orders) do
|
119
|
+
subject.work_orders! do |r|
|
120
|
+
r.should == [work_orders_with_headers]
|
121
|
+
subject.work_orders! do |r| r.should == []
|
122
|
+
subject.work_orders! {|r| r.should == []; done} #does not throw an error if queue is alreay empty
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
it "should take the oldest sender off the job board (FIFO)" do
|
129
|
+
foreman.enqueue_work_order(work_orders) do
|
130
|
+
oldest_sender = sender.to_s
|
131
|
+
most_recent_sender = 'most_recent_sender'
|
132
|
+
most_recent_foreman = WorkerRoulette.a_foreman(most_recent_sender)
|
133
|
+
most_recent_foreman.enqueue_work_order(work_orders) do
|
134
|
+
redis.zrange(subject.job_board_key, 0, -1).should == [oldest_sender, most_recent_sender]
|
135
|
+
subject.work_orders! { redis.zrange(subject.job_board_key, 0, -1).should == [most_recent_sender]; done }
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
it "should get the sender and work_order list transactionally" do
|
141
|
+
EM::Hiredis::Client.any_instance.should_receive(:multi).and_call_original
|
142
|
+
subject.work_orders! {done}
|
143
|
+
end
|
144
|
+
|
145
|
+
it "should get the work_orders from the next queue when a new job is ready" do
|
146
|
+
subject.should_receive(:work_orders!).and_call_original
|
147
|
+
|
148
|
+
subject.wait_for_work_orders do |redis_work_orders, message, channel|
|
149
|
+
subject.sender.should == "katie_80"
|
150
|
+
redis_work_orders.should == [work_orders_with_headers]
|
151
|
+
done
|
152
|
+
end
|
153
|
+
|
154
|
+
foreman.enqueue_work_order(work_orders)
|
155
|
+
end
|
156
|
+
|
157
|
+
it "should publish and subscribe on custom channels" do
|
158
|
+
good_subscribed = false
|
159
|
+
bad_subscribed = false
|
160
|
+
|
161
|
+
tradesman = WorkerRoulette.a_tradesman('good_channel')
|
162
|
+
evil_tradesman = WorkerRoulette.a_tradesman('bad_channel')
|
163
|
+
|
164
|
+
good_foreman = WorkerRoulette.a_foreman('foreman', 'good_channel')
|
165
|
+
bad_foreman = WorkerRoulette.a_foreman('foreman', 'bad_channel')
|
166
|
+
|
167
|
+
good_publish = ->(a,b) {good_subscribed = true}
|
168
|
+
bad_publish = ->(a,b) {bad_subscribed = true}
|
169
|
+
|
170
|
+
tradesman.should_receive(:work_orders!).and_call_original
|
171
|
+
evil_tradesman.should_receive(:work_orders!).and_call_original
|
172
|
+
|
173
|
+
#They are double subscribing; is it possible that it is the connection pool?
|
174
|
+
|
175
|
+
tradesman.wait_for_work_orders(good_publish) do |good_work|
|
176
|
+
good_work.to_s.should match("old fashion")
|
177
|
+
good_work.to_s.should_not match("evil")
|
178
|
+
good_subscribed.should == true
|
179
|
+
end
|
180
|
+
|
181
|
+
evil_tradesman.wait_for_work_orders(bad_publish) do |bad_work|
|
182
|
+
bad_work.to_s.should_not match("old fashion")
|
183
|
+
bad_work.to_s.should match("evil")
|
184
|
+
bad_subscribed.should == true
|
185
|
+
end
|
186
|
+
|
187
|
+
good_foreman.enqueue_work_order('some old fashion work')
|
188
|
+
bad_foreman.enqueue_work_order('evil biddings you should not carry out')
|
189
|
+
done(0.2)
|
190
|
+
end
|
191
|
+
|
192
|
+
it "should unsubscribe from the job board" do
|
193
|
+
subject.wait_for_work_orders do |redis_work_orders, message, channel|
|
194
|
+
subject.unsubscribe {done}
|
195
|
+
end
|
196
|
+
EM::Hiredis::PubsubClient.any_instance.should_receive(:close_connection).and_call_original
|
197
|
+
foreman.enqueue_work_order(work_orders)
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
context "Failure" do
|
202
|
+
it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; done; end
|
203
|
+
end
|
204
|
+
|
205
|
+
context "Concurrent Access" do
|
206
|
+
it "should not leak connections"
|
207
|
+
it "should checkout a readlock for a queue and put it back when its done processing; lock should expire after 5 minutes?"
|
208
|
+
it "should retry doing work on a queue 3 times if it is locked (ex backoff)"
|
209
|
+
it "should not delete the messages from the queue until they have been processed succcesfully"
|
210
|
+
it "should periodically (10 seconds?) poll the job board for new work"
|
211
|
+
|
212
|
+
it "should be fork() proof" do
|
213
|
+
@subject = WorkerRoulette.a_tradesman
|
214
|
+
@subject.work_orders! do
|
215
|
+
fork do
|
216
|
+
@subject.work_orders!
|
217
|
+
end
|
218
|
+
end
|
219
|
+
done(1)
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
@@ -85,7 +85,7 @@ describe WorkerRoulette do
|
|
85
85
|
result = notification
|
86
86
|
redis_tradesman.unsubscribe(WorkerRoulette::JOB_NOTIFICATIONS)
|
87
87
|
end
|
88
|
-
|
88
|
+
end
|
89
89
|
|
90
90
|
result.should == WorkerRoulette::JOB_NOTIFICATIONS
|
91
91
|
end
|
@@ -141,7 +141,7 @@ describe WorkerRoulette do
|
|
141
141
|
subject.work_orders!
|
142
142
|
subject.should_receive(:work_orders!).and_call_original
|
143
143
|
|
144
|
-
publisher = -> {
|
144
|
+
publisher = -> {foreman.enqueue_work_order(work_orders); subject.unsubscribe}
|
145
145
|
|
146
146
|
subject.wait_for_work_orders(publisher) do |redis_work_orders|
|
147
147
|
redis_work_orders.should == [work_orders_with_headers]
|
@@ -164,6 +164,7 @@ describe WorkerRoulette do
|
|
164
164
|
|
165
165
|
tradesman.wait_for_work_orders(publish) do |work|
|
166
166
|
work.to_s.should match("some old fashion work")
|
167
|
+
work.to_s.should_not match("evil")
|
167
168
|
end
|
168
169
|
end
|
169
170
|
|
@@ -192,15 +193,6 @@ describe WorkerRoulette do
|
|
192
193
|
expect {WorkerRoulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}}.to raise_error(Redis::InheritedError)
|
193
194
|
end
|
194
195
|
end
|
195
|
-
|
196
|
-
it "should use optionally non-blocking I/O" do
|
197
|
-
EM.synchrony do
|
198
|
-
WorkerRoulette.start(:driver => :synchrony)
|
199
|
-
WorkerRoulette.foreman("muddle_man").enqueue_work_order("foo")
|
200
|
-
WorkerRoulette.tradesman.work_orders!.should == [work_orders_with_headers]
|
201
|
-
EM.stop
|
202
|
-
end
|
203
|
-
end
|
204
196
|
end
|
205
197
|
end
|
206
|
-
end
|
198
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
require 'worker_roulette'
|
2
|
-
require '
|
2
|
+
require 'evented-spec'
|
3
3
|
require 'simplecov'
|
4
4
|
require 'simplecov-rcov'
|
5
|
+
require 'rspec'
|
5
6
|
class SimpleCov::Formatter::MergedFormatter
|
6
7
|
def format(result)
|
7
8
|
SimpleCov::Formatter::HTMLFormatter.new.format(result)
|
@@ -16,30 +17,7 @@ include WorkerRoulette
|
|
16
17
|
|
17
18
|
Dir[File.join(File.dirname(__FILE__), 'helpers', '**/*.rb')].sort.each { |file| require file.gsub(".rb", "")}
|
18
19
|
|
19
|
-
|
20
|
-
module Core
|
21
|
-
class ExampleGroup
|
22
|
-
|
23
|
-
class << self
|
24
|
-
alias_method :run_alias, :run
|
25
|
-
|
26
|
-
def run(reporter)
|
27
|
-
if EM.reactor_running?
|
28
|
-
run_alias reporter
|
29
|
-
else
|
30
|
-
out = nil
|
31
|
-
EM.synchrony do
|
32
|
-
out = run_alias reporter
|
33
|
-
EM.stop
|
34
|
-
end
|
35
|
-
out
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
20
|
+
EM::Hiredis.reconnect_timeout = 0.01
|
43
21
|
|
44
22
|
RSpec.configure do |c|
|
45
23
|
c.after(:each) do
|
data/worker_roulette.gemspec
CHANGED
@@ -20,8 +20,9 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.add_dependency 'oj'
|
21
21
|
spec.add_dependency 'redis', '~> 3.0.7'
|
22
22
|
spec.add_dependency 'hiredis', '~> 0.4.5'
|
23
|
-
spec.add_dependency 'em-
|
23
|
+
spec.add_dependency 'em-hiredis', '~> 0.2.1'
|
24
24
|
spec.add_dependency 'connection_pool'
|
25
|
+
spec.add_dependency 'eventmachine', '~> 1.0.3'
|
25
26
|
|
26
27
|
spec.add_development_dependency "bundler"
|
27
28
|
spec.add_development_dependency "rake"
|
@@ -30,4 +31,5 @@ Gem::Specification.new do |spec|
|
|
30
31
|
spec.add_development_dependency 'simplecov'
|
31
32
|
spec.add_development_dependency 'simplecov-rcov'
|
32
33
|
spec.add_development_dependency 'rspec_junit_formatter'
|
34
|
+
spec.add_development_dependency 'evented-spec'
|
33
35
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: worker_roulette
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-02-
|
12
|
+
date: 2014-02-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: oj
|
@@ -60,21 +60,21 @@ dependencies:
|
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: 0.4.5
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
|
-
name: em-
|
63
|
+
name: em-hiredis
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
65
65
|
none: false
|
66
66
|
requirements:
|
67
|
-
- -
|
67
|
+
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version:
|
69
|
+
version: 0.2.1
|
70
70
|
type: :runtime
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
|
-
- -
|
75
|
+
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version:
|
77
|
+
version: 0.2.1
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
79
|
name: connection_pool
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -91,6 +91,22 @@ dependencies:
|
|
91
91
|
- - ! '>='
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: eventmachine
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ~>
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 1.0.3
|
102
|
+
type: :runtime
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ~>
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 1.0.3
|
94
110
|
- !ruby/object:Gem::Dependency
|
95
111
|
name: bundler
|
96
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -203,6 +219,22 @@ dependencies:
|
|
203
219
|
- - ! '>='
|
204
220
|
- !ruby/object:Gem::Version
|
205
221
|
version: '0'
|
222
|
+
- !ruby/object:Gem::Dependency
|
223
|
+
name: evented-spec
|
224
|
+
requirement: !ruby/object:Gem::Requirement
|
225
|
+
none: false
|
226
|
+
requirements:
|
227
|
+
- - ! '>='
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: '0'
|
230
|
+
type: :development
|
231
|
+
prerelease: false
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
233
|
+
none: false
|
234
|
+
requirements:
|
235
|
+
- - ! '>='
|
236
|
+
- !ruby/object:Gem::Version
|
237
|
+
version: '0'
|
206
238
|
description: Write a gem description
|
207
239
|
email:
|
208
240
|
- classicist@gmail.com
|
@@ -218,11 +250,14 @@ files:
|
|
218
250
|
- README.md
|
219
251
|
- Rakefile
|
220
252
|
- lib/worker_roulette.rb
|
253
|
+
- lib/worker_roulette/a_foreman.rb
|
254
|
+
- lib/worker_roulette/a_tradesman.rb
|
221
255
|
- lib/worker_roulette/foreman.rb
|
222
256
|
- lib/worker_roulette/tradesman.rb
|
223
257
|
- lib/worker_roulette/version.rb
|
224
258
|
- spec/benchmark/perf_test.rb
|
225
259
|
- spec/helpers/.gitkeep
|
260
|
+
- spec/integration/evented_worker_roulette_spec.rb
|
226
261
|
- spec/integration/worker_roulette_spec.rb
|
227
262
|
- spec/spec_helper.rb
|
228
263
|
- worker_roulette.gemspec
|
@@ -253,6 +288,7 @@ summary: Write a gem summary
|
|
253
288
|
test_files:
|
254
289
|
- spec/benchmark/perf_test.rb
|
255
290
|
- spec/helpers/.gitkeep
|
291
|
+
- spec/integration/evented_worker_roulette_spec.rb
|
256
292
|
- spec/integration/worker_roulette_spec.rb
|
257
293
|
- spec/spec_helper.rb
|
258
294
|
has_rdoc:
|