worker_roulette 0.1.7 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +31 -88
- data/lib/worker_roulette/foreman.rb +13 -14
- data/lib/worker_roulette/lua.rb +19 -15
- data/lib/worker_roulette/tradesman.rb +100 -72
- data/lib/worker_roulette/version.rb +1 -1
- data/lib/worker_roulette.rb +66 -73
- data/spec/benchmark/irb_demo_runner.rb +39 -0
- data/spec/benchmark/perf_test.rb +22 -26
- data/spec/integration/evented_worker_roulette_spec.rb +165 -186
- data/spec/integration/worker_roulette_spec.rb +123 -145
- data/spec/spec_helper.rb +13 -12
- data/spec/unit/evented_readlock_spec.rb +22 -21
- data/spec/unit/lua_spec.rb +12 -14
- data/spec/unit/readlock_spec.rb +10 -11
- data/worker_roulette.gemspec +1 -0
- metadata +18 -5
- data/lib/worker_roulette/a_tradesman.rb +0 -46
- data/spec/unit/worker_roulette_spec.rb +0 -14
|
@@ -1,248 +1,227 @@
|
|
|
1
1
|
require "spec_helper"
|
|
2
|
+
module WorkerRoulette
|
|
3
|
+
describe WorkerRoulette do
|
|
4
|
+
include EventedSpec::EMSpec
|
|
5
|
+
|
|
6
|
+
let(:sender) {'katie_80'}
|
|
7
|
+
let(:work_orders) {["hello", "foreman"]}
|
|
8
|
+
let(:default_headers) {Hash['headers' => {'sender' => sender}]}
|
|
9
|
+
let(:hello_work_order) {Hash['payload' => "hello"]}
|
|
10
|
+
let(:foreman_work_order) {Hash['payload' => "foreman"]}
|
|
11
|
+
let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
|
|
12
|
+
let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
|
|
13
|
+
let(:worker_roulette) { WorkerRoulette.start(evented: true) }
|
|
14
|
+
let(:redis) {Redis.new(worker_roulette.redis_config)}
|
|
15
|
+
|
|
16
|
+
context "Evented Foreman" do
|
|
17
|
+
let(:subject) {worker_roulette.foreman(sender)}
|
|
18
|
+
|
|
19
|
+
it "enqueues work" do
|
|
20
|
+
called = false
|
|
21
|
+
foreman = worker_roulette.foreman('foreman')
|
|
22
|
+
foreman.enqueue_work_order('some old fashion work') do |redis_response, stuff|
|
|
23
|
+
called = true
|
|
24
|
+
end
|
|
25
|
+
done(0.1) { expect(called).to be_truthy }
|
|
26
|
+
end
|
|
2
27
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
|
|
12
|
-
let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
|
|
13
|
-
|
|
14
|
-
let(:redis) {Redis.new(WorkerRoulette.redis_config)}
|
|
15
|
-
|
|
16
|
-
em_before do
|
|
17
|
-
WorkerRoulette.start(evented: true)
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
context "Evented Foreman" do
|
|
21
|
-
let(:subject) {WorkerRoulette.a_foreman(sender)}
|
|
22
|
-
|
|
23
|
-
it "enqueues work" do
|
|
24
|
-
called = false
|
|
25
|
-
foreman = WorkerRoulette.a_foreman('foreman')
|
|
26
|
-
foreman.enqueue_work_order('some old fashion work') do |redis_response, stuff|
|
|
27
|
-
called = true
|
|
28
|
+
it "should enqueue_work_order two work_orders in the sender's slot in the job board" do
|
|
29
|
+
subject.enqueue_work_order(work_orders.first) do
|
|
30
|
+
subject.enqueue_work_order(work_orders.last) do
|
|
31
|
+
expected = work_orders.map { |m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) }
|
|
32
|
+
expect(redis.lrange(sender, 0, -1)).to eq(expected)
|
|
33
|
+
done
|
|
34
|
+
end
|
|
35
|
+
end
|
|
28
36
|
end
|
|
29
|
-
done(0.1) { expect(called).to be_truthy }
|
|
30
|
-
end
|
|
31
37
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
expected = work_orders.map { |m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) }
|
|
36
|
-
expect(redis.lrange(sender, 0, -1)).to eq(expected)
|
|
38
|
+
it "should enqueue_work_order an array of work_orders without headers in the sender's slot in the job board" do
|
|
39
|
+
subject.enqueue_work_order_without_headers(work_orders) do
|
|
40
|
+
expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders)])
|
|
37
41
|
done
|
|
38
42
|
end
|
|
39
43
|
end
|
|
40
|
-
end
|
|
41
44
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
it "should enqueue_work_order an array of work_orders with default headers in the sender's slot in the job board" do
|
|
50
|
-
subject.enqueue_work_order(work_orders) do
|
|
51
|
-
expect(redis.lrange(sender, 0, -1)).to eq(jsonized_work_orders_with_headers)
|
|
52
|
-
done
|
|
45
|
+
it "should enqueue_work_order an array of work_orders with default headers in the sender's slot in the job board" do
|
|
46
|
+
subject.enqueue_work_order(work_orders) do
|
|
47
|
+
expect(redis.lrange(sender, 0, -1)).to eq(jsonized_work_orders_with_headers)
|
|
48
|
+
done
|
|
49
|
+
end
|
|
53
50
|
end
|
|
54
|
-
end
|
|
55
51
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
52
|
+
it "should enqueue_work_order an array of work_orders with additional headers in the sender's slot in the job board" do
|
|
53
|
+
extra_headers = {'foo' => 'bars'}
|
|
54
|
+
subject.enqueue_work_order(work_orders, extra_headers) do
|
|
55
|
+
work_orders_with_headers['headers'].merge!(extra_headers)
|
|
56
|
+
expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders_with_headers)])
|
|
57
|
+
done
|
|
58
|
+
end
|
|
62
59
|
end
|
|
63
|
-
end
|
|
64
60
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
61
|
+
it "should post the sender's id to the job board with an order number" do
|
|
62
|
+
first_foreman = worker_roulette.foreman('first_foreman')
|
|
63
|
+
first_foreman.enqueue_work_order('foo') do
|
|
64
|
+
subject.enqueue_work_order(work_orders.first) do
|
|
65
|
+
subject.enqueue_work_order(work_orders.last) do
|
|
66
|
+
expect(redis.zrange(subject.job_board_key, 0, -1, with_scores: true)).to eq([["first_foreman", 1.0], ["katie_80", 2.0]])
|
|
67
|
+
done
|
|
68
|
+
end
|
|
72
69
|
end
|
|
73
70
|
end
|
|
74
71
|
end
|
|
75
|
-
end
|
|
76
72
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
expect(redis.get(subject.counter_key)).to eq("1")
|
|
82
|
-
first_foreman.enqueue_work_order(work_orders.last) do
|
|
73
|
+
it "should generate a monotically increasing score for senders not on the job board, but not for senders already there" do
|
|
74
|
+
first_foreman = worker_roulette.foreman('first_foreman')
|
|
75
|
+
expect(redis.get(subject.counter_key)).to be_nil
|
|
76
|
+
first_foreman.enqueue_work_order(work_orders.first) do
|
|
83
77
|
expect(redis.get(subject.counter_key)).to eq("1")
|
|
84
|
-
|
|
85
|
-
expect(redis.get(subject.counter_key)).to eq("
|
|
86
|
-
|
|
78
|
+
first_foreman.enqueue_work_order(work_orders.last) do
|
|
79
|
+
expect(redis.get(subject.counter_key)).to eq("1")
|
|
80
|
+
subject.enqueue_work_order(work_orders.first) do
|
|
81
|
+
expect(redis.get(subject.counter_key)).to eq("2")
|
|
82
|
+
done
|
|
83
|
+
end
|
|
87
84
|
end
|
|
88
85
|
end
|
|
89
86
|
end
|
|
90
87
|
end
|
|
91
88
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
subscriber.subscribe(WorkerRoulette::JOB_NOTIFICATIONS) do |message|
|
|
96
|
-
subscriber.unsubscribe(WorkerRoulette::JOB_NOTIFICATIONS)
|
|
97
|
-
expect(message).to eq(WorkerRoulette::JOB_NOTIFICATIONS)
|
|
98
|
-
done
|
|
99
|
-
end.callback { subject.enqueue_work_order(work_orders) }
|
|
100
|
-
end
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
context "Evented Tradesman" do
|
|
104
|
-
let(:foreman) {WorkerRoulette.a_foreman(sender)}
|
|
105
|
-
let(:subject) {WorkerRoulette.a_tradesman}
|
|
89
|
+
context "Evented Tradesman" do
|
|
90
|
+
let(:foreman) {worker_roulette.foreman(sender)}
|
|
91
|
+
let(:subject) {worker_roulette.tradesman(nil, 0.01) }
|
|
106
92
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
93
|
+
it "should be working on behalf of a sender" do
|
|
94
|
+
foreman.enqueue_work_order(work_orders) do
|
|
95
|
+
subject.work_orders! do |r|
|
|
96
|
+
expect(subject.last_sender).to eq(sender)
|
|
97
|
+
done
|
|
98
|
+
end
|
|
112
99
|
end
|
|
113
100
|
end
|
|
114
|
-
end
|
|
115
101
|
|
|
116
102
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
103
|
+
it "should drain one set of work_orders from the sender's slot in the job board" do
|
|
104
|
+
foreman.enqueue_work_order(work_orders) do
|
|
105
|
+
subject.work_orders! do |r|
|
|
106
|
+
expect(r).to eq([work_orders_with_headers])
|
|
107
|
+
subject.work_orders! do |r| expect(r).to be_empty
|
|
108
|
+
subject.work_orders! {|r| expect(r).to be_empty; done} #does not throw an error if queue is alreay empty
|
|
109
|
+
end
|
|
123
110
|
end
|
|
124
111
|
end
|
|
125
112
|
end
|
|
126
|
-
end
|
|
127
113
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
114
|
+
it "should take the oldest sender off the job board (FIFO)" do
|
|
115
|
+
foreman.enqueue_work_order(work_orders) do
|
|
116
|
+
oldest_sender = sender.to_s
|
|
117
|
+
most_recent_sender = 'most_recent_sender'
|
|
118
|
+
most_recent_foreman = worker_roulette.foreman(most_recent_sender)
|
|
119
|
+
most_recent_foreman.enqueue_work_order(work_orders) do
|
|
120
|
+
expect(redis.zrange(subject.job_board_key, 0, -1)).to eq([oldest_sender, most_recent_sender])
|
|
121
|
+
subject.work_orders! { expect(redis.zrange(subject.job_board_key, 0, -1)).to eq([most_recent_sender]); done }
|
|
122
|
+
end
|
|
136
123
|
end
|
|
137
124
|
end
|
|
138
|
-
end
|
|
139
125
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
126
|
+
it "should get the work_orders from the next queue when a new job is ready" do
|
|
127
|
+
#tradesman polls every so often, we care that it is called at least twice, but did not use
|
|
128
|
+
#the built in rspec syntax for that bc if the test ends while we're talking to redis, redis
|
|
129
|
+
#throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
|
|
130
|
+
#call so as not to hurt redis' feelings.
|
|
143
131
|
|
|
144
|
-
|
|
145
|
-
expect(
|
|
146
|
-
expect(subject.last_sender).to be_nil
|
|
147
|
-
done
|
|
148
|
-
end
|
|
149
|
-
end
|
|
132
|
+
expect(subject).to receive(:work_orders!).and_call_original
|
|
133
|
+
expect(subject).to receive(:work_orders!)
|
|
150
134
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
135
|
+
foreman.enqueue_work_order(work_orders) do
|
|
136
|
+
subject.wait_for_work_orders do |redis_work_orders|
|
|
137
|
+
expect(redis_work_orders).to eq([work_orders_with_headers])
|
|
138
|
+
expect(subject.last_sender).to match(/katie_80/)
|
|
139
|
+
done(0.1)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
154
143
|
|
|
155
|
-
|
|
156
|
-
|
|
144
|
+
it "should publish and subscribe on custom channels" do
|
|
145
|
+
good_subscribed = false
|
|
146
|
+
bad_subscribed = false
|
|
157
147
|
|
|
158
|
-
|
|
159
|
-
|
|
148
|
+
tradesman = worker_roulette.tradesman('good_channel', 0.001)
|
|
149
|
+
evil_tradesman = worker_roulette.tradesman('bad_channel', 0.001)
|
|
160
150
|
|
|
161
|
-
|
|
162
|
-
|
|
151
|
+
good_foreman = worker_roulette.foreman('foreman', 'good_channel')
|
|
152
|
+
bad_foreman = worker_roulette.foreman('foreman', 'bad_channel')
|
|
163
153
|
|
|
164
|
-
|
|
165
|
-
|
|
154
|
+
#tradesman polls every so often, we care that it is called at least twice, but did not use
|
|
155
|
+
#the built in rspec syntax for that bc if the test ends while we're talking to redis, redis
|
|
156
|
+
#throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
|
|
157
|
+
#call so as not to hurt redis' feelings.
|
|
158
|
+
expect(tradesman).to receive(:work_orders!).and_call_original
|
|
159
|
+
expect(tradesman).to receive(:work_orders!)
|
|
166
160
|
|
|
167
|
-
|
|
161
|
+
expect(evil_tradesman).to receive(:work_orders!).and_call_original
|
|
162
|
+
expect(evil_tradesman).to receive(:work_orders!)
|
|
168
163
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
expect(good_work.to_s).not_to match("evil")
|
|
172
|
-
end
|
|
164
|
+
good_foreman.enqueue_work_order('some old fashion work') do
|
|
165
|
+
bad_foreman.enqueue_work_order('evil biddings you should not carry out') do
|
|
173
166
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
167
|
+
tradesman.wait_for_work_orders do |good_work|
|
|
168
|
+
expect(good_work.to_s).to match("old fashion")
|
|
169
|
+
expect(good_work.to_s).not_to match("evil")
|
|
170
|
+
end
|
|
178
171
|
|
|
179
|
-
|
|
180
|
-
|
|
172
|
+
evil_tradesman.wait_for_work_orders do |bad_work|
|
|
173
|
+
expect(bad_work.to_s).not_to match("old fashion")
|
|
174
|
+
expect(bad_work.to_s).to match("evil")
|
|
175
|
+
end
|
|
176
|
+
done(0.1)
|
|
181
177
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
subject.wait_for_work_orders(publish) do |redis_work_orders, message, channel|
|
|
185
|
-
subject.unsubscribe {done}
|
|
186
|
-
end
|
|
187
|
-
expect_any_instance_of(EM::Hiredis::PubsubClient).to receive(:close_connection).and_call_original
|
|
188
|
-
end
|
|
189
|
-
|
|
190
|
-
it "should periodically (random time between 20 and 25 seconds?) poll the job board for new work, in case it missed a notification" do
|
|
191
|
-
expect(EM::PeriodicTimer).to receive(:new) {|time| expect(time).to be_within(2.5).of(22.5)}
|
|
192
|
-
publish = proc {foreman.enqueue_work_order('foo')}
|
|
193
|
-
subject.wait_for_work_orders(publish) {done}
|
|
194
|
-
end
|
|
195
|
-
|
|
196
|
-
pending "cancels the old timer when the on_message callback is called" do
|
|
197
|
-
publish = proc {foreman.enqueue_work_order('foo')}
|
|
198
|
-
subject.wait_for_work_orders(publish) do
|
|
199
|
-
expect(subject.send(:timer)).to receive(:cancel).and_call_original
|
|
200
|
-
done
|
|
178
|
+
end
|
|
179
|
+
end
|
|
201
180
|
end
|
|
202
|
-
end
|
|
203
181
|
|
|
204
|
-
|
|
205
|
-
|
|
182
|
+
it "should pull off work orders for more than one sender" do
|
|
183
|
+
tradesman = worker_roulette.tradesman('good_channel')
|
|
206
184
|
|
|
207
|
-
|
|
208
|
-
|
|
185
|
+
good_foreman = worker_roulette.foreman('good_foreman', 'good_channel')
|
|
186
|
+
lazy_foreman = worker_roulette.foreman('lazy_foreman', 'good_channel')
|
|
209
187
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
188
|
+
got_good = false
|
|
189
|
+
got_lazy = false
|
|
190
|
+
good_foreman.enqueue_work_order('do good work') do
|
|
191
|
+
tradesman.work_orders! do |r|
|
|
192
|
+
got_good = true
|
|
193
|
+
expect(r.first['payload']).to eq('do good work')
|
|
194
|
+
end
|
|
216
195
|
end
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
196
|
+
lazy_foreman.enqueue_work_order('just get it done') do
|
|
197
|
+
tradesman.work_orders! do |r|
|
|
198
|
+
got_lazy = true
|
|
199
|
+
expect(r.first['payload']).to eq('just get it done')
|
|
200
|
+
end
|
|
222
201
|
end
|
|
223
|
-
end
|
|
224
202
|
|
|
225
|
-
|
|
203
|
+
done(0.2) {expect(got_good && got_lazy).to eq(true)}
|
|
204
|
+
end
|
|
226
205
|
end
|
|
227
|
-
end
|
|
228
206
|
|
|
229
|
-
|
|
207
|
+
pending "should return a hash with a string in the payload if OJ cannot parse the json"
|
|
230
208
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
209
|
+
context "Failure" do
|
|
210
|
+
it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; done; end
|
|
211
|
+
end
|
|
234
212
|
|
|
235
|
-
|
|
236
|
-
|
|
213
|
+
context "Concurrent Access" do
|
|
214
|
+
it "should not leak connections"
|
|
237
215
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
216
|
+
it "should be fork() proof" do
|
|
217
|
+
@subject = worker_roulette.tradesman
|
|
218
|
+
@subject.work_orders! do
|
|
219
|
+
fork do
|
|
220
|
+
@subject.work_orders!
|
|
221
|
+
end
|
|
243
222
|
end
|
|
223
|
+
done(1)
|
|
244
224
|
end
|
|
245
|
-
done(1)
|
|
246
225
|
end
|
|
247
226
|
end
|
|
248
227
|
end
|