job_dispatch 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.rspec +1 -0
- data/.travis.yml +13 -0
- data/Gemfile +20 -0
- data/Guardfile +13 -0
- data/LICENSE.txt +22 -0
- data/README.md +85 -0
- data/Rakefile +10 -0
- data/bin/job-dispatcher +34 -0
- data/bin/job-status +69 -0
- data/bin/job-worker +40 -0
- data/examples/mongoid-job.rb +43 -0
- data/job_dispatch.gemspec +33 -0
- data/lib/job_dispatch/broker/command.rb +45 -0
- data/lib/job_dispatch/broker/internal_job.rb +32 -0
- data/lib/job_dispatch/broker/socket.rb +85 -0
- data/lib/job_dispatch/broker.rb +523 -0
- data/lib/job_dispatch/client/proxy.rb +34 -0
- data/lib/job_dispatch/client/proxy_error.rb +18 -0
- data/lib/job_dispatch/client/synchronous_proxy.rb +29 -0
- data/lib/job_dispatch/client.rb +49 -0
- data/lib/job_dispatch/configuration.rb +7 -0
- data/lib/job_dispatch/identity.rb +54 -0
- data/lib/job_dispatch/job.rb +44 -0
- data/lib/job_dispatch/signaller.rb +30 -0
- data/lib/job_dispatch/sockets/enqueue.rb +18 -0
- data/lib/job_dispatch/status.rb +79 -0
- data/lib/job_dispatch/version.rb +3 -0
- data/lib/job_dispatch/worker/item.rb +43 -0
- data/lib/job_dispatch/worker/socket.rb +96 -0
- data/lib/job_dispatch/worker.rb +120 -0
- data/lib/job_dispatch.rb +97 -0
- data/spec/factories/jobs.rb +19 -0
- data/spec/job_dispatch/broker/socket_spec.rb +53 -0
- data/spec/job_dispatch/broker_spec.rb +737 -0
- data/spec/job_dispatch/identity_spec.rb +88 -0
- data/spec/job_dispatch/job_spec.rb +77 -0
- data/spec/job_dispatch/worker/socket_spec.rb +32 -0
- data/spec/job_dispatch/worker_spec.rb +24 -0
- data/spec/job_dispatch_spec.rb +0 -0
- data/spec/spec_helper.rb +23 -0
- data/spec/support/test_job.rb +30 -0
- metadata +255 -0
@@ -0,0 +1,737 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require 'spec_helper'
|
3
|
+
|
4
|
+
# dummy Job class
|
5
|
+
Job = TestJob
|
6
|
+
|
7
|
+
describe JobDispatch::Broker do
|
8
|
+
|
9
|
+
Command ||= JobDispatch::Broker::Command
|
10
|
+
Identity ||= JobDispatch::Identity
|
11
|
+
|
12
|
+
subject { JobDispatch::Broker.new('tcp://localhost:2000', 'tcp://localhost:2001') }
|
13
|
+
|
14
|
+
let(:worker_id) { Identity.new([0, 0x80, 0, 0x41, 0x31].pack('c*')) }
|
15
|
+
let(:worker_id2) { Identity.new([0, 0x80, 0, 0x41, 0x32].pack('c*')) }
|
16
|
+
let(:worker_id3) { Identity.new([0, 0x80, 0, 0x41, 0x33].pack('c*')) }
|
17
|
+
|
18
|
+
before :each do
|
19
|
+
JobDispatch.config.job_class = double('JobClass')
|
20
|
+
end
|
21
|
+
|
22
|
+
context "tracking communication state" do
|
23
|
+
|
24
|
+
it "reading a command adds the requester to the list of connections awaiting reply" do
|
25
|
+
command = Command.new(worker_id, {commmand: 'ready'})
|
26
|
+
socket = double('Broker::Socket', :read_command => command)
|
27
|
+
subject.stub(:socket => socket)
|
28
|
+
subject.read_command
|
29
|
+
expect(subject.workers_waiting_for_reply).to include(worker_id)
|
30
|
+
end
|
31
|
+
|
32
|
+
it "sending a command removes the worker from the list of workers awaiting reply" do
|
33
|
+
subject.workers_waiting_for_reply << worker_id
|
34
|
+
command = Command.new(worker_id, {commmand: 'idle'})
|
35
|
+
socket = double('Broker::Socket', :send_command => command)
|
36
|
+
socket.should_receive(:send_command).with(command)
|
37
|
+
subject.stub(:socket => socket)
|
38
|
+
subject.send_command(command)
|
39
|
+
expect(subject.workers_waiting_for_reply).not_to include(worker_id)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "does not send a command to a worker unless it is awaiting a reply" do
|
43
|
+
command = Command.new(worker_id, {commmand: 'idle'})
|
44
|
+
socket = double('Broker::Socket', :send_command => command)
|
45
|
+
socket.should_not_receive(:send_command)
|
46
|
+
subject.stub(:socket => socket)
|
47
|
+
expect { subject.send_command(command) }.to raise_error
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context "responding to command" do
|
52
|
+
|
53
|
+
context "'status'" do
|
54
|
+
let(:command) { Command.new(worker_id, {command: 'status'}) }
|
55
|
+
|
56
|
+
before :each do
|
57
|
+
subject.reply_exceptions = false
|
58
|
+
end
|
59
|
+
|
60
|
+
it "returns a command" do
|
61
|
+
expect(subject.process_command(command)).to be_a(Command)
|
62
|
+
end
|
63
|
+
|
64
|
+
it "returns a status object" do
|
65
|
+
result = subject.process_command(command)
|
66
|
+
expect(result.worker_id).to eq(worker_id)
|
67
|
+
expect(result.parameters).to be_a(Hash)
|
68
|
+
expect(result.parameters[:status]).to eq('OK')
|
69
|
+
expect(result.parameters[:queues]).to be_a(Hash)
|
70
|
+
end
|
71
|
+
|
72
|
+
it "returns a JSONable parameters object" do
|
73
|
+
subject.process_command(Command.new(worker_id2, {command: 'ready', worker_name: 'test worker'}))
|
74
|
+
result = subject.process_command(command)
|
75
|
+
expect { json = JSON.dump(result.parameters) }.not_to raise_error
|
76
|
+
end
|
77
|
+
|
78
|
+
it "returns a list of workers including idle and working" do
|
79
|
+
subject.workers_waiting_for_reply << worker_id2
|
80
|
+
subject.process_command(Command.new(worker_id2, {command: 'ready', worker_name: 'test worker 1'}))
|
81
|
+
subject.workers_waiting_for_reply << worker_id3
|
82
|
+
subject.process_command(Command.new(worker_id3, {command: 'ready', worker_name: 'test worker 2'}))
|
83
|
+
|
84
|
+
|
85
|
+
@job = FactoryGirl.build :job
|
86
|
+
@socket = double('Broker::Socket', :send_command => nil)
|
87
|
+
subject.stub(:socket => @socket)
|
88
|
+
@socket.should_receive(:send_command) do |cmd|
|
89
|
+
#expect(cmd.worker_id).to eq(worker_id)
|
90
|
+
expect(cmd.parameters[:command]).to eq('job')
|
91
|
+
expect(cmd.parameters[:target]).to eq(@job.target)
|
92
|
+
end
|
93
|
+
|
94
|
+
job_class = double('JobClass')
|
95
|
+
job_class.stub(:dequeue_job_for_queue).and_return(@job)
|
96
|
+
JobDispatch.config.job_class = job_class
|
97
|
+
|
98
|
+
# dispatch a job to a worker
|
99
|
+
subject.dispatch_jobs_to_workers
|
100
|
+
|
101
|
+
# now get status
|
102
|
+
result = subject.process_command(command)
|
103
|
+
|
104
|
+
expect(result.parameters[:queues]).to be_a(Hash)
|
105
|
+
expect(result.parameters[:queues].size).to eq(1)
|
106
|
+
expect(result.parameters[:queues][:default]).to be_a(Hash)
|
107
|
+
expect(result.parameters[:queues][:default][worker_id2.to_hex]).to be_a(Hash)
|
108
|
+
expect(result.parameters[:queues][:default][worker_id2.to_hex][:status]).to eq(:processing)
|
109
|
+
expect(result.parameters[:queues][:default][worker_id2.to_hex][:job_id]).to eq(@job.id)
|
110
|
+
expect(result.parameters[:queues][:default][worker_id2.to_hex][:name]).to eq('test worker 1')
|
111
|
+
expect(result.parameters[:queues][:default][worker_id3.to_hex]).to be_a(Hash)
|
112
|
+
expect(result.parameters[:queues][:default][worker_id3.to_hex][:status]).to eq(:idle)
|
113
|
+
expect(result.parameters[:queues][:default][worker_id3.to_hex][:name]).to eq('test worker 2')
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
context "'quit'" do
|
118
|
+
let(:command) { Command.new(worker_id, {command: 'quit'}) }
|
119
|
+
|
120
|
+
it "returns a command" do
|
121
|
+
@result = subject.process_command(command)
|
122
|
+
expect(@result).to be_a(Command)
|
123
|
+
end
|
124
|
+
|
125
|
+
it "acknowledges the command" do
|
126
|
+
@result = subject.process_command(command)
|
127
|
+
expect(@result.parameters[:status]).to eq("bye")
|
128
|
+
end
|
129
|
+
|
130
|
+
it "sets running to false" do
|
131
|
+
@result = subject.process_command(command)
|
132
|
+
expect(subject.running?).to be_false
|
133
|
+
end
|
134
|
+
|
135
|
+
it "sends a quit message to a waiting worker" do
|
136
|
+
socket = double('Broker::Socket', :send_command => command)
|
137
|
+
socket.should_receive(:send_command) do |cmd|
|
138
|
+
expect(cmd.worker_id).to eq(worker_id)
|
139
|
+
end
|
140
|
+
subject.workers_waiting_for_reply << worker_id
|
141
|
+
subject.process_command(Command.new(worker_id, command: 'ready'))
|
142
|
+
subject.stub(:socket => socket)
|
143
|
+
@result = subject.process_command(command)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
# when a worker is ready for work!
|
148
|
+
context "'ready'" do
|
149
|
+
let(:command) { Command.new(worker_id, {command: 'ready', queue: 'example', worker_name: 'ruby worker'}) }
|
150
|
+
it "returns nil" do
|
151
|
+
@result = subject.process_command(command)
|
152
|
+
expect(@result).to be_nil
|
153
|
+
end
|
154
|
+
|
155
|
+
it "adds the worker to the list of workers awaiting replies" do
|
156
|
+
@result = subject.process_command(command)
|
157
|
+
expect(subject.workers_waiting_for_jobs.keys).to include(worker_id)
|
158
|
+
end
|
159
|
+
|
160
|
+
it "adds the worker to the queue" do
|
161
|
+
@result = subject.process_command(command)
|
162
|
+
expect(subject.queues[:example]).to include(worker_id)
|
163
|
+
end
|
164
|
+
|
165
|
+
it "adds the worker's name to the hash of worker names" do
|
166
|
+
@result = subject.process_command(command)
|
167
|
+
expect(subject.worker_names[worker_id]).to eq('ruby worker')
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
|
172
|
+
context "'goodbye'" do
|
173
|
+
# the goodbye command is sent from a new socket, so the actual socket identity will be different from
|
174
|
+
# the socket waiting for a job or idle command.
|
175
|
+
let(:worker_name) { 'ruby worker' }
|
176
|
+
let(:goodbye_command) { Command.new(worker_id2, {command: 'goodbye', worker_name: worker_name}) }
|
177
|
+
let(:ready_command) { Command.new(worker_id, {command: 'ready', queue: 'example', worker_name: worker_name}) }
|
178
|
+
context 'with idle worker' do
|
179
|
+
before :each do
|
180
|
+
subject.process_command(ready_command)
|
181
|
+
end
|
182
|
+
|
183
|
+
it "returns an object" do
|
184
|
+
@result = subject.process_command(goodbye_command)
|
185
|
+
expect(@result).to be_a(Command)
|
186
|
+
expect(@result.parameters).to be_a(Hash)
|
187
|
+
end
|
188
|
+
|
189
|
+
it "removes the worker's name from the worker_name hash" do
|
190
|
+
@result = subject.process_command(goodbye_command)
|
191
|
+
expect(subject.worker_names).not_to include(worker_id)
|
192
|
+
end
|
193
|
+
|
194
|
+
it "removes the worker from the list waiting reply" do
|
195
|
+
subject.process_command(goodbye_command)
|
196
|
+
expect(subject.workers_waiting_for_reply).not_to include(worker_id)
|
197
|
+
end
|
198
|
+
|
199
|
+
it "removes the worker from the list of workers ready for jobs" do
|
200
|
+
subject.process_command(goodbye_command)
|
201
|
+
expect(subject.workers_waiting_for_jobs).not_to include(worker_id)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
context 'without an idle worker' do
|
206
|
+
it "returns an object" do
|
207
|
+
@result = subject.process_command(goodbye_command)
|
208
|
+
expect(@result).to be_a(Command)
|
209
|
+
expect(@result.parameters).to be_a(Hash)
|
210
|
+
end
|
211
|
+
|
212
|
+
it "removes the worker's name from the worker_name hash" do
|
213
|
+
@result = subject.process_command(goodbye_command)
|
214
|
+
expect(subject.worker_names).not_to include(worker_id)
|
215
|
+
end
|
216
|
+
|
217
|
+
it "removes the worker from the list waiting reply" do
|
218
|
+
subject.process_command(goodbye_command)
|
219
|
+
expect(subject.workers_waiting_for_reply).not_to include(worker_id)
|
220
|
+
end
|
221
|
+
|
222
|
+
it "removes the worker from the list of workers ready for jobs" do
|
223
|
+
subject.process_command(goodbye_command)
|
224
|
+
expect(subject.workers_waiting_for_jobs).not_to include(worker_id)
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
# a worker has completed a task and is optionally asking for another one.
|
230
|
+
context "'completed'" do
|
231
|
+
let(:job_id) { '1234' }
|
232
|
+
before :each do
|
233
|
+
@job = FactoryGirl.build :job, :id => job_id
|
234
|
+
@job.stub(:succeeded!) do
|
235
|
+
@job.status = JobDispatch::Job::COMPLETED
|
236
|
+
true
|
237
|
+
end
|
238
|
+
@job.stub(:failed!) do
|
239
|
+
@job.status = JobDispatch::Job::FAILED
|
240
|
+
true
|
241
|
+
end
|
242
|
+
@job.stub(:reload => true)
|
243
|
+
subject.jobs_in_progress[job_id] = @job
|
244
|
+
JobDispatch.config.job_class.stub(:find) do |id|
|
245
|
+
raise StandardError, "Job not found" if id != @job.id
|
246
|
+
@job
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
context "and ask for another job" do
|
251
|
+
let(:command) { Command.new(worker_id, {
|
252
|
+
command: 'completed',
|
253
|
+
job_id: job_id,
|
254
|
+
status: 'success',
|
255
|
+
result: 'the result',
|
256
|
+
ready: true,
|
257
|
+
queue: 'example'
|
258
|
+
}) }
|
259
|
+
|
260
|
+
it "returns nil" do
|
261
|
+
@result = subject.process_command(command)
|
262
|
+
expect(@result).to be_nil
|
263
|
+
end
|
264
|
+
|
265
|
+
it "adds the worker to the list of workers awaiting replies" do
|
266
|
+
@result = subject.process_command(command)
|
267
|
+
expect(subject.workers_waiting_for_jobs.keys).to include(worker_id)
|
268
|
+
end
|
269
|
+
|
270
|
+
it "adds the worker to the queue" do
|
271
|
+
@result = subject.process_command(command)
|
272
|
+
expect(subject.queues[:example]).to include(worker_id)
|
273
|
+
end
|
274
|
+
|
275
|
+
it "marks the job as succeeded" do
|
276
|
+
@job.should_receive(:succeeded!).with('the result')
|
277
|
+
subject.process_command(command)
|
278
|
+
end
|
279
|
+
end
|
280
|
+
|
281
|
+
context 'and not asking for another job' do
|
282
|
+
let(:command) { Command.new(worker_id, {
|
283
|
+
command: 'completed',
|
284
|
+
job_id: job_id,
|
285
|
+
status: 'success',
|
286
|
+
result: 'the result'
|
287
|
+
}) }
|
288
|
+
|
289
|
+
it "returns thanks" do
|
290
|
+
@result = subject.process_command(command)
|
291
|
+
expect(@result).to be_a(Command)
|
292
|
+
expect(@result.parameters[:status]).to eq('thanks')
|
293
|
+
end
|
294
|
+
|
295
|
+
it "adds the worker to the list of workers awaiting replies" do
|
296
|
+
@result = subject.process_command(command)
|
297
|
+
expect(subject.workers_waiting_for_jobs.keys).not_to include(worker_id)
|
298
|
+
end
|
299
|
+
|
300
|
+
it "adds the worker to the queue" do
|
301
|
+
@result = subject.process_command(command)
|
302
|
+
expect(subject.queues[:example]).not_to include(worker_id)
|
303
|
+
end
|
304
|
+
|
305
|
+
it "marks the job as succeeded" do
|
306
|
+
@job.should_receive(:succeeded!).with('the result')
|
307
|
+
subject.process_command(command)
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
context 'when the job fails, it is marked as failed' do
|
312
|
+
let(:command) { Command.new(worker_id, {
|
313
|
+
command: 'completed',
|
314
|
+
job_id: job_id,
|
315
|
+
status: 'error',
|
316
|
+
result: 'the error message'
|
317
|
+
}) }
|
318
|
+
|
319
|
+
it "marks the job as failed" do
|
320
|
+
@job.should_receive(:failed!).with('the error message')
|
321
|
+
subject.process_command(command)
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
context "when the job doesn't exist" do
|
326
|
+
let(:command) { Command.new(worker_id, {
|
327
|
+
command: 'completed',
|
328
|
+
job_id: 'wrong',
|
329
|
+
status: 'success',
|
330
|
+
result: 'the result'
|
331
|
+
}) }
|
332
|
+
|
333
|
+
it "returns 'thanks' anyway" do
|
334
|
+
JobDispatch.config.job_class.stub(:find).and_raise(StandardError, "Job not found")
|
335
|
+
@result = subject.process_command(command)
|
336
|
+
expect(@result).to be_a(Command)
|
337
|
+
expect(@result.parameters[:status]).to eq('thanks')
|
338
|
+
end
|
339
|
+
end
|
340
|
+
|
341
|
+
|
342
|
+
# This context is for when a job is being executed by a worker and the timeout has been reached.
|
343
|
+
# The broker will have purged the job from the list of jobs in progress, and the worker will be
|
344
|
+
# offline since it has not yet asked for more work to do. If the job completes successfully, the
|
345
|
+
# broker needs to update the status of the job to completed (or failed) appropriately, which will
|
346
|
+
# override any retry. In practice, long jobs should use the "touch" command to let the broker
|
347
|
+
# know that they are in fact still working on the job and to please extend the timeout deadline.
|
348
|
+
context "after the job has timed out" do
|
349
|
+
let(:command) { Command.new(worker_id, {
|
350
|
+
command: 'completed',
|
351
|
+
job_id: @job.id,
|
352
|
+
status: 'success',
|
353
|
+
result: 'the result',
|
354
|
+
ready: true,
|
355
|
+
queue: 'example'
|
356
|
+
}) }
|
357
|
+
|
358
|
+
before :each do
|
359
|
+
@job.stub(:as_json).and_return({status: @job.status, result: @job.result, id: @job.id})
|
360
|
+
@job.status = 1 # mark as in progress
|
361
|
+
JobDispatch.config.job_class.stub(:dequeue_job_for_queue).and_return(@job)
|
362
|
+
|
363
|
+
# simulate the job has expired:
|
364
|
+
@job.stub(:timed_out? => true)
|
365
|
+
subject.expire_timed_out_jobs
|
366
|
+
end
|
367
|
+
|
368
|
+
it "Updates the job status to be completed" do
|
369
|
+
subject.process_command(command)
|
370
|
+
expect(@job.status).to eq(JobDispatch::Job::COMPLETED)
|
371
|
+
end
|
372
|
+
|
373
|
+
it "adds the worker to the queue" do
|
374
|
+
subject.process_command(command)
|
375
|
+
expect(subject.queues[:example]).to include(worker_id)
|
376
|
+
end
|
377
|
+
|
378
|
+
it "marks the job as succeeded" do
|
379
|
+
@job.should_receive(:succeeded!).with('the result')
|
380
|
+
subject.process_command(command)
|
381
|
+
end
|
382
|
+
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
end
|
387
|
+
|
388
|
+
|
389
|
+
context "idle workers" do
|
390
|
+
|
391
|
+
before :each do
|
392
|
+
@socket = double('Broker::Socket', :send_command => nil)
|
393
|
+
subject.stub(:socket => @socket)
|
394
|
+
|
395
|
+
@time = Time.now
|
396
|
+
# this worker will be IDLE
|
397
|
+
Timecop.freeze(@time) do
|
398
|
+
subject.workers_waiting_for_reply << worker_id # ugly: simulating a prior read_command (implementation detail!)
|
399
|
+
command = Command.new(worker_id, {command: 'ready', queue: 'example'})
|
400
|
+
@result = subject.process_command(command)
|
401
|
+
end
|
402
|
+
# this worker should stay in the queue
|
403
|
+
Timecop.freeze(@time + 5) do
|
404
|
+
subject.workers_waiting_for_reply << worker_id2 # ugly: simulating a prior read_command (implementation detail!)
|
405
|
+
command = Command.new(worker_id2, {command: 'ready', queue: 'example'})
|
406
|
+
@result = subject.process_command(command)
|
407
|
+
end
|
408
|
+
|
409
|
+
end
|
410
|
+
|
411
|
+
it "that have waited long enough receive idle commands" do
|
412
|
+
@socket.should_receive(:send_command) do |cmd|
|
413
|
+
expect(cmd.worker_id).to eq(worker_id)
|
414
|
+
expect(cmd.parameters[:command]).to eq('idle')
|
415
|
+
end
|
416
|
+
|
417
|
+
Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
|
418
|
+
subject.send_idle_commands
|
419
|
+
end
|
420
|
+
|
421
|
+
expect(subject.workers_waiting_for_reply).not_to include(worker_id)
|
422
|
+
expect(subject.queues[:example]).not_to include(worker_id)
|
423
|
+
end
|
424
|
+
|
425
|
+
it "that have not waited long enough are still waiting" do
|
426
|
+
@socket.should_receive(:send_command) do |cmd|
|
427
|
+
expect(cmd.worker_id).not_to eq(worker_id2)
|
428
|
+
end
|
429
|
+
|
430
|
+
Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
|
431
|
+
subject.send_idle_commands
|
432
|
+
end
|
433
|
+
|
434
|
+
expect(subject.workers_waiting_for_reply).to include(worker_id2)
|
435
|
+
expect(subject.queues[:example]).to include(worker_id2)
|
436
|
+
end
|
437
|
+
end
|
438
|
+
|
439
|
+
|
440
|
+
context "dispatching jobs" do
|
441
|
+
context "when there are jobs in a queue" do
|
442
|
+
before :each do
|
443
|
+
@job = FactoryGirl.build :job
|
444
|
+
@socket = double('Broker::Socket', :send_command => nil)
|
445
|
+
subject.stub(:socket => @socket)
|
446
|
+
end
|
447
|
+
|
448
|
+
it "the job is sent to an idle worker" do
|
449
|
+
@socket.should_receive(:send_command) do |cmd|
|
450
|
+
expect(cmd.worker_id).to eq(worker_id)
|
451
|
+
expect(cmd.parameters[:command]).to eq('job')
|
452
|
+
expect(cmd.parameters[:target]).to eq(@job.target)
|
453
|
+
end
|
454
|
+
|
455
|
+
job_class = double('JobClass')
|
456
|
+
job_class.stub(:dequeue_job_for_queue).and_return(@job)
|
457
|
+
job_class.should_receive(:dequeue_job_for_queue).with('example')
|
458
|
+
JobDispatch.config.job_class = job_class
|
459
|
+
|
460
|
+
# send ready command => adds idle worker state
|
461
|
+
subject.workers_waiting_for_reply << worker_id # simulating read_command
|
462
|
+
@result = subject.process_command(Command.new(worker_id, {
|
463
|
+
command: 'ready',
|
464
|
+
queue: 'example',
|
465
|
+
worker_name: 'ruby worker',
|
466
|
+
}))
|
467
|
+
expect(@result).to be_nil # no immediate response
|
468
|
+
expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
|
469
|
+
|
470
|
+
subject.dispatch_jobs_to_workers
|
471
|
+
end
|
472
|
+
end
|
473
|
+
|
474
|
+
context "when an error occurs dequeuing jobs" do
|
475
|
+
before :each do
|
476
|
+
@job_class = double('JobClass')
|
477
|
+
@job_class.stub(:dequeue_job_for_queue).and_raise(StandardError, "something bad happened")
|
478
|
+
JobDispatch.config.job_class = @job_class
|
479
|
+
end
|
480
|
+
|
481
|
+
it "behaves as if there was no job" do
|
482
|
+
# send ready command => adds idle worker state
|
483
|
+
subject.workers_waiting_for_reply << worker_id # simulating read_command
|
484
|
+
@result = subject.process_command(Command.new(worker_id, {
|
485
|
+
command: 'ready',
|
486
|
+
queue: 'example',
|
487
|
+
worker_name: 'ruby worker',
|
488
|
+
}))
|
489
|
+
expect(@result).to be_nil # no immediate response
|
490
|
+
expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
|
491
|
+
|
492
|
+
# no job should be sent
|
493
|
+
subject.should_not_receive(:send_job_to_worker)
|
494
|
+
expect { subject.dispatch_jobs_to_workers }.not_to raise_error
|
495
|
+
end
|
496
|
+
end
|
497
|
+
end
|
498
|
+
|
499
|
+
context "expired jobs" do
|
500
|
+
it "are removed from jobs list when they expire" do
|
501
|
+
time = Time.now
|
502
|
+
@job = FactoryGirl.build :job, :expire_execution_at => time - 5.seconds
|
503
|
+
@job_id = @job.id.to_s
|
504
|
+
subject.jobs_in_progress[@job_id] = @job
|
505
|
+
subject.jobs_in_progress_workers[@job_id] = worker_id
|
506
|
+
subject.expire_timed_out_jobs
|
507
|
+
expect(subject.jobs_in_progress).to be_empty
|
508
|
+
expect(subject.jobs_in_progress_workers).to be_empty
|
509
|
+
end
|
510
|
+
|
511
|
+
|
512
|
+
it "include InternalJob commands" do
|
513
|
+
socket = double('Broker::Socket')
|
514
|
+
subject.stub(:socket => socket)
|
515
|
+
socket.stub(:send_command => true)
|
516
|
+
|
517
|
+
# send ready command => adds idle worker state
|
518
|
+
subject.workers_waiting_for_reply << worker_id # simulating read_command
|
519
|
+
@result = subject.process_command(Command.new(worker_id, {
|
520
|
+
command: 'ready',
|
521
|
+
queue: 'example',
|
522
|
+
worker_name: 'ruby worker',
|
523
|
+
}))
|
524
|
+
subject.send_idle_commands(Time.now + JobDispatch::Broker::WORKER_IDLE_TIME + 10)
|
525
|
+
expect(subject.jobs_in_progress_workers.length).to eq(1)
|
526
|
+
@time = Time.now + JobDispatch::Broker::WORKER_IDLE_TIME + 10 + JobDispatch::Job::DEFAULT_EXECUTION_TIMEOUT + 10
|
527
|
+
JobDispatch::Broker::InternalJob.any_instance.should_not_receive(:failed!)
|
528
|
+
Timecop.freeze(@time) do
|
529
|
+
subject.expire_timed_out_jobs
|
530
|
+
end
|
531
|
+
end
|
532
|
+
end
|
533
|
+
|
534
|
+
context "touching a job" do
|
535
|
+
before :each do
|
536
|
+
@time = Time.now
|
537
|
+
# this worker will be IDLE
|
538
|
+
@job = FactoryGirl.build :job, :expire_execution_at => @time + 5.seconds
|
539
|
+
@job_id = @job.id.to_s
|
540
|
+
subject.jobs_in_progress[@job_id] = @job
|
541
|
+
subject.jobs_in_progress_workers[@job_id] = worker_id
|
542
|
+
@socket = double('Broker::Socket', :send_command => nil)
|
543
|
+
subject.stub(:socket => @socket)
|
544
|
+
@socket.stub(:read_command).and_return(nil)
|
545
|
+
@job.stub(:save)
|
546
|
+
end
|
547
|
+
|
548
|
+
it "updates the expires_execute_at time" do
|
549
|
+
Timecop.freeze(@time) do
|
550
|
+
subject.touch_job(Command.new(worker_id, {command: "touch", job_id: @job_id}))
|
551
|
+
end
|
552
|
+
expect(@job.expire_execution_at).to eq(@time + JobDispatch::Job::DEFAULT_EXECUTION_TIMEOUT)
|
553
|
+
end
|
554
|
+
|
555
|
+
it "updates the expire_execution_at time with a custom timeout" do
|
556
|
+
Timecop.freeze(@time) do
|
557
|
+
subject.touch_job(Command.new(worker_id, {command: "touch", job_id: @job_id, timeout: 100}))
|
558
|
+
end
|
559
|
+
expect(@job.expire_execution_at).to eq(@time + 100.seconds)
|
560
|
+
end
|
561
|
+
end
|
562
|
+
|
563
|
+
context "enqueue a job" do
|
564
|
+
before :each do
|
565
|
+
@job_attrs = FactoryGirl.attributes_for :job
|
566
|
+
end
|
567
|
+
|
568
|
+
it "Creates a job" do
|
569
|
+
JobDispatch.config.job_class = double('JobClass')
|
570
|
+
JobDispatch.config.job_class.should_receive(:create!).with(@job_attrs)
|
571
|
+
command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
|
572
|
+
subject.process_command(command)
|
573
|
+
end
|
574
|
+
|
575
|
+
it "returns the job id" do
|
576
|
+
job_id = 12345
|
577
|
+
JobDispatch.config.job_class = double('JobClass')
|
578
|
+
JobDispatch.config.job_class.stub(:create! => double('Job', :id => job_id))
|
579
|
+
command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
|
580
|
+
result = subject.process_command(command)
|
581
|
+
expect(result.parameters[:job_id]).to eq(job_id.to_s)
|
582
|
+
job_id
|
583
|
+
end
|
584
|
+
|
585
|
+
it "returns an error if the arguments are no good" do
|
586
|
+
JobDispatch.config.job_class = double('JobClass')
|
587
|
+
JobDispatch.config.job_class.stub(:create!).and_raise("no good") # simulate some database save error
|
588
|
+
command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
|
589
|
+
result = subject.process_command(command)
|
590
|
+
expect(result.parameters[:status]).to eq('error')
|
591
|
+
expect(result.parameters[:message]).to eq('no good')
|
592
|
+
end
|
593
|
+
|
594
|
+
it "returns an error if the 'job' parameter is missing" do
|
595
|
+
JobDispatch.config.job_class = double('JobClass')
|
596
|
+
command = Command.new(:some_client, {command: "enqueue"})
|
597
|
+
result = subject.process_command(command)
|
598
|
+
expect(result.parameters[:status]).to eq('error')
|
599
|
+
end
|
600
|
+
end
|
601
|
+
|
602
|
+
|
603
|
+
context "'notify'" do
|
604
|
+
|
605
|
+
before :each do
|
606
|
+
@job = FactoryGirl.build :job
|
607
|
+
@job_class = double('JobClass')
|
608
|
+
@job_class.stub(:dequeue_job_for_queue).and_return(@job)
|
609
|
+
@job_class.stub(:find) do |id|
|
610
|
+
@job if id == @job.id
|
611
|
+
end
|
612
|
+
JobDispatch.config.job_class = @job_class
|
613
|
+
end
|
614
|
+
|
615
|
+
context "with no jobs" do
|
616
|
+
it "returns no such job" do
|
617
|
+
command = Command.new(:client, {command: 'notify', job_id: 1234})
|
618
|
+
@job_class.should_receive(:find).with(1234).and_raise(StandardError, "bozo")
|
619
|
+
result = subject.process_command(command)
|
620
|
+
expect(result.parameters[:status]).to eq('error')
|
621
|
+
expect(result.parameters[:message]).to eq('bozo')
|
622
|
+
end
|
623
|
+
end
|
624
|
+
|
625
|
+
context "with a completed job" do
|
626
|
+
it "returns the job result" do
|
627
|
+
@job = FactoryGirl.build :job, status: JobDispatch::Job::COMPLETED, result: 'hooray'
|
628
|
+
command = Command.new(:client, {command: 'notify', job_id: @job.id})
|
629
|
+
@job_class.should_receive(:find).with(@job.id).and_return(@job)
|
630
|
+
result = subject.process_command(command)
|
631
|
+
expect(result.parameters[:status]).to eq('completed')
|
632
|
+
end
|
633
|
+
end
|
634
|
+
|
635
|
+
context "with a job in progress" do
|
636
|
+
before :each do
|
637
|
+
@socket = double('Broker::Socket')
|
638
|
+
subject.stub(:socket => @socket)
|
639
|
+
@socket.stub(:send_command => nil)
|
640
|
+
|
641
|
+
# worker ready command
|
642
|
+
|
643
|
+
subject.workers_waiting_for_reply << worker_id # simulating read_command
|
644
|
+
@result = subject.process_command(Command.new(worker_id, {
|
645
|
+
command: 'ready',
|
646
|
+
queue: 'example',
|
647
|
+
worker_name: 'ruby worker',
|
648
|
+
}))
|
649
|
+
expect(@result).to be_nil # no immediate response
|
650
|
+
expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
|
651
|
+
|
652
|
+
# dispatch job to worker
|
653
|
+
|
654
|
+
subject.dispatch_jobs_to_workers
|
655
|
+
|
656
|
+
# send notify command
|
657
|
+
|
658
|
+
subject.workers_waiting_for_reply << worker_id2 # simulating read_command
|
659
|
+
@result = subject.process_command(Command.new(worker_id2, {
|
660
|
+
command: 'notify',
|
661
|
+
job_id: @job.id
|
662
|
+
}))
|
663
|
+
end
|
664
|
+
|
665
|
+
it "registers the job subscriber" do
|
666
|
+
expect(@result).to be_nil # no immediate response
|
667
|
+
expect(subject.job_subscribers[@job.id]).to include(worker_id2)
|
668
|
+
expect(subject.workers_waiting_for_reply).to include(worker_id2)
|
669
|
+
end
|
670
|
+
|
671
|
+
it "returns when the job completes" do
|
672
|
+
# when the worker completes, the notify socket should be notified that
|
673
|
+
# the job completed.
|
674
|
+
socket2 = double('Broker::Socket')
|
675
|
+
subject.stub(:socket => socket2)
|
676
|
+
socket2.should_receive(:send_command) do |cmd|
|
677
|
+
expect(cmd.worker_id).to eq(worker_id2)
|
678
|
+
expect(cmd.parameters[:status]).to eq('completed')
|
679
|
+
expect(cmd.parameters[:job_id]).to eq(@job.id)
|
680
|
+
end
|
681
|
+
|
682
|
+
expect(subject.workers_waiting_for_reply).to include(worker_id2)
|
683
|
+
|
684
|
+
# worker completed job -> should send response to notify socket.
|
685
|
+
subject.workers_waiting_for_reply << worker_id # simulating read_command
|
686
|
+
@result = subject.process_command(Command.new(worker_id, {
|
687
|
+
command: 'completed',
|
688
|
+
job_id: @job.id,
|
689
|
+
status: 'success',
|
690
|
+
result: 'foobar'
|
691
|
+
}))
|
692
|
+
end
|
693
|
+
|
694
|
+
it "returns when a job fails" do
|
695
|
+
# when the worker completes, the notify socket should be notified that
|
696
|
+
# the job completed.
|
697
|
+
socket2 = double('Broker::Socket')
|
698
|
+
subject.stub(:socket => socket2)
|
699
|
+
socket2.should_receive(:send_command) do |cmd|
|
700
|
+
expect(cmd.worker_id).to eq(worker_id2)
|
701
|
+
expect(cmd.parameters[:status]).to eq('failed')
|
702
|
+
expect(cmd.parameters[:job_id]).to eq(@job.id)
|
703
|
+
end
|
704
|
+
|
705
|
+
expect(subject.workers_waiting_for_reply).to include(worker_id2)
|
706
|
+
|
707
|
+
# worker completed job -> should send response to notify socket.
|
708
|
+
subject.workers_waiting_for_reply << worker_id # simulating read_command
|
709
|
+
@result = subject.process_command(Command.new(worker_id, {
|
710
|
+
command: 'completed',
|
711
|
+
job_id: @job.id,
|
712
|
+
status: 'error',
|
713
|
+
result: 'foobar'
|
714
|
+
}))
|
715
|
+
end
|
716
|
+
|
717
|
+
|
718
|
+
it "returns when a job times out" do
|
719
|
+
# when the worker completes, the notify socket should be notified that
|
720
|
+
# the job completed.
|
721
|
+
socket2 = double('Broker::Socket')
|
722
|
+
subject.stub(:socket => socket2)
|
723
|
+
socket2.should_receive(:send_command) do |cmd|
|
724
|
+
expect(cmd.worker_id).to eq(worker_id2)
|
725
|
+
expect(cmd.parameters[:status]).to eq('failed')
|
726
|
+
expect(cmd.parameters[:job_id]).to eq(@job.id)
|
727
|
+
end
|
728
|
+
|
729
|
+
expect(subject.workers_waiting_for_reply).to include(worker_id2)
|
730
|
+
|
731
|
+
subject.jobs_in_progress[@job.id].stub(:timed_out? => true)
|
732
|
+
subject.expire_timed_out_jobs
|
733
|
+
end
|
734
|
+
end
|
735
|
+
end
|
736
|
+
|
737
|
+
end
|