job_dispatch 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +19 -0
  3. data/.rspec +1 -0
  4. data/.travis.yml +13 -0
  5. data/Gemfile +20 -0
  6. data/Guardfile +13 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +85 -0
  9. data/Rakefile +10 -0
  10. data/bin/job-dispatcher +34 -0
  11. data/bin/job-status +69 -0
  12. data/bin/job-worker +40 -0
  13. data/examples/mongoid-job.rb +43 -0
  14. data/job_dispatch.gemspec +33 -0
  15. data/lib/job_dispatch/broker/command.rb +45 -0
  16. data/lib/job_dispatch/broker/internal_job.rb +32 -0
  17. data/lib/job_dispatch/broker/socket.rb +85 -0
  18. data/lib/job_dispatch/broker.rb +523 -0
  19. data/lib/job_dispatch/client/proxy.rb +34 -0
  20. data/lib/job_dispatch/client/proxy_error.rb +18 -0
  21. data/lib/job_dispatch/client/synchronous_proxy.rb +29 -0
  22. data/lib/job_dispatch/client.rb +49 -0
  23. data/lib/job_dispatch/configuration.rb +7 -0
  24. data/lib/job_dispatch/identity.rb +54 -0
  25. data/lib/job_dispatch/job.rb +44 -0
  26. data/lib/job_dispatch/signaller.rb +30 -0
  27. data/lib/job_dispatch/sockets/enqueue.rb +18 -0
  28. data/lib/job_dispatch/status.rb +79 -0
  29. data/lib/job_dispatch/version.rb +3 -0
  30. data/lib/job_dispatch/worker/item.rb +43 -0
  31. data/lib/job_dispatch/worker/socket.rb +96 -0
  32. data/lib/job_dispatch/worker.rb +120 -0
  33. data/lib/job_dispatch.rb +97 -0
  34. data/spec/factories/jobs.rb +19 -0
  35. data/spec/job_dispatch/broker/socket_spec.rb +53 -0
  36. data/spec/job_dispatch/broker_spec.rb +737 -0
  37. data/spec/job_dispatch/identity_spec.rb +88 -0
  38. data/spec/job_dispatch/job_spec.rb +77 -0
  39. data/spec/job_dispatch/worker/socket_spec.rb +32 -0
  40. data/spec/job_dispatch/worker_spec.rb +24 -0
  41. data/spec/job_dispatch_spec.rb +0 -0
  42. data/spec/spec_helper.rb +23 -0
  43. data/spec/support/test_job.rb +30 -0
  44. metadata +255 -0
@@ -0,0 +1,737 @@
1
+ require 'rspec'
2
+ require 'spec_helper'
3
+
4
+ # dummy Job class
5
+ Job = TestJob
6
+
7
+ describe JobDispatch::Broker do
8
+
9
+ Command ||= JobDispatch::Broker::Command
10
+ Identity ||= JobDispatch::Identity
11
+
12
+ subject { JobDispatch::Broker.new('tcp://localhost:2000', 'tcp://localhost:2001') }
13
+
14
+ let(:worker_id) { Identity.new([0, 0x80, 0, 0x41, 0x31].pack('c*')) }
15
+ let(:worker_id2) { Identity.new([0, 0x80, 0, 0x41, 0x32].pack('c*')) }
16
+ let(:worker_id3) { Identity.new([0, 0x80, 0, 0x41, 0x33].pack('c*')) }
17
+
18
+ before :each do
19
+ JobDispatch.config.job_class = double('JobClass')
20
+ end
21
+
22
+ context "tracking communication state" do
23
+
24
+ it "reading a command adds the requester to the list of connections awaiting reply" do
25
+ command = Command.new(worker_id, {commmand: 'ready'})
26
+ socket = double('Broker::Socket', :read_command => command)
27
+ subject.stub(:socket => socket)
28
+ subject.read_command
29
+ expect(subject.workers_waiting_for_reply).to include(worker_id)
30
+ end
31
+
32
+ it "sending a command removes the worker from the list of workers awaiting reply" do
33
+ subject.workers_waiting_for_reply << worker_id
34
+ command = Command.new(worker_id, {commmand: 'idle'})
35
+ socket = double('Broker::Socket', :send_command => command)
36
+ socket.should_receive(:send_command).with(command)
37
+ subject.stub(:socket => socket)
38
+ subject.send_command(command)
39
+ expect(subject.workers_waiting_for_reply).not_to include(worker_id)
40
+ end
41
+
42
+ it "does not send a command to a worker unless it is awaiting a reply" do
43
+ command = Command.new(worker_id, {commmand: 'idle'})
44
+ socket = double('Broker::Socket', :send_command => command)
45
+ socket.should_not_receive(:send_command)
46
+ subject.stub(:socket => socket)
47
+ expect { subject.send_command(command) }.to raise_error
48
+ end
49
+ end
50
+
51
+ context "responding to command" do
52
+
53
+ context "'status'" do
54
+ let(:command) { Command.new(worker_id, {command: 'status'}) }
55
+
56
+ before :each do
57
+ subject.reply_exceptions = false
58
+ end
59
+
60
+ it "returns a command" do
61
+ expect(subject.process_command(command)).to be_a(Command)
62
+ end
63
+
64
+ it "returns a status object" do
65
+ result = subject.process_command(command)
66
+ expect(result.worker_id).to eq(worker_id)
67
+ expect(result.parameters).to be_a(Hash)
68
+ expect(result.parameters[:status]).to eq('OK')
69
+ expect(result.parameters[:queues]).to be_a(Hash)
70
+ end
71
+
72
+ it "returns a JSONable parameters object" do
73
+ subject.process_command(Command.new(worker_id2, {command: 'ready', worker_name: 'test worker'}))
74
+ result = subject.process_command(command)
75
+ expect { json = JSON.dump(result.parameters) }.not_to raise_error
76
+ end
77
+
78
+ it "returns a list of workers including idle and working" do
79
+ subject.workers_waiting_for_reply << worker_id2
80
+ subject.process_command(Command.new(worker_id2, {command: 'ready', worker_name: 'test worker 1'}))
81
+ subject.workers_waiting_for_reply << worker_id3
82
+ subject.process_command(Command.new(worker_id3, {command: 'ready', worker_name: 'test worker 2'}))
83
+
84
+
85
+ @job = FactoryGirl.build :job
86
+ @socket = double('Broker::Socket', :send_command => nil)
87
+ subject.stub(:socket => @socket)
88
+ @socket.should_receive(:send_command) do |cmd|
89
+ #expect(cmd.worker_id).to eq(worker_id)
90
+ expect(cmd.parameters[:command]).to eq('job')
91
+ expect(cmd.parameters[:target]).to eq(@job.target)
92
+ end
93
+
94
+ job_class = double('JobClass')
95
+ job_class.stub(:dequeue_job_for_queue).and_return(@job)
96
+ JobDispatch.config.job_class = job_class
97
+
98
+ # dispatch a job to a worker
99
+ subject.dispatch_jobs_to_workers
100
+
101
+ # now get status
102
+ result = subject.process_command(command)
103
+
104
+ expect(result.parameters[:queues]).to be_a(Hash)
105
+ expect(result.parameters[:queues].size).to eq(1)
106
+ expect(result.parameters[:queues][:default]).to be_a(Hash)
107
+ expect(result.parameters[:queues][:default][worker_id2.to_hex]).to be_a(Hash)
108
+ expect(result.parameters[:queues][:default][worker_id2.to_hex][:status]).to eq(:processing)
109
+ expect(result.parameters[:queues][:default][worker_id2.to_hex][:job_id]).to eq(@job.id)
110
+ expect(result.parameters[:queues][:default][worker_id2.to_hex][:name]).to eq('test worker 1')
111
+ expect(result.parameters[:queues][:default][worker_id3.to_hex]).to be_a(Hash)
112
+ expect(result.parameters[:queues][:default][worker_id3.to_hex][:status]).to eq(:idle)
113
+ expect(result.parameters[:queues][:default][worker_id3.to_hex][:name]).to eq('test worker 2')
114
+ end
115
+ end
116
+
117
+ context "'quit'" do
118
+ let(:command) { Command.new(worker_id, {command: 'quit'}) }
119
+
120
+ it "returns a command" do
121
+ @result = subject.process_command(command)
122
+ expect(@result).to be_a(Command)
123
+ end
124
+
125
+ it "acknowledges the command" do
126
+ @result = subject.process_command(command)
127
+ expect(@result.parameters[:status]).to eq("bye")
128
+ end
129
+
130
+ it "sets running to false" do
131
+ @result = subject.process_command(command)
132
+ expect(subject.running?).to be_false
133
+ end
134
+
135
+ it "sends a quit message to a waiting worker" do
136
+ socket = double('Broker::Socket', :send_command => command)
137
+ socket.should_receive(:send_command) do |cmd|
138
+ expect(cmd.worker_id).to eq(worker_id)
139
+ end
140
+ subject.workers_waiting_for_reply << worker_id
141
+ subject.process_command(Command.new(worker_id, command: 'ready'))
142
+ subject.stub(:socket => socket)
143
+ @result = subject.process_command(command)
144
+ end
145
+ end
146
+
147
+ # when a worker is ready for work!
148
+ context "'ready'" do
149
+ let(:command) { Command.new(worker_id, {command: 'ready', queue: 'example', worker_name: 'ruby worker'}) }
150
+ it "returns nil" do
151
+ @result = subject.process_command(command)
152
+ expect(@result).to be_nil
153
+ end
154
+
155
+ it "adds the worker to the list of workers awaiting replies" do
156
+ @result = subject.process_command(command)
157
+ expect(subject.workers_waiting_for_jobs.keys).to include(worker_id)
158
+ end
159
+
160
+ it "adds the worker to the queue" do
161
+ @result = subject.process_command(command)
162
+ expect(subject.queues[:example]).to include(worker_id)
163
+ end
164
+
165
+ it "adds the worker's name to the hash of worker names" do
166
+ @result = subject.process_command(command)
167
+ expect(subject.worker_names[worker_id]).to eq('ruby worker')
168
+ end
169
+ end
170
+
171
+
172
+ context "'goodbye'" do
173
+ # the goodbye command is sent from a new socket, so the actual socket identity will be different from
174
+ # the socket waiting for a job or idle command.
175
+ let(:worker_name) { 'ruby worker' }
176
+ let(:goodbye_command) { Command.new(worker_id2, {command: 'goodbye', worker_name: worker_name}) }
177
+ let(:ready_command) { Command.new(worker_id, {command: 'ready', queue: 'example', worker_name: worker_name}) }
178
+ context 'with idle worker' do
179
+ before :each do
180
+ subject.process_command(ready_command)
181
+ end
182
+
183
+ it "returns an object" do
184
+ @result = subject.process_command(goodbye_command)
185
+ expect(@result).to be_a(Command)
186
+ expect(@result.parameters).to be_a(Hash)
187
+ end
188
+
189
+ it "removes the worker's name from the worker_name hash" do
190
+ @result = subject.process_command(goodbye_command)
191
+ expect(subject.worker_names).not_to include(worker_id)
192
+ end
193
+
194
+ it "removes the worker from the list waiting reply" do
195
+ subject.process_command(goodbye_command)
196
+ expect(subject.workers_waiting_for_reply).not_to include(worker_id)
197
+ end
198
+
199
+ it "removes the worker from the list of workers ready for jobs" do
200
+ subject.process_command(goodbye_command)
201
+ expect(subject.workers_waiting_for_jobs).not_to include(worker_id)
202
+ end
203
+ end
204
+
205
+ context 'without an idle worker' do
206
+ it "returns an object" do
207
+ @result = subject.process_command(goodbye_command)
208
+ expect(@result).to be_a(Command)
209
+ expect(@result.parameters).to be_a(Hash)
210
+ end
211
+
212
+ it "removes the worker's name from the worker_name hash" do
213
+ @result = subject.process_command(goodbye_command)
214
+ expect(subject.worker_names).not_to include(worker_id)
215
+ end
216
+
217
+ it "removes the worker from the list waiting reply" do
218
+ subject.process_command(goodbye_command)
219
+ expect(subject.workers_waiting_for_reply).not_to include(worker_id)
220
+ end
221
+
222
+ it "removes the worker from the list of workers ready for jobs" do
223
+ subject.process_command(goodbye_command)
224
+ expect(subject.workers_waiting_for_jobs).not_to include(worker_id)
225
+ end
226
+ end
227
+ end
228
+
229
+ # a worker has completed a task and is optionally asking for another one.
230
+ context "'completed'" do
231
+ let(:job_id) { '1234' }
232
+ before :each do
233
+ @job = FactoryGirl.build :job, :id => job_id
234
+ @job.stub(:succeeded!) do
235
+ @job.status = JobDispatch::Job::COMPLETED
236
+ true
237
+ end
238
+ @job.stub(:failed!) do
239
+ @job.status = JobDispatch::Job::FAILED
240
+ true
241
+ end
242
+ @job.stub(:reload => true)
243
+ subject.jobs_in_progress[job_id] = @job
244
+ JobDispatch.config.job_class.stub(:find) do |id|
245
+ raise StandardError, "Job not found" if id != @job.id
246
+ @job
247
+ end
248
+ end
249
+
250
+ context "and ask for another job" do
251
+ let(:command) { Command.new(worker_id, {
252
+ command: 'completed',
253
+ job_id: job_id,
254
+ status: 'success',
255
+ result: 'the result',
256
+ ready: true,
257
+ queue: 'example'
258
+ }) }
259
+
260
+ it "returns nil" do
261
+ @result = subject.process_command(command)
262
+ expect(@result).to be_nil
263
+ end
264
+
265
+ it "adds the worker to the list of workers awaiting replies" do
266
+ @result = subject.process_command(command)
267
+ expect(subject.workers_waiting_for_jobs.keys).to include(worker_id)
268
+ end
269
+
270
+ it "adds the worker to the queue" do
271
+ @result = subject.process_command(command)
272
+ expect(subject.queues[:example]).to include(worker_id)
273
+ end
274
+
275
+ it "marks the job as succeeded" do
276
+ @job.should_receive(:succeeded!).with('the result')
277
+ subject.process_command(command)
278
+ end
279
+ end
280
+
281
+ context 'and not asking for another job' do
282
+ let(:command) { Command.new(worker_id, {
283
+ command: 'completed',
284
+ job_id: job_id,
285
+ status: 'success',
286
+ result: 'the result'
287
+ }) }
288
+
289
+ it "returns thanks" do
290
+ @result = subject.process_command(command)
291
+ expect(@result).to be_a(Command)
292
+ expect(@result.parameters[:status]).to eq('thanks')
293
+ end
294
+
295
+ it "adds the worker to the list of workers awaiting replies" do
296
+ @result = subject.process_command(command)
297
+ expect(subject.workers_waiting_for_jobs.keys).not_to include(worker_id)
298
+ end
299
+
300
+ it "adds the worker to the queue" do
301
+ @result = subject.process_command(command)
302
+ expect(subject.queues[:example]).not_to include(worker_id)
303
+ end
304
+
305
+ it "marks the job as succeeded" do
306
+ @job.should_receive(:succeeded!).with('the result')
307
+ subject.process_command(command)
308
+ end
309
+ end
310
+
311
+ context 'when the job fails, it is marked as failed' do
312
+ let(:command) { Command.new(worker_id, {
313
+ command: 'completed',
314
+ job_id: job_id,
315
+ status: 'error',
316
+ result: 'the error message'
317
+ }) }
318
+
319
+ it "marks the job as failed" do
320
+ @job.should_receive(:failed!).with('the error message')
321
+ subject.process_command(command)
322
+ end
323
+ end
324
+
325
+ context "when the job doesn't exist" do
326
+ let(:command) { Command.new(worker_id, {
327
+ command: 'completed',
328
+ job_id: 'wrong',
329
+ status: 'success',
330
+ result: 'the result'
331
+ }) }
332
+
333
+ it "returns 'thanks' anyway" do
334
+ JobDispatch.config.job_class.stub(:find).and_raise(StandardError, "Job not found")
335
+ @result = subject.process_command(command)
336
+ expect(@result).to be_a(Command)
337
+ expect(@result.parameters[:status]).to eq('thanks')
338
+ end
339
+ end
340
+
341
+
342
+ # This context is for when a job is being executed by a worker and the timeout has been reached.
343
+ # The broker will have purged the job from the list of jobs in progress, and the worker will be
344
+ # offline since it has not yet asked for more work to do. If the job completes successfully, the
345
+ # broker needs to update the status of the job to completed (or failed) appropriately, which will
346
+ # override any retry. In practice, long jobs should use the "touch" command to let the broker
347
+ # know that they are in fact still working on the job and to please extend the timeout deadline.
348
+ context "after the job has timed out" do
349
+ let(:command) { Command.new(worker_id, {
350
+ command: 'completed',
351
+ job_id: @job.id,
352
+ status: 'success',
353
+ result: 'the result',
354
+ ready: true,
355
+ queue: 'example'
356
+ }) }
357
+
358
+ before :each do
359
+ @job.stub(:as_json).and_return({status: @job.status, result: @job.result, id: @job.id})
360
+ @job.status = 1 # mark as in progress
361
+ JobDispatch.config.job_class.stub(:dequeue_job_for_queue).and_return(@job)
362
+
363
+ # simulate the job has expired:
364
+ @job.stub(:timed_out? => true)
365
+ subject.expire_timed_out_jobs
366
+ end
367
+
368
+ it "Updates the job status to be completed" do
369
+ subject.process_command(command)
370
+ expect(@job.status).to eq(JobDispatch::Job::COMPLETED)
371
+ end
372
+
373
+ it "adds the worker to the queue" do
374
+ subject.process_command(command)
375
+ expect(subject.queues[:example]).to include(worker_id)
376
+ end
377
+
378
+ it "marks the job as succeeded" do
379
+ @job.should_receive(:succeeded!).with('the result')
380
+ subject.process_command(command)
381
+ end
382
+
383
+ end
384
+ end
385
+
386
+ end
387
+
388
+
389
+ context "idle workers" do
390
+
391
+ before :each do
392
+ @socket = double('Broker::Socket', :send_command => nil)
393
+ subject.stub(:socket => @socket)
394
+
395
+ @time = Time.now
396
+ # this worker will be IDLE
397
+ Timecop.freeze(@time) do
398
+ subject.workers_waiting_for_reply << worker_id # ugly: simulating a prior read_command (implementation detail!)
399
+ command = Command.new(worker_id, {command: 'ready', queue: 'example'})
400
+ @result = subject.process_command(command)
401
+ end
402
+ # this worker should stay in the queue
403
+ Timecop.freeze(@time + 5) do
404
+ subject.workers_waiting_for_reply << worker_id2 # ugly: simulating a prior read_command (implementation detail!)
405
+ command = Command.new(worker_id2, {command: 'ready', queue: 'example'})
406
+ @result = subject.process_command(command)
407
+ end
408
+
409
+ end
410
+
411
+ it "that have waited long enough receive idle commands" do
412
+ @socket.should_receive(:send_command) do |cmd|
413
+ expect(cmd.worker_id).to eq(worker_id)
414
+ expect(cmd.parameters[:command]).to eq('idle')
415
+ end
416
+
417
+ Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
418
+ subject.send_idle_commands
419
+ end
420
+
421
+ expect(subject.workers_waiting_for_reply).not_to include(worker_id)
422
+ expect(subject.queues[:example]).not_to include(worker_id)
423
+ end
424
+
425
+ it "that have not waited long enough are still waiting" do
426
+ @socket.should_receive(:send_command) do |cmd|
427
+ expect(cmd.worker_id).not_to eq(worker_id2)
428
+ end
429
+
430
+ Timecop.freeze(@time + JobDispatch::Broker::WORKER_IDLE_TIME + 1) do
431
+ subject.send_idle_commands
432
+ end
433
+
434
+ expect(subject.workers_waiting_for_reply).to include(worker_id2)
435
+ expect(subject.queues[:example]).to include(worker_id2)
436
+ end
437
+ end
438
+
439
+
440
+ context "dispatching jobs" do
441
+ context "when there are jobs in a queue" do
442
+ before :each do
443
+ @job = FactoryGirl.build :job
444
+ @socket = double('Broker::Socket', :send_command => nil)
445
+ subject.stub(:socket => @socket)
446
+ end
447
+
448
+ it "the job is sent to an idle worker" do
449
+ @socket.should_receive(:send_command) do |cmd|
450
+ expect(cmd.worker_id).to eq(worker_id)
451
+ expect(cmd.parameters[:command]).to eq('job')
452
+ expect(cmd.parameters[:target]).to eq(@job.target)
453
+ end
454
+
455
+ job_class = double('JobClass')
456
+ job_class.stub(:dequeue_job_for_queue).and_return(@job)
457
+ job_class.should_receive(:dequeue_job_for_queue).with('example')
458
+ JobDispatch.config.job_class = job_class
459
+
460
+ # send ready command => adds idle worker state
461
+ subject.workers_waiting_for_reply << worker_id # simulating read_command
462
+ @result = subject.process_command(Command.new(worker_id, {
463
+ command: 'ready',
464
+ queue: 'example',
465
+ worker_name: 'ruby worker',
466
+ }))
467
+ expect(@result).to be_nil # no immediate response
468
+ expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
469
+
470
+ subject.dispatch_jobs_to_workers
471
+ end
472
+ end
473
+
474
+ context "when an error occurs dequeuing jobs" do
475
+ before :each do
476
+ @job_class = double('JobClass')
477
+ @job_class.stub(:dequeue_job_for_queue).and_raise(StandardError, "something bad happened")
478
+ JobDispatch.config.job_class = @job_class
479
+ end
480
+
481
+ it "behaves as if there was no job" do
482
+ # send ready command => adds idle worker state
483
+ subject.workers_waiting_for_reply << worker_id # simulating read_command
484
+ @result = subject.process_command(Command.new(worker_id, {
485
+ command: 'ready',
486
+ queue: 'example',
487
+ worker_name: 'ruby worker',
488
+ }))
489
+ expect(@result).to be_nil # no immediate response
490
+ expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
491
+
492
+ # no job should be sent
493
+ subject.should_not_receive(:send_job_to_worker)
494
+ expect { subject.dispatch_jobs_to_workers }.not_to raise_error
495
+ end
496
+ end
497
+ end
498
+
499
+ context "expired jobs" do
500
+ it "are removed from jobs list when they expire" do
501
+ time = Time.now
502
+ @job = FactoryGirl.build :job, :expire_execution_at => time - 5.seconds
503
+ @job_id = @job.id.to_s
504
+ subject.jobs_in_progress[@job_id] = @job
505
+ subject.jobs_in_progress_workers[@job_id] = worker_id
506
+ subject.expire_timed_out_jobs
507
+ expect(subject.jobs_in_progress).to be_empty
508
+ expect(subject.jobs_in_progress_workers).to be_empty
509
+ end
510
+
511
+
512
+ it "include InternalJob commands" do
513
+ socket = double('Broker::Socket')
514
+ subject.stub(:socket => socket)
515
+ socket.stub(:send_command => true)
516
+
517
+ # send ready command => adds idle worker state
518
+ subject.workers_waiting_for_reply << worker_id # simulating read_command
519
+ @result = subject.process_command(Command.new(worker_id, {
520
+ command: 'ready',
521
+ queue: 'example',
522
+ worker_name: 'ruby worker',
523
+ }))
524
+ subject.send_idle_commands(Time.now + JobDispatch::Broker::WORKER_IDLE_TIME + 10)
525
+ expect(subject.jobs_in_progress_workers.length).to eq(1)
526
+ @time = Time.now + JobDispatch::Broker::WORKER_IDLE_TIME + 10 + JobDispatch::Job::DEFAULT_EXECUTION_TIMEOUT + 10
527
+ JobDispatch::Broker::InternalJob.any_instance.should_not_receive(:failed!)
528
+ Timecop.freeze(@time) do
529
+ subject.expire_timed_out_jobs
530
+ end
531
+ end
532
+ end
533
+
534
+ context "touching a job" do
535
+ before :each do
536
+ @time = Time.now
537
+ # this worker will be IDLE
538
+ @job = FactoryGirl.build :job, :expire_execution_at => @time + 5.seconds
539
+ @job_id = @job.id.to_s
540
+ subject.jobs_in_progress[@job_id] = @job
541
+ subject.jobs_in_progress_workers[@job_id] = worker_id
542
+ @socket = double('Broker::Socket', :send_command => nil)
543
+ subject.stub(:socket => @socket)
544
+ @socket.stub(:read_command).and_return(nil)
545
+ @job.stub(:save)
546
+ end
547
+
548
+ it "updates the expires_execute_at time" do
549
+ Timecop.freeze(@time) do
550
+ subject.touch_job(Command.new(worker_id, {command: "touch", job_id: @job_id}))
551
+ end
552
+ expect(@job.expire_execution_at).to eq(@time + JobDispatch::Job::DEFAULT_EXECUTION_TIMEOUT)
553
+ end
554
+
555
+ it "updates the expire_execution_at time with a custom timeout" do
556
+ Timecop.freeze(@time) do
557
+ subject.touch_job(Command.new(worker_id, {command: "touch", job_id: @job_id, timeout: 100}))
558
+ end
559
+ expect(@job.expire_execution_at).to eq(@time + 100.seconds)
560
+ end
561
+ end
562
+
563
+ context "enqueue a job" do
564
+ before :each do
565
+ @job_attrs = FactoryGirl.attributes_for :job
566
+ end
567
+
568
+ it "Creates a job" do
569
+ JobDispatch.config.job_class = double('JobClass')
570
+ JobDispatch.config.job_class.should_receive(:create!).with(@job_attrs)
571
+ command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
572
+ subject.process_command(command)
573
+ end
574
+
575
+ it "returns the job id" do
576
+ job_id = 12345
577
+ JobDispatch.config.job_class = double('JobClass')
578
+ JobDispatch.config.job_class.stub(:create! => double('Job', :id => job_id))
579
+ command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
580
+ result = subject.process_command(command)
581
+ expect(result.parameters[:job_id]).to eq(job_id.to_s)
582
+ job_id
583
+ end
584
+
585
+ it "returns an error if the arguments are no good" do
586
+ JobDispatch.config.job_class = double('JobClass')
587
+ JobDispatch.config.job_class.stub(:create!).and_raise("no good") # simulate some database save error
588
+ command = Command.new(:some_client, {command: "enqueue", job: @job_attrs})
589
+ result = subject.process_command(command)
590
+ expect(result.parameters[:status]).to eq('error')
591
+ expect(result.parameters[:message]).to eq('no good')
592
+ end
593
+
594
+ it "returns an error if the 'job' parameter is missing" do
595
+ JobDispatch.config.job_class = double('JobClass')
596
+ command = Command.new(:some_client, {command: "enqueue"})
597
+ result = subject.process_command(command)
598
+ expect(result.parameters[:status]).to eq('error')
599
+ end
600
+ end
601
+
602
+
603
+ context "'notify'" do
604
+
605
+ before :each do
606
+ @job = FactoryGirl.build :job
607
+ @job_class = double('JobClass')
608
+ @job_class.stub(:dequeue_job_for_queue).and_return(@job)
609
+ @job_class.stub(:find) do |id|
610
+ @job if id == @job.id
611
+ end
612
+ JobDispatch.config.job_class = @job_class
613
+ end
614
+
615
+ context "with no jobs" do
616
+ it "returns no such job" do
617
+ command = Command.new(:client, {command: 'notify', job_id: 1234})
618
+ @job_class.should_receive(:find).with(1234).and_raise(StandardError, "bozo")
619
+ result = subject.process_command(command)
620
+ expect(result.parameters[:status]).to eq('error')
621
+ expect(result.parameters[:message]).to eq('bozo')
622
+ end
623
+ end
624
+
625
+ context "with a completed job" do
626
+ it "returns the job result" do
627
+ @job = FactoryGirl.build :job, status: JobDispatch::Job::COMPLETED, result: 'hooray'
628
+ command = Command.new(:client, {command: 'notify', job_id: @job.id})
629
+ @job_class.should_receive(:find).with(@job.id).and_return(@job)
630
+ result = subject.process_command(command)
631
+ expect(result.parameters[:status]).to eq('completed')
632
+ end
633
+ end
634
+
635
+ context "with a job in progress" do
636
+ before :each do
637
+ @socket = double('Broker::Socket')
638
+ subject.stub(:socket => @socket)
639
+ @socket.stub(:send_command => nil)
640
+
641
+ # worker ready command
642
+
643
+ subject.workers_waiting_for_reply << worker_id # simulating read_command
644
+ @result = subject.process_command(Command.new(worker_id, {
645
+ command: 'ready',
646
+ queue: 'example',
647
+ worker_name: 'ruby worker',
648
+ }))
649
+ expect(@result).to be_nil # no immediate response
650
+ expect(subject.workers_waiting_for_jobs[worker_id]).not_to be_nil
651
+
652
+ # dispatch job to worker
653
+
654
+ subject.dispatch_jobs_to_workers
655
+
656
+ # send notify command
657
+
658
+ subject.workers_waiting_for_reply << worker_id2 # simulating read_command
659
+ @result = subject.process_command(Command.new(worker_id2, {
660
+ command: 'notify',
661
+ job_id: @job.id
662
+ }))
663
+ end
664
+
665
+ it "registers the job subscriber" do
666
+ expect(@result).to be_nil # no immediate response
667
+ expect(subject.job_subscribers[@job.id]).to include(worker_id2)
668
+ expect(subject.workers_waiting_for_reply).to include(worker_id2)
669
+ end
670
+
671
+ it "returns when the job completes" do
672
+ # when the worker completes, the notify socket should be notified that
673
+ # the job completed.
674
+ socket2 = double('Broker::Socket')
675
+ subject.stub(:socket => socket2)
676
+ socket2.should_receive(:send_command) do |cmd|
677
+ expect(cmd.worker_id).to eq(worker_id2)
678
+ expect(cmd.parameters[:status]).to eq('completed')
679
+ expect(cmd.parameters[:job_id]).to eq(@job.id)
680
+ end
681
+
682
+ expect(subject.workers_waiting_for_reply).to include(worker_id2)
683
+
684
+ # worker completed job -> should send response to notify socket.
685
+ subject.workers_waiting_for_reply << worker_id # simulating read_command
686
+ @result = subject.process_command(Command.new(worker_id, {
687
+ command: 'completed',
688
+ job_id: @job.id,
689
+ status: 'success',
690
+ result: 'foobar'
691
+ }))
692
+ end
693
+
694
+ it "returns when a job fails" do
695
+ # when the worker completes, the notify socket should be notified that
696
+ # the job completed.
697
+ socket2 = double('Broker::Socket')
698
+ subject.stub(:socket => socket2)
699
+ socket2.should_receive(:send_command) do |cmd|
700
+ expect(cmd.worker_id).to eq(worker_id2)
701
+ expect(cmd.parameters[:status]).to eq('failed')
702
+ expect(cmd.parameters[:job_id]).to eq(@job.id)
703
+ end
704
+
705
+ expect(subject.workers_waiting_for_reply).to include(worker_id2)
706
+
707
+ # worker completed job -> should send response to notify socket.
708
+ subject.workers_waiting_for_reply << worker_id # simulating read_command
709
+ @result = subject.process_command(Command.new(worker_id, {
710
+ command: 'completed',
711
+ job_id: @job.id,
712
+ status: 'error',
713
+ result: 'foobar'
714
+ }))
715
+ end
716
+
717
+
718
+ it "returns when a job times out" do
719
+ # when the worker completes, the notify socket should be notified that
720
+ # the job completed.
721
+ socket2 = double('Broker::Socket')
722
+ subject.stub(:socket => socket2)
723
+ socket2.should_receive(:send_command) do |cmd|
724
+ expect(cmd.worker_id).to eq(worker_id2)
725
+ expect(cmd.parameters[:status]).to eq('failed')
726
+ expect(cmd.parameters[:job_id]).to eq(@job.id)
727
+ end
728
+
729
+ expect(subject.workers_waiting_for_reply).to include(worker_id2)
730
+
731
+ subject.jobs_in_progress[@job.id].stub(:timed_out? => true)
732
+ subject.expire_timed_out_jobs
733
+ end
734
+ end
735
+ end
736
+
737
+ end