rrrspec-server 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,497 @@
1
+ require 'spec_helper'
2
+
3
+ module RRRSpec
4
+ module Server
5
+ describe Arbiter do
6
+ before do
7
+ RRRSpec.configuration = Configuration.new
8
+ RRRSpec.configuration.redis = @redis
9
+ end
10
+
11
+ let(:taskset) do
12
+ Taskset.create(
13
+ 'testuser', 'echo 1', 'echo 2', 'default', 'default', 3, 3, 5, 5
14
+ )
15
+ end
16
+
17
+ let(:task1) do
18
+ Task.create(taskset, 10, 'spec/test_spec.rb')
19
+ end
20
+
21
+ before do
22
+ ActiveTaskset.add(taskset)
23
+ end
24
+
25
+ describe '.cancel' do
26
+ context 'with the taskset running' do
27
+ before { taskset.update_status('running') }
28
+
29
+ it 'cancels the taskset' do
30
+ Arbiter.cancel(taskset)
31
+ expect(taskset.status).to eq('cancelled')
32
+ expect(taskset.queue_empty?).to be_true
33
+ expect(taskset.finished_at).not_to be_nil
34
+ expect(ActiveTaskset.list).not_to include(taskset)
35
+ expect(PersisterQueue.empty?).to be_false
36
+ end
37
+ end
38
+
39
+ context 'with the taskset failed' do
40
+ before { taskset.update_status('failed') }
41
+
42
+ it 'does nothing' do
43
+ Arbiter.cancel(taskset)
44
+ expect(taskset.status).not_to eq('cancelled')
45
+ expect(ActiveTaskset.list).not_to include(taskset)
46
+ expect(PersisterQueue.empty?).to be_false
47
+ end
48
+ end
49
+ end
50
+
51
+ describe '.check' do
52
+ before do
53
+ taskset.add_task(task1)
54
+ taskset.enqueue_task(task1)
55
+ end
56
+
57
+ context 'when the taskset is succeeded or failed or cancelled' do
58
+ before { taskset.update_status('failed') }
59
+ it 'does nothing' do
60
+ Arbiter.check(taskset)
61
+ expect(ActiveTaskset.list).not_to include(taskset)
62
+ expect(PersisterQueue.empty?).to be_false
63
+ end
64
+ end
65
+
66
+ context 'with the taskset running' do
67
+ before { taskset.update_status('running') }
68
+
69
+ context 'with tasks_left having item' do
70
+ it 'calls check_task' do
71
+ expect(Arbiter).to receive(:check_task)
72
+ Arbiter.check(taskset)
73
+ end
74
+ end
75
+
76
+ context 'with tasks_left non-empty after check_task' do
77
+ context 'with the queue empty' do
78
+ before do
79
+ taskset.dequeue_task(0)
80
+ end
81
+ it 'calls requeue_speculative' do
82
+ expect(Arbiter).to receive(:requeue_speculative)
83
+ Arbiter.check(taskset)
84
+ end
85
+ end
86
+
87
+ context 'with the queue non-empty' do
88
+ it 'does nothing' do
89
+ Arbiter.check(taskset)
90
+ expect(taskset.status).to eq('running')
91
+ expect(taskset.finished_at).to be_blank
92
+ expect(ActiveTaskset.list).to include(taskset)
93
+ expect(PersisterQueue.empty?).to be_true
94
+ end
95
+ end
96
+ end
97
+
98
+ context 'with tasks_left empty after check_task' do
99
+ before { taskset.finish_task(task1) }
100
+
101
+ context 'with all tasks succeeded' do
102
+ before { taskset.incr_succeeded_count }
103
+
104
+ it 'marks the taskset as succeeded' do
105
+ Arbiter.check(taskset)
106
+ expect(taskset.status).to eq("succeeded")
107
+ expect(taskset.finished_at).not_to be_blank
108
+ expect(ActiveTaskset.list).not_to include(taskset)
109
+ expect(PersisterQueue.empty?).to be_false
110
+ end
111
+ end
112
+
113
+ context 'with some tasks failed' do
114
+ before { taskset.incr_failed_count }
115
+
116
+ it 'marks the taskset as failed' do
117
+ Arbiter.check(taskset)
118
+ expect(taskset.status).to eq("failed")
119
+ expect(taskset.finished_at).not_to be_nil
120
+ expect(ActiveTaskset.list).not_to include(taskset)
121
+ expect(PersisterQueue.empty?).to be_false
122
+ end
123
+ end
124
+ end
125
+ end
126
+ end
127
+
128
+ describe '.check_task' do
129
+ before do
130
+ taskset.add_task(task1)
131
+ taskset.enqueue_task(task1)
132
+ end
133
+
134
+ let(:logger) { TimedLogger.new(taskset) }
135
+ let(:slave) { Slave.create }
136
+
137
+ context 'with some trials running' do
138
+ let(:trial1) { Trial.create(task1, slave) }
139
+
140
+ before do
141
+ trial1.start
142
+ end
143
+
144
+ context 'with the slave alive' do
145
+ before { slave.heartbeat(30) }
146
+
147
+ it 'does nothing' do
148
+ Arbiter.check_task(logger, taskset, task1)
149
+ expect(trial1.status).to be_blank
150
+ end
151
+ end
152
+
153
+ context 'with the slave failed' do
154
+ it 'marks error' do
155
+ Arbiter.check_task(logger, taskset, task1)
156
+ expect(trial1.status).to eq('error')
157
+ end
158
+ end
159
+ end
160
+
161
+ context 'with a trial passed' do
162
+ let(:trial1) { Trial.create(task1, slave) }
163
+ let(:trial2) { Trial.create(task1, slave) }
164
+
165
+ before do
166
+ trial1.start
167
+ trial1.finish('passed', '', '', nil, nil, nil)
168
+ trial2.start
169
+ trial2.finish('error', '', '', nil, nil, nil)
170
+ end
171
+
172
+ it 'sets status passed' do
173
+ Arbiter.check_task(logger, taskset, task1)
174
+ expect(task1.status).to eq('passed')
175
+ expect(taskset.tasks_left).not_to include(task1)
176
+ expect(taskset.succeeded_count).to eq(1)
177
+ expect(taskset.failed_count).to eq(0)
178
+ end
179
+ end
180
+
181
+ context 'with a trial pending' do
182
+ let(:trial1) { Trial.create(task1, slave) }
183
+ let(:trial2) { Trial.create(task1, slave) }
184
+
185
+ before do
186
+ trial1.start
187
+ trial1.finish('pending', '', '', nil, nil, nil)
188
+ trial2.start
189
+ trial2.finish('error', '', '', nil, nil, nil)
190
+ end
191
+
192
+ it 'sets status pending' do
193
+ Arbiter.check_task(logger, taskset, task1)
194
+ expect(task1.status).to eq('pending')
195
+ expect(taskset.tasks_left).not_to include(task1)
196
+ expect(taskset.succeeded_count).to eq(1)
197
+ expect(taskset.failed_count).to eq(0)
198
+ end
199
+ end
200
+
201
+ context 'with the task fully tried' do
202
+ let(:trial1) { Trial.create(task1, slave) }
203
+ let(:trial2) { Trial.create(task1, slave) }
204
+ let(:trial3) { Trial.create(task1, slave) }
205
+
206
+ before do
207
+ trial1.start
208
+ trial1.finish('error', '', '', nil, nil, nil)
209
+ trial2.start
210
+ trial2.finish('error', '', '', nil, nil, nil)
211
+ trial3.start
212
+ trial3.finish('error', '', '', nil, nil, nil)
213
+ end
214
+
215
+ it 'marks failed' do
216
+ Arbiter.check_task(logger, taskset, task1)
217
+ expect(task1.status).to eq('failed')
218
+ expect(taskset.tasks_left).not_to include(task1)
219
+ expect(taskset.succeeded_count).to eq(0)
220
+ expect(taskset.failed_count).to eq(1)
221
+ end
222
+ end
223
+
224
+ context 'when re-triable' do
225
+ let(:trial1) { Trial.create(task1, slave) }
226
+ let(:trial2) { Trial.create(task1, slave) }
227
+
228
+ before do
229
+ trial1.start
230
+ trial1.finish('error', '', '', nil, nil, nil)
231
+ trial2.start
232
+ trial2.finish('error', '', '', nil, nil, nil)
233
+ end
234
+
235
+ it 'does nothing' do
236
+ Arbiter.check_task(logger, taskset, task1)
237
+ expect(task1.status).to be_blank
238
+ expect(taskset.tasks_left).to include(task1)
239
+ expect(taskset.succeeded_count).to eq(0)
240
+ expect(taskset.failed_count).to eq(0)
241
+ end
242
+ end
243
+ end
244
+
245
+ describe '.requeue_speculative' do
246
+ let(:logger) { TimedLogger.new(taskset) }
247
+ let(:slave) { Slave.create }
248
+
249
+ let(:task2) do
250
+ Task.create(taskset, 10, 'spec/test_spec2.rb')
251
+ end
252
+
253
+ before do
254
+ taskset.add_task(task1)
255
+ taskset.add_task(task2)
256
+ end
257
+
258
+ context 'with some no-running-trial tasks' do
259
+ let(:trial1_1) { Trial.create(task1, slave) }
260
+
261
+ before do
262
+ trial1_1.start
263
+ end
264
+
265
+ it 'enqueues one no-running-trial task' do
266
+ Arbiter.requeue_speculative(logger, taskset, [task1, task2])
267
+ expect(taskset).not_to be_queue_empty
268
+ expect(taskset.dequeue_task(0)).to eq(task2)
269
+ end
270
+ end
271
+
272
+ context 'with all tasks running' do
273
+ let(:trial1_1) { Trial.create(task1, slave) }
274
+ let(:trial1_2) { Trial.create(task2, slave) }
275
+ let(:trial2_2) { Trial.create(task2, slave) }
276
+
277
+ before do
278
+ trial1_1.start
279
+ trial1_2.start
280
+ trial2_2.start
281
+ end
282
+
283
+ it 'enqueues one least tried task' do
284
+ Arbiter.requeue_speculative(logger, taskset, [task1, task2])
285
+ expect(taskset).not_to be_queue_empty
286
+ expect(taskset.dequeue_task(0)).to eq(task1)
287
+ end
288
+ end
289
+ end
290
+
291
+ describe '.fail' do
292
+ context 'with the taskset running' do
293
+ before { taskset.update_status('running') }
294
+
295
+ it 'fails the taskset' do
296
+ Arbiter.fail(taskset)
297
+ expect(taskset.status).to eq('failed')
298
+ expect(taskset.queue_empty?).to be_true
299
+ expect(taskset.finished_at).not_to be_nil
300
+ expect(ActiveTaskset.list).not_to include(taskset)
301
+ expect(PersisterQueue.empty?).to be_false
302
+ end
303
+ end
304
+
305
+ context 'with the taskset cancelled' do
306
+ before { taskset.update_status('cancelled') }
307
+
308
+ it 'does nothing' do
309
+ Arbiter.fail(taskset)
310
+ expect(taskset.status).not_to eq('failed')
311
+ expect(ActiveTaskset.list).not_to include(taskset)
312
+ expect(PersisterQueue.empty?).to be_false
313
+ end
314
+ end
315
+ end
316
+
317
+ describe '.trial' do
318
+ before do
319
+ taskset.add_task(task1)
320
+ taskset.enqueue_task(task1)
321
+ end
322
+
323
+ before do
324
+ taskset.dequeue_task(0)
325
+ end
326
+
327
+ let(:slave) { Slave.create }
328
+ let(:trial) { Trial.create(task1, slave) }
329
+
330
+ context 'when the task is already finished' do
331
+ before { task1.update_status('passed') }
332
+
333
+ it 'does nothing' do
334
+ Arbiter.trial(trial)
335
+ expect(trial.status).to be_nil
336
+ expect(task1.status).to eq('passed')
337
+ end
338
+ end
339
+
340
+ context 'with the trial not-running' do
341
+ context 'with maximally retried' do
342
+ let(:trial2) { Trial.create(task1, slave) }
343
+ let(:trial3) { Trial.create(task1, slave) }
344
+
345
+ before do
346
+ trial2.finish('error', '', '', nil, nil, nil)
347
+ trial3.finish('error', '', '', nil, nil, nil)
348
+ end
349
+
350
+ it 'finishes the trial and marks the task failed' do
351
+ Arbiter.trial(trial)
352
+ expect(trial.status).to eq('error')
353
+ expect(task1.status).to eq('failed')
354
+ expect(taskset.tasks_left).not_to include(task1)
355
+ expect(taskset.queue_empty?).to be_true
356
+ expect(taskset.succeeded_count).to eq(0)
357
+ expect(taskset.failed_count).to eq(1)
358
+ end
359
+ end
360
+
361
+ context 'when re-triable' do
362
+ it 'finishes the trial and requeues the task' do
363
+ Arbiter.trial(trial)
364
+ expect(trial.status).to eq('error')
365
+ expect(task1.status).to be_nil
366
+ expect(taskset.tasks_left).to include(task1)
367
+ expect(taskset.dequeue_task(0)).to eq(task1)
368
+ expect(taskset.succeeded_count).to eq(0)
369
+ expect(taskset.failed_count).to eq(0)
370
+ end
371
+ end
372
+ end
373
+
374
+ context 'with the trial passed' do
375
+ before do
376
+ trial.finish('passed', '', '', nil, nil, nil)
377
+ end
378
+
379
+ it 'marks the task as passed' do
380
+ Arbiter.trial(trial)
381
+ expect(task1.status).to eq('passed')
382
+ expect(taskset.tasks_left).not_to include(task1)
383
+ expect(taskset.queue_empty?).to be_true
384
+ expect(taskset.succeeded_count).to eq(1)
385
+ expect(taskset.failed_count).to eq(0)
386
+ end
387
+ end
388
+
389
+ context 'with the trial pending' do
390
+ before { trial.finish('pending', '', '', 0, 1, 0) }
391
+
392
+ it 'marks the task as pending' do
393
+ Arbiter.trial(trial)
394
+ expect(task1.status).to eq('pending')
395
+ expect(taskset.tasks_left).not_to include(task1)
396
+ expect(taskset.queue_empty?).to be_true
397
+ expect(taskset.succeeded_count).to eq(1)
398
+ expect(taskset.failed_count).to eq(0)
399
+ end
400
+ end
401
+
402
+ context 'with the trial failed' do
403
+ before { trial.finish('failed', '', '', 0, 0, 1) }
404
+
405
+ context 'with another trial running' do
406
+ let(:trial2) { Trial.create(task1, slave) }
407
+
408
+ before { trial2 }
409
+
410
+ context 'maximally retried' do
411
+ let(:trial3) { Trial.create(task1, slave) }
412
+ let(:trial4) { Trial.create(task1, slave) }
413
+
414
+ before do
415
+ trial3
416
+ trial4
417
+ end
418
+
419
+ context 'enough to judge failed' do
420
+ before do
421
+ trial3.finish('error', '', '', nil, nil, nil)
422
+ trial4.finish('error', '', '', nil, nil, nil)
423
+ end
424
+
425
+ it 'finishes the trial and marks the task failed' do
426
+ Arbiter.trial(trial)
427
+ expect(trial.status).to eq('failed')
428
+ expect(task1.status).to eq('failed')
429
+ expect(taskset.tasks_left).not_to include(task1)
430
+ expect(taskset.queue_empty?).to be_true
431
+ expect(taskset.succeeded_count).to eq(0)
432
+ expect(taskset.failed_count).to eq(1)
433
+ end
434
+ end
435
+
436
+ context 'not-enough to judge failed' do
437
+ it 'does nothing' do
438
+ Arbiter.trial(trial)
439
+ expect(task1.status).to be_blank
440
+ expect(taskset.tasks_left).to include(task1)
441
+ expect(taskset.queue_empty?).to be_true
442
+ expect(taskset.succeeded_count).to eq(0)
443
+ expect(taskset.failed_count).to eq(0)
444
+ end
445
+ end
446
+ end
447
+
448
+ context 'retriable' do
449
+ before do
450
+ trial2.finish('error', '', '', nil, nil, nil)
451
+ end
452
+
453
+ it 'requeue the task' do
454
+ Arbiter.trial(trial)
455
+ expect(task1.status).to be_nil
456
+ expect(taskset.tasks_left).to include(task1)
457
+ expect(taskset.dequeue_task(0)).to eq(task1)
458
+ expect(taskset.succeeded_count).to eq(0)
459
+ expect(taskset.failed_count).to eq(0)
460
+ end
461
+ end
462
+ end
463
+
464
+ context 'maximally retried' do
465
+ let(:trial2) { Trial.create(task1, slave) }
466
+ let(:trial3) { Trial.create(task1, slave) }
467
+
468
+ before do
469
+ trial2.finish('error', '', '', nil, nil, nil)
470
+ trial3.finish('error', '', '', nil, nil, nil)
471
+ end
472
+
473
+ it 'marks the task as failed' do
474
+ Arbiter.trial(trial)
475
+ expect(task1.status).to eq('failed')
476
+ expect(taskset.tasks_left).not_to include(task1)
477
+ expect(taskset.queue_empty?).to be_true
478
+ expect(taskset.succeeded_count).to eq(0)
479
+ expect(taskset.failed_count).to eq(1)
480
+ end
481
+ end
482
+
483
+ context 'when re-triable' do
484
+ it 'requeue the task' do
485
+ Arbiter.trial(trial)
486
+ expect(task1.status).to be_nil
487
+ expect(taskset.tasks_left).to include(task1)
488
+ expect(taskset.dequeue_task(0)).to eq(task1)
489
+ expect(taskset.succeeded_count).to eq(0)
490
+ expect(taskset.failed_count).to eq(0)
491
+ end
492
+ end
493
+ end
494
+ end
495
+ end
496
+ end
497
+ end
@@ -0,0 +1,53 @@
1
+ require 'spec_helper'
2
+
3
+ module RRRSpec
4
+ module Server
5
+ describe Dispatcher do
6
+ before do
7
+ RRRSpec.configuration = Configuration.new
8
+ RRRSpec.configuration.redis = @redis
9
+ end
10
+
11
+ describe '#work' do
12
+ context 'when the worker no longer exists' do
13
+ let(:worker) { Worker.create('default', 'hostname1') }
14
+ before { worker }
15
+
16
+ it 'evicts the worker' do
17
+ Dispatcher.work
18
+ expect(Worker.list).not_to include(worker)
19
+ end
20
+ end
21
+
22
+ context 'a worker exists' do
23
+ let(:taskset) do
24
+ Taskset.create(
25
+ 'testuser', 'echo 1', 'echo 2', 'default', 'default', 3, 3, 5, 5
26
+ )
27
+ end
28
+
29
+ let(:worker1) { Worker.create('default', 'hostname1') }
30
+ let(:worker2) { Worker.create('default', 'hostname2') }
31
+ let(:worker3) { Worker.create('default', 'hostname3') }
32
+ let(:worker4) { Worker.create('default', 'hostname4') }
33
+
34
+ before do
35
+ ActiveTaskset.add(taskset)
36
+ worker1.update_current_taskset(taskset)
37
+ worker1.heartbeat(30)
38
+ worker2.heartbeat(30)
39
+ worker3.heartbeat(30)
40
+ worker4.heartbeat(30)
41
+ end
42
+
43
+ it 'assignes worker upto the max_workers' do
44
+ Dispatcher.work
45
+ expect(worker1.queue_empty?).to be_true
46
+ workers = [worker1, worker2, worker3, worker4]
47
+ expect(workers.count { |worker| worker.queue_empty? }).to eq(2)
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,54 @@
1
+ require 'spec_helper'
2
+
3
+ module RRRSpec
4
+ module Server
5
+ module Persistence
6
+ describe Taskset do
7
+ before do
8
+ RRRSpec.configuration = RRRSpec::Configuration.new
9
+ RRRSpec.configuration.redis = @redis
10
+ end
11
+
12
+ before do
13
+ @worker, @taskset, @task, @worker_log, @slave, @trial =
14
+ RRRSpec.finished_fullset
15
+ Persister.persist(@taskset)
16
+ end
17
+
18
+ describe '#as_nodetail_json' do
19
+ it 'produces a json object' do
20
+ h = @taskset.to_h
21
+ h.delete('slaves')
22
+ h.delete('tasks')
23
+ h.delete('worker_logs')
24
+
25
+ expect(
26
+ JSON.parse(JSON.generate(Taskset.first.as_nodetail_json()))
27
+ ).to eq(
28
+ JSON.parse(JSON.generate(h))
29
+ )
30
+ end
31
+ end
32
+
33
+ describe '#as_full_json' do
34
+ it 'produces a json object' do
35
+ h = @taskset.to_h
36
+ h['slaves'] = [@slave.to_h]
37
+
38
+ task_h = @task.to_h
39
+ task_h['trials'] = [@trial.to_h]
40
+ h['tasks'] = [task_h]
41
+
42
+ h['worker_logs'] = [@worker_log.to_h]
43
+
44
+ expect(
45
+ JSON.parse(JSON.generate(Taskset.first.as_full_json()))
46
+ ).to eq(
47
+ JSON.parse(JSON.generate(h))
48
+ )
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end