sidejob 4.0.2 → 4.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +38 -15
- data/lib/sidejob.rb +82 -26
- data/lib/sidejob/job.rb +121 -33
- data/lib/sidejob/port.rb +153 -50
- data/lib/sidejob/server_middleware.rb +16 -4
- data/lib/sidejob/testing.rb +6 -13
- data/lib/sidejob/version.rb +1 -1
- data/lib/sidejob/worker.rb +5 -5
- data/spec/integration/channels_spec.rb +36 -0
- data/spec/sidejob/job_spec.rb +141 -8
- data/spec/sidejob/port_spec.rb +208 -68
- data/spec/sidejob/server_middleware_spec.rb +30 -15
- data/spec/sidejob/testing_spec.rb +0 -18
- data/spec/sidejob/worker_spec.rb +11 -6
- data/spec/sidejob_spec.rb +85 -33
- data/web/Gemfile.lock +2 -2
- data/web/app.rb +22 -19
- metadata +3 -2
data/spec/sidejob/port_spec.rb
CHANGED
@@ -119,8 +119,8 @@ describe SideJob::Port do
|
|
119
119
|
|
120
120
|
it 'can return null default value' do
|
121
121
|
@port1.default = nil
|
122
|
-
expect(@port1.default).to
|
123
|
-
expect(@port1.default?).to
|
122
|
+
expect(@port1.default).to eq nil
|
123
|
+
expect(@port1.default?).to eq true
|
124
124
|
end
|
125
125
|
end
|
126
126
|
|
@@ -150,22 +150,65 @@ describe SideJob::Port do
|
|
150
150
|
end
|
151
151
|
end
|
152
152
|
|
153
|
+
describe '#channels, #channels=' do
|
154
|
+
before do
|
155
|
+
@channels = ['abc123', 'def456']
|
156
|
+
end
|
157
|
+
|
158
|
+
it 'can set and return channels for inport' do
|
159
|
+
expect(@port1.channels).to eq []
|
160
|
+
@channels.each do |ch|
|
161
|
+
expect(SideJob.redis.smembers("channel:#{ch}")).to eq []
|
162
|
+
end
|
163
|
+
@port1.channels = @channels
|
164
|
+
expect(@port1.channels).to match_array(@channels)
|
165
|
+
@channels.each do |ch|
|
166
|
+
expect(SideJob.redis.smembers("channel:#{ch}")).to eq [@port1.job.id.to_s]
|
167
|
+
end
|
168
|
+
@port1.channels = []
|
169
|
+
expect(@port1.channels).to eq []
|
170
|
+
@channels.each do |ch|
|
171
|
+
# We don't remove old jobs until we publish to the channel
|
172
|
+
expect(SideJob.redis.smembers("channel:#{ch}")).to eq [@port1.job.id.to_s]
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
it 'can set and return channels for outport' do
|
177
|
+
expect(@out1.channels).to eq []
|
178
|
+
@out1.channels = @channels
|
179
|
+
@channels.each do |ch|
|
180
|
+
expect(SideJob.redis.smembers("channel:#{ch}")).to eq []
|
181
|
+
end
|
182
|
+
expect(@out1.channels).to match_array(@channels)
|
183
|
+
@channels.each do |ch|
|
184
|
+
expect(SideJob.redis.smembers("channel:#{ch}")).to eq []
|
185
|
+
end
|
186
|
+
@out1.channels = []
|
187
|
+
expect(@out1.channels).to eq []
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
153
191
|
describe '#write' do
|
154
192
|
it 'can write different types of data to a port' do
|
155
|
-
['abc', 123, true, false, nil, {abc
|
156
|
-
data
|
157
|
-
expect(
|
193
|
+
data = ['abc', 123, true, false, nil, {'abc' => 123}, [1, {'foo' => true}]]
|
194
|
+
data.each {|x| @port1.write x}
|
195
|
+
expect(@port1.entries).to eq(data)
|
158
196
|
end
|
159
197
|
|
160
198
|
it 'logs writes' do
|
161
|
-
|
162
|
-
|
163
|
-
SideJob::Port.log_group do
|
199
|
+
expect(SideJob).to receive(:log).with({read: [], write: [{job: @port1.job.id, inport: :port1, data: ['abc', 123]}]})
|
200
|
+
SideJob::Port.group do
|
164
201
|
@port1.write 'abc'
|
165
202
|
@port1.write 123
|
166
203
|
end
|
167
|
-
|
168
|
-
|
204
|
+
end
|
205
|
+
|
206
|
+
it 'can disable logging for both reading and writing' do
|
207
|
+
expect(SideJob).not_to receive(:log)
|
208
|
+
SideJob::Port.group(log: false) do
|
209
|
+
@port1.write 'abc'
|
210
|
+
end
|
211
|
+
expect(@port1.read).to eq 'abc'
|
169
212
|
end
|
170
213
|
|
171
214
|
it 'raises error if port does not exist' do
|
@@ -191,6 +234,34 @@ describe SideJob::Port do
|
|
191
234
|
expect(@job.status).to eq 'completed'
|
192
235
|
expect(parent.status).to eq 'queued'
|
193
236
|
end
|
237
|
+
|
238
|
+
it 'can disable running job' do
|
239
|
+
parent = SideJob.queue('testq', 'TestWorker')
|
240
|
+
parent.adopt(@job, 'child')
|
241
|
+
parent.status = 'completed'
|
242
|
+
@job.status = 'completed'
|
243
|
+
SideJob::Port.group(notify: false) do
|
244
|
+
@port1.write 3
|
245
|
+
@out1.write 3
|
246
|
+
end
|
247
|
+
expect(@job.status).to eq 'completed'
|
248
|
+
expect(parent.status).to eq 'completed'
|
249
|
+
end
|
250
|
+
|
251
|
+
it 'publishes writes to associated output port channel' do
|
252
|
+
data = {'abc' => [1,2]}
|
253
|
+
@out1.channels = ['mychannel']
|
254
|
+
expect(SideJob).to receive(:publish).with('mychannel', data)
|
255
|
+
expect(SideJob).to receive(:publish)
|
256
|
+
@out1.write data
|
257
|
+
end
|
258
|
+
|
259
|
+
it 'does not publish writes to associated input port channel' do
|
260
|
+
data = {'abc' => [1,2]}
|
261
|
+
@port1.channels = ['mychannel']
|
262
|
+
expect(SideJob).not_to receive(:publish).with('mychannel', data)
|
263
|
+
@port1.write data
|
264
|
+
end
|
194
265
|
end
|
195
266
|
|
196
267
|
describe '#read' do
|
@@ -225,21 +296,25 @@ describe SideJob::Port do
|
|
225
296
|
|
226
297
|
it 'can use false default value' do
|
227
298
|
port = @job.input(:default_false)
|
228
|
-
expect(port.read).to
|
299
|
+
expect(port.read).to eq false
|
229
300
|
end
|
230
301
|
|
231
302
|
it 'logs reads' do
|
232
|
-
now = Time.now
|
233
|
-
allow(Time).to receive(:now) { now }
|
234
303
|
['abc', 123].each {|x| @port1.write x}
|
235
|
-
SideJob.
|
236
|
-
SideJob::Port.
|
304
|
+
expect(SideJob).to receive(:log).with({read: [{job: @port1.job.id, inport: :port1, data: ['abc', 123]}], write: []})
|
305
|
+
SideJob::Port.group do
|
306
|
+
expect(@port1.read).to eq('abc')
|
307
|
+
expect(@port1.read).to eq(123)
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
it 'can disable read logging' do
|
312
|
+
['abc', 123].each {|x| @port1.write x}
|
313
|
+
expect(SideJob).not_to receive(:log)
|
314
|
+
SideJob::Port.group(log: false) do
|
237
315
|
expect(@port1.read).to eq('abc')
|
238
316
|
expect(@port1.read).to eq(123)
|
239
317
|
end
|
240
|
-
expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp,
|
241
|
-
'read' => ['job' => @job.id, 'inport' => 'port1', 'data' => ['abc', 123]],
|
242
|
-
'write' => []}]
|
243
318
|
end
|
244
319
|
end
|
245
320
|
|
@@ -320,34 +395,29 @@ describe SideJob::Port do
|
|
320
395
|
end
|
321
396
|
|
322
397
|
it 'logs data' do
|
323
|
-
now = Time.now
|
324
|
-
allow(Time).to receive(:now) { now }
|
325
398
|
@out1.write 1
|
326
399
|
@out1.write [2,3]
|
327
|
-
SideJob.
|
400
|
+
expect(SideJob).to receive(:log).with({read: [{job: @out1.job.id, outport: :out1, data: [1,[2,3]]}], write: [{job: @out1.job.id, inport: :port1, data: [1,[2,3]]}]})
|
328
401
|
@out1.connect_to(@port1)
|
329
|
-
expect(SideJob.logs).to eq([{'timestamp' => SideJob.timestamp,
|
330
|
-
'read' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [1,[2,3]]}],
|
331
|
-
'write' => [{'job' => @job.id, 'inport' => 'port1', 'data' => [1,[2,3]]}] }])
|
332
|
-
end
|
333
|
-
|
334
|
-
it 'can use SideJob.log_context to specify additional metadata' do
|
335
|
-
now = Time.now
|
336
|
-
allow(Time).to receive(:now) { now }
|
337
|
-
@out1.write 1
|
338
|
-
@out1.write [2,3]
|
339
|
-
SideJob.logs(clear: true)
|
340
|
-
SideJob.log_context(user: 'test') do
|
341
|
-
@out1.connect_to(@port1)
|
342
|
-
end
|
343
|
-
expect(SideJob.logs).to eq([{'timestamp' => SideJob.timestamp, 'user' => 'test',
|
344
|
-
'read' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [1,[2,3]]}],
|
345
|
-
'write' => [{'job' => @job.id, 'inport' => 'port1', 'data' => [1,[2,3]]}] }])
|
346
402
|
end
|
347
403
|
|
348
404
|
it 'does not log if no data on port' do
|
405
|
+
expect(SideJob).not_to receive(:publish)
|
349
406
|
@out1.connect_to(@port1)
|
350
|
-
|
407
|
+
end
|
408
|
+
|
409
|
+
it 'publishes to associated outport channels' do
|
410
|
+
dest = [@port1, @out1]
|
411
|
+
@out1.channels = ['channel1']
|
412
|
+
@port1.channels = ['channel2']
|
413
|
+
@default.channels = ['channel3']
|
414
|
+
@default.write 1
|
415
|
+
@default.write [2,3]
|
416
|
+
|
417
|
+
allow(SideJob).to receive(:publish)
|
418
|
+
expect(SideJob).to receive(:publish).with('channel1', 1)
|
419
|
+
expect(SideJob).to receive(:publish).with('channel1', [2,3])
|
420
|
+
@default.connect_to dest
|
351
421
|
end
|
352
422
|
end
|
353
423
|
|
@@ -400,67 +470,137 @@ describe SideJob::Port do
|
|
400
470
|
end
|
401
471
|
end
|
402
472
|
|
403
|
-
describe '.
|
404
|
-
before do
|
405
|
-
now = Time.now
|
406
|
-
allow(Time).to receive(:now) { now }
|
407
|
-
end
|
408
|
-
|
473
|
+
describe '.group' do
|
409
474
|
it 'does not log anything if no port operations occur within the block' do
|
410
|
-
SideJob
|
411
|
-
|
475
|
+
expect(SideJob).not_to receive(:log)
|
476
|
+
SideJob::Port.group {}
|
412
477
|
end
|
413
478
|
|
414
479
|
it 'groups all port logs within the block' do
|
415
|
-
SideJob
|
480
|
+
expect(SideJob).to receive(:log).with({
|
481
|
+
read: [{job: @port1.job.id, inport: :port1, data: ['abc']}],
|
482
|
+
write: [{job: @port1.job.id, inport: :port1, data: ['abc', 'def']},
|
483
|
+
{job: @out1.job.id, outport: :out1, data: ['xyz']}],
|
484
|
+
})
|
485
|
+
SideJob::Port.group do
|
416
486
|
@port1.write 'abc'
|
417
487
|
@port1.read
|
418
488
|
@port1.write 'def'
|
419
489
|
@out1.write 'xyz'
|
420
490
|
end
|
421
|
-
expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp,
|
422
|
-
'read' => [{'job' => @port1.job.id, 'inport' => 'port1', 'data' => ['abc']}],
|
423
|
-
'write' => [{'job' => @port1.job.id, 'inport' => 'port1', 'data' => ['abc', 'def']}, {'job' => @out1.job.id, 'outport' => 'out1', 'data' => ['xyz']} ]}]
|
424
491
|
end
|
425
492
|
|
426
493
|
it 'does not write out log until the end of outermost group' do
|
427
|
-
SideJob
|
494
|
+
expect(SideJob).to receive(:log).with({
|
495
|
+
read: [],
|
496
|
+
write: [{job: @port1.job.id, inport: :port1, data: ['hello', 2]}],
|
497
|
+
})
|
498
|
+
SideJob::Port.group do
|
428
499
|
@port1.write 'hello'
|
429
|
-
SideJob::Port.
|
500
|
+
SideJob::Port.group do
|
430
501
|
@port1.write 2
|
431
502
|
end
|
432
|
-
expect(SideJob.logs.length).to eq 0
|
433
503
|
end
|
434
|
-
expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp,
|
435
|
-
'read' => [],
|
436
|
-
'write' => [{'job' => @port1.job.id, 'inport' => 'port1', 'data' => ['hello', 2]}]}]
|
437
504
|
end
|
438
505
|
|
439
|
-
it 'works with SideJob.
|
440
|
-
|
441
|
-
|
506
|
+
it 'works with SideJob.context' do
|
507
|
+
now = Time.now
|
508
|
+
allow(Time).to receive(:now) { now }
|
509
|
+
allow(SideJob).to receive(:publish)
|
510
|
+
expect(SideJob).to receive(:publish).with('/sidejob/log', {
|
511
|
+
user: 'foo',
|
512
|
+
read: [{job: @port1.job.id, inport: :port1, data: ['abc']}],
|
513
|
+
write: [{job: @port1.job.id, inport: :port1, data: ['abc']}],
|
514
|
+
timestamp: SideJob.timestamp})
|
515
|
+
SideJob.context(user: 'foo') do
|
516
|
+
SideJob::Port.group do
|
442
517
|
@port1.write 'abc'
|
443
518
|
@port1.read
|
444
519
|
end
|
445
520
|
end
|
446
|
-
expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'user' => 'foo',
|
447
|
-
'read' => [{'job' => @port1.job.id, 'inport' => 'port1', 'data' => ['abc']}],
|
448
|
-
'write' => [{'job' => @port1.job.id, 'inport' => 'port1', 'data' => ['abc']}]}]
|
449
521
|
end
|
450
522
|
|
451
523
|
it 'logs correctly even if data is changed' do
|
524
|
+
expect(SideJob).to receive(:log).with({
|
525
|
+
read: [{job: @port1.job.id, inport: :port1, data: [{'x' => [1,2]},{'x' => [1,2,3]}]}],
|
526
|
+
write: [{job: @port1.job.id, inport: :port1, data: [{'x' => [1,2]},{'x' => [1,2,3]}]}],
|
527
|
+
})
|
452
528
|
data = {'x' => [1,2]}
|
453
|
-
SideJob::Port.
|
529
|
+
SideJob::Port.group do
|
454
530
|
@port1.write data
|
455
531
|
expect(@port1.read).to eq data
|
456
532
|
data['x'].push 3
|
457
533
|
@port1.write data
|
458
534
|
expect(@port1.read).to eq data
|
459
535
|
end
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
536
|
+
end
|
537
|
+
|
538
|
+
it 'can set options' do
|
539
|
+
SideJob::Port.group(log: true, notify: false, set_default: true) do
|
540
|
+
expect(Thread.current[:sidejob_port_group][:options]).to eq({log: true, notify: false, set_default: true})
|
541
|
+
end
|
542
|
+
expect(Thread.current[:sidejob_port_group]).to be nil
|
543
|
+
end
|
544
|
+
|
545
|
+
it 'can merge options in nested groups' do
|
546
|
+
SideJob::Port.group(log: false) do
|
547
|
+
expect(Thread.current[:sidejob_port_group][:options]).to eq({log: false})
|
548
|
+
SideJob::Port.group(notify: true) do
|
549
|
+
expect(Thread.current[:sidejob_port_group][:options]).to eq({log: false, notify: true})
|
550
|
+
end
|
551
|
+
expect(Thread.current[:sidejob_port_group][:options]).to eq({log: false})
|
552
|
+
end
|
553
|
+
expect(Thread.current[:sidejob_port_group]).to be nil
|
554
|
+
end
|
555
|
+
end
|
556
|
+
|
557
|
+
describe '.encode_data' do
|
558
|
+
it 'encodes data with no context' do
|
559
|
+
expect(JSON.parse(SideJob::Port.encode_data(5))).to eq({ 'data' => 5 })
|
560
|
+
end
|
561
|
+
|
562
|
+
it 'handles context' do
|
563
|
+
SideJob.context({xyz: 456}) do
|
564
|
+
expect(JSON.parse(SideJob::Port.encode_data([1,2]))).to eq({ 'context' => {'xyz' => 456}, 'data' => [1,2] })
|
565
|
+
end
|
566
|
+
end
|
567
|
+
|
568
|
+
it 'handles port group options' do
|
569
|
+
SideJob::Port.group(log: true, notify: false) do
|
570
|
+
expect(JSON.parse(SideJob::Port.encode_data([1,2]))).to eq({ 'options' => {'log' => true, 'notify' => false}, 'data' => [1,2] })
|
571
|
+
end
|
572
|
+
end
|
573
|
+
end
|
574
|
+
|
575
|
+
describe '.decode_data' do
|
576
|
+
it 'returns None if no data' do
|
577
|
+
expect(SideJob::Port.decode_data(nil)).to be SideJob::Port::None
|
578
|
+
end
|
579
|
+
|
580
|
+
it 'decodes object with no context' do
|
581
|
+
x = SideJob::Port.decode_data(SideJob::Port.encode_data(1.23))
|
582
|
+
expect(x.sidejob_context).to eq({})
|
583
|
+
end
|
584
|
+
|
585
|
+
it 'handles context' do
|
586
|
+
SideJob.context({abc: 'foo'}) do
|
587
|
+
SideJob.context({xyz: 456}) do
|
588
|
+
x = SideJob::Port.decode_data(SideJob::Port.encode_data(nil))
|
589
|
+
expect(x.sidejob_context).to eq({'abc' => 'foo', 'xyz' => 456})
|
590
|
+
end
|
591
|
+
end
|
592
|
+
end
|
593
|
+
|
594
|
+
it 'handles port group options' do
|
595
|
+
SideJob::Port.group(log: true, notify: false) do
|
596
|
+
expect(SideJob::Port.decode_data(SideJob::Port.encode_data([1,2])).sidejob_options).to eq({ 'log' => true, 'notify' => false})
|
597
|
+
end
|
598
|
+
end
|
599
|
+
|
600
|
+
it 'decoded data should equal original data' do
|
601
|
+
['abc', [1,2], {'abc' => [1,2]}, 1, 1.23, true, false, nil].each do |x|
|
602
|
+
expect(SideJob::Port.decode_data(SideJob::Port.encode_data(x))).to eq x
|
603
|
+
end
|
464
604
|
end
|
465
605
|
end
|
466
606
|
end
|
@@ -1,6 +1,17 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe SideJob::ServerMiddleware do
|
4
|
+
class TestWorkerStartup
|
5
|
+
include SideJob::Worker
|
6
|
+
register
|
7
|
+
attr_accessor :startup_called
|
8
|
+
def startup
|
9
|
+
@startup_called = true
|
10
|
+
end
|
11
|
+
def perform
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
4
15
|
class TestWorkerShutdown
|
5
16
|
include SideJob::Worker
|
6
17
|
register
|
@@ -73,7 +84,7 @@ describe SideJob::ServerMiddleware do
|
|
73
84
|
it 'sets the ran_at time at the beginning of the run' do
|
74
85
|
now = Time.now
|
75
86
|
allow(Time).to receive(:now) { now }
|
76
|
-
process(@job) { |worker| @ran_at = worker.
|
87
|
+
process(@job) { |worker| @ran_at = worker.info[:ran_at] }
|
77
88
|
expect(@ran_at).to eq SideJob.timestamp
|
78
89
|
expect(@job.status).to eq 'completed'
|
79
90
|
end
|
@@ -147,15 +158,11 @@ describe SideJob::ServerMiddleware do
|
|
147
158
|
it 'sets status to failed on exception and logs error' do
|
148
159
|
now = Time.now
|
149
160
|
allow(Time).to receive(:now) { now }
|
150
|
-
|
161
|
+
SideJob::ServerMiddleware.raise_errors = false
|
162
|
+
exception = RuntimeError.new('oops')
|
163
|
+
expect(SideJob).to receive(:log).with(exception)
|
164
|
+
process(@job) { raise exception }
|
151
165
|
expect(@job.status).to eq 'failed'
|
152
|
-
|
153
|
-
log = SideJob.logs.select {|log| log['error'] }
|
154
|
-
expect(log.size).to eq(1)
|
155
|
-
expect(log[0]['job']).to eq @job.id
|
156
|
-
expect(log[0]['error']).to eq('oops')
|
157
|
-
# check that we trim down backtrace to remove sidekiq lines
|
158
|
-
expect(log[0]['backtrace']).to_not match(/sidekiq/)
|
159
166
|
end
|
160
167
|
|
161
168
|
it 'does not set status to failed if status is terminating' do
|
@@ -190,13 +197,19 @@ describe SideJob::ServerMiddleware do
|
|
190
197
|
end
|
191
198
|
end
|
192
199
|
|
200
|
+
describe 'handles job startup' do
|
201
|
+
it 'calls worker startup method' do
|
202
|
+
@job = SideJob.queue(@queue, 'TestWorkerStartup')
|
203
|
+
worker = process(@job) { }
|
204
|
+
expect(worker.startup_called).to be true
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
193
208
|
describe 'handles job termination' do
|
194
209
|
it 'sets status to terminated upon run' do
|
195
210
|
@job.status = 'terminating'
|
196
211
|
process(@job) { raise 'should not be called' }
|
197
212
|
expect(@job.status).to eq 'terminated'
|
198
|
-
errors = SideJob.logs.select {|log| log['type'] == 'error'}
|
199
|
-
expect(errors.size).to eq 0
|
200
213
|
end
|
201
214
|
|
202
215
|
it 'runs parent' do
|
@@ -217,11 +230,13 @@ describe SideJob::ServerMiddleware do
|
|
217
230
|
it 'logs but ignores exceptions thrown during shutdown' do
|
218
231
|
@job = SideJob.queue(@queue, 'TestWorkerShutdownError')
|
219
232
|
@job.status = 'terminating'
|
233
|
+
logged = false
|
234
|
+
expect(SideJob).to receive(:log) do |exception|
|
235
|
+
expect(exception.message).to eq 'shutdown error'
|
236
|
+
logged = true
|
237
|
+
end
|
220
238
|
worker = process(@job) { raise 'not reached' }
|
221
|
-
|
222
|
-
expect(logs.size).to eq 1
|
223
|
-
expect(logs[0]['job']).to eq @job.id
|
224
|
-
expect(logs[0]['error']).to eq 'shutdown error'
|
239
|
+
expect(logged).to be true
|
225
240
|
end
|
226
241
|
end
|
227
242
|
end
|