rocketjob 3.5.2 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +63 -1
- data/bin/rocketjob +1 -0
- data/bin/rocketjob_batch_perf +11 -0
- data/lib/rocket_job/batch.rb +32 -0
- data/lib/rocket_job/batch/callbacks.rb +40 -0
- data/lib/rocket_job/batch/io.rb +154 -0
- data/lib/rocket_job/batch/logger.rb +57 -0
- data/lib/rocket_job/batch/lower_priority.rb +54 -0
- data/lib/rocket_job/batch/model.rb +157 -0
- data/lib/rocket_job/batch/performance.rb +99 -0
- data/lib/rocket_job/batch/result.rb +8 -0
- data/lib/rocket_job/batch/results.rb +9 -0
- data/lib/rocket_job/batch/state_machine.rb +102 -0
- data/lib/rocket_job/batch/statistics.rb +88 -0
- data/lib/rocket_job/batch/tabular.rb +56 -0
- data/lib/rocket_job/batch/tabular/input.rb +123 -0
- data/lib/rocket_job/batch/tabular/output.rb +59 -0
- data/lib/rocket_job/batch/throttle.rb +91 -0
- data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
- data/lib/rocket_job/batch/worker.rb +288 -0
- data/lib/rocket_job/cli.rb +29 -7
- data/lib/rocket_job/config.rb +1 -1
- data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
- data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
- data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
- data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
- data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
- data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
- data/lib/rocket_job/jobs/performance_job.rb +18 -0
- data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
- data/lib/rocket_job/plugins/document.rb +2 -8
- data/lib/rocket_job/plugins/job/persistence.rb +6 -4
- data/lib/rocket_job/plugins/job/throttle.rb +3 -6
- data/lib/rocket_job/plugins/job/worker.rb +2 -2
- data/lib/rocket_job/server.rb +14 -3
- data/lib/rocket_job/sliced/input.rb +336 -0
- data/lib/rocket_job/sliced/output.rb +99 -0
- data/lib/rocket_job/sliced/slice.rb +166 -0
- data/lib/rocket_job/sliced/slices.rb +166 -0
- data/lib/rocket_job/sliced/writer/input.rb +60 -0
- data/lib/rocket_job/sliced/writer/output.rb +82 -0
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +2 -2
- data/lib/rocketjob.rb +28 -0
- metadata +51 -62
- data/test/config/database.yml +0 -5
- data/test/config/mongoid.yml +0 -88
- data/test/config_test.rb +0 -10
- data/test/dirmon_entry_test.rb +0 -313
- data/test/dirmon_job_test.rb +0 -216
- data/test/files/text.txt +0 -3
- data/test/job_test.rb +0 -71
- data/test/jobs/housekeeping_job_test.rb +0 -102
- data/test/jobs/on_demand_job_test.rb +0 -59
- data/test/jobs/upload_file_job_test.rb +0 -107
- data/test/plugins/cron_test.rb +0 -166
- data/test/plugins/job/callbacks_test.rb +0 -166
- data/test/plugins/job/defaults_test.rb +0 -53
- data/test/plugins/job/logger_test.rb +0 -56
- data/test/plugins/job/model_test.rb +0 -94
- data/test/plugins/job/persistence_test.rb +0 -94
- data/test/plugins/job/state_machine_test.rb +0 -116
- data/test/plugins/job/throttle_test.rb +0 -111
- data/test/plugins/job/worker_test.rb +0 -199
- data/test/plugins/processing_window_test.rb +0 -109
- data/test/plugins/restart_test.rb +0 -193
- data/test/plugins/retry_test.rb +0 -88
- data/test/plugins/singleton_test.rb +0 -92
- data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
- data/test/plugins/state_machine_test.rb +0 -67
- data/test/plugins/transaction_test.rb +0 -84
- data/test/test_db.sqlite3 +0 -0
- data/test/test_helper.rb +0 -17
@@ -1,166 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class CallbacksTest < Minitest::Test
|
7
|
-
# This job adds each callback as they run into an array
|
8
|
-
class BeforePerformJob < RocketJob::Job
|
9
|
-
field :call_list, type: Array, default: []
|
10
|
-
|
11
|
-
before_perform do
|
12
|
-
call_list << 'before_perform_block'
|
13
|
-
end
|
14
|
-
|
15
|
-
before_perform :before_perform_method
|
16
|
-
|
17
|
-
def perform
|
18
|
-
call_list << 'perform'
|
19
|
-
end
|
20
|
-
|
21
|
-
private
|
22
|
-
|
23
|
-
def before_perform_method
|
24
|
-
call_list << 'before_perform_method'
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
# This job adds each callback as they run into an array
|
29
|
-
class AfterPerformJob < RocketJob::Job
|
30
|
-
field :call_list, type: Array, default: []
|
31
|
-
|
32
|
-
after_perform do
|
33
|
-
call_list << 'after_perform_block'
|
34
|
-
end
|
35
|
-
|
36
|
-
after_perform :after_perform_method
|
37
|
-
|
38
|
-
def perform
|
39
|
-
call_list << 'perform'
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
def after_perform_method
|
45
|
-
call_list << 'after_perform_method'
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
# This job adds each callback as they run into an array
|
50
|
-
class AroundPerformJob < RocketJob::Job
|
51
|
-
field :call_list, type: Array, default: []
|
52
|
-
|
53
|
-
around_perform do |_job, block|
|
54
|
-
call_list << 'around_perform_block_before'
|
55
|
-
block.call
|
56
|
-
call_list << 'around_perform_block_after'
|
57
|
-
end
|
58
|
-
|
59
|
-
around_perform :around_perform_method
|
60
|
-
|
61
|
-
def perform
|
62
|
-
call_list << 'perform'
|
63
|
-
end
|
64
|
-
|
65
|
-
private
|
66
|
-
|
67
|
-
def around_perform_method
|
68
|
-
call_list << 'around_perform_method_before'
|
69
|
-
yield
|
70
|
-
call_list << 'around_perform_method_after'
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
# This job adds each callback as they run into an array
|
75
|
-
class CombinedPerformJob < RocketJob::Job
|
76
|
-
field :call_list, type: Array, default: []
|
77
|
-
|
78
|
-
before_perform do
|
79
|
-
call_list << 'before_perform_block'
|
80
|
-
end
|
81
|
-
|
82
|
-
after_perform do
|
83
|
-
call_list << 'after_perform_block'
|
84
|
-
end
|
85
|
-
|
86
|
-
around_perform do |_job, block|
|
87
|
-
call_list << 'around_perform_block_before'
|
88
|
-
block.call
|
89
|
-
call_list << 'around_perform_block_after'
|
90
|
-
end
|
91
|
-
|
92
|
-
before_perform :before_perform_method
|
93
|
-
|
94
|
-
around_perform :around_perform_method
|
95
|
-
|
96
|
-
after_perform :after_perform_method
|
97
|
-
|
98
|
-
def perform
|
99
|
-
call_list << 'perform'
|
100
|
-
end
|
101
|
-
|
102
|
-
private
|
103
|
-
|
104
|
-
def before_perform_method
|
105
|
-
call_list << 'before_perform_method'
|
106
|
-
end
|
107
|
-
|
108
|
-
def around_perform_method
|
109
|
-
call_list << 'around_perform_method_before'
|
110
|
-
yield
|
111
|
-
call_list << 'around_perform_method_after'
|
112
|
-
end
|
113
|
-
|
114
|
-
def after_perform_method
|
115
|
-
call_list << 'after_perform_method'
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
describe RocketJob::Plugins::Job::Callbacks do
|
120
|
-
after do
|
121
|
-
@job.destroy if @job && !@job.new_record?
|
122
|
-
end
|
123
|
-
|
124
|
-
describe '#before_perform' do
|
125
|
-
it 'runs blocks and functions' do
|
126
|
-
@job = BeforePerformJob.new
|
127
|
-
@job.perform_now
|
128
|
-
assert @job.completed?, @job.attributes.ai
|
129
|
-
expected = %w[before_perform_block before_perform_method perform]
|
130
|
-
assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
describe '#after_perform' do
|
135
|
-
it 'runs blocks and functions' do
|
136
|
-
@job = AfterPerformJob.new
|
137
|
-
@job.perform_now
|
138
|
-
assert @job.completed?, @job.attributes.ai
|
139
|
-
expected = %w[perform after_perform_method after_perform_block]
|
140
|
-
assert_equal expected, @job.call_list, 'Sequence of after_perform callbacks is incorrect'
|
141
|
-
end
|
142
|
-
end
|
143
|
-
|
144
|
-
describe '#around_perform' do
|
145
|
-
it 'runs blocks and functions' do
|
146
|
-
@job = AroundPerformJob.new
|
147
|
-
@job.perform_now
|
148
|
-
assert @job.completed?, @job.attributes.ai
|
149
|
-
expected = %w[around_perform_block_before around_perform_method_before perform around_perform_method_after around_perform_block_after]
|
150
|
-
assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
|
151
|
-
end
|
152
|
-
end
|
153
|
-
|
154
|
-
describe 'all callbacks' do
|
155
|
-
it 'runs them in the right order' do
|
156
|
-
@job = CombinedPerformJob.new
|
157
|
-
@job.perform_now
|
158
|
-
assert @job.completed?, @job.attributes.ai
|
159
|
-
expected = %w[before_perform_block around_perform_block_before before_perform_method around_perform_method_before perform after_perform_method around_perform_method_after around_perform_block_after after_perform_block]
|
160
|
-
assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
|
161
|
-
end
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
165
|
-
end
|
166
|
-
end
|
@@ -1,53 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class DefaultsTest < Minitest::Test
|
7
|
-
class ParentJob < RocketJob::Job
|
8
|
-
self.priority = 53
|
9
|
-
self.description = 'Hello'
|
10
|
-
|
11
|
-
def perform
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
class ChildJob < ParentJob
|
16
|
-
self.priority = 72
|
17
|
-
|
18
|
-
def perform
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
describe RocketJob::Plugins::Job do
|
23
|
-
after do
|
24
|
-
@job.destroy if @job && !@job.new_record?
|
25
|
-
end
|
26
|
-
|
27
|
-
describe '.rocket_job' do
|
28
|
-
it 'sets defaults after initialize' do
|
29
|
-
@job = ParentJob.new
|
30
|
-
assert_equal 53, @job.priority
|
31
|
-
assert_equal 'Hello', @job.description
|
32
|
-
end
|
33
|
-
|
34
|
-
it 'can override defaults on initialize' do
|
35
|
-
@job = ParentJob.new(priority: 72, description: 'More')
|
36
|
-
assert_equal 72, @job.priority
|
37
|
-
assert_equal 'More', @job.description
|
38
|
-
end
|
39
|
-
|
40
|
-
it 'allows a child to override parent defaults' do
|
41
|
-
@job = ChildJob.new
|
42
|
-
assert_equal 72, @job.priority
|
43
|
-
end
|
44
|
-
|
45
|
-
it 'passes down parent defaults' do
|
46
|
-
@job = ChildJob.new
|
47
|
-
assert_equal 'Hello', @job.description
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
@@ -1,56 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
# Unit Test for RocketJob::Job
|
4
|
-
module Plugins
|
5
|
-
module Job
|
6
|
-
class LoggerTest < Minitest::Test
|
7
|
-
class LoggerJob < RocketJob::Job
|
8
|
-
def perform
|
9
|
-
logger.debug('DONE', value: 123, other_value: 'HI')
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
describe RocketJob::Plugins::Job::Logger do
|
14
|
-
before do
|
15
|
-
LoggerJob.delete_all
|
16
|
-
end
|
17
|
-
|
18
|
-
after do
|
19
|
-
@job.destroy if @job && !@job.new_record?
|
20
|
-
SemanticLogger.flush
|
21
|
-
end
|
22
|
-
|
23
|
-
describe '#logger' do
|
24
|
-
it 'uses semantic logger' do
|
25
|
-
@job = LoggerJob.new
|
26
|
-
assert_kind_of SemanticLogger::Logger, @job.logger, @job.logger.ai
|
27
|
-
assert_equal @job.class.name, @job.logger.name, @job.logger.ai
|
28
|
-
end
|
29
|
-
|
30
|
-
it 'allows perform to log its own data' do
|
31
|
-
@job = LoggerJob.new
|
32
|
-
@job.perform_now
|
33
|
-
end
|
34
|
-
|
35
|
-
it 'adds start logging' do
|
36
|
-
@job = LoggerJob.new
|
37
|
-
info_called = false
|
38
|
-
@job.logger.stub(:info, ->(description) { info_called = true if description == 'Start #perform' }) do
|
39
|
-
@job.perform_now
|
40
|
-
end
|
41
|
-
assert info_called, "In Plugins::Job::Logger.around_perform logger.info('Start #perform') not called"
|
42
|
-
end
|
43
|
-
|
44
|
-
it 'adds completed logging' do
|
45
|
-
@job = LoggerJob.new
|
46
|
-
measure_called = false
|
47
|
-
@job.logger.stub(:measure_info, ->(description, *_args) { measure_called = true if description == 'Completed #perform' }) do
|
48
|
-
@job.perform_now
|
49
|
-
end
|
50
|
-
assert measure_called, "In Plugins::Job::Logger.around_perform logger.measure_info('Completed #perform') not called"
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,94 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class ModelTest < Minitest::Test
|
7
|
-
class SimpleJob < RocketJob::Job
|
8
|
-
def perform
|
9
|
-
10
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
class TwoArgumentJob < RocketJob::Job
|
14
|
-
self.priority = 53
|
15
|
-
|
16
|
-
def perform(a, b)
|
17
|
-
a + b
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
describe RocketJob::Plugins::Job::Model do
|
22
|
-
after do
|
23
|
-
@job.destroy if @job && !@job.new_record?
|
24
|
-
@job2.destroy if @job2 && !@job2.new_record?
|
25
|
-
@job3.destroy if @job3 && !@job3.new_record?
|
26
|
-
end
|
27
|
-
|
28
|
-
describe '#scheduled?' do
|
29
|
-
it 'returns true if job is queued to run in the future' do
|
30
|
-
@job = SimpleJob.new(run_at: 1.day.from_now)
|
31
|
-
assert_equal true, @job.queued?
|
32
|
-
assert_equal true, @job.scheduled?
|
33
|
-
@job.start
|
34
|
-
assert_equal true, @job.running?
|
35
|
-
assert_equal false, @job.scheduled?
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'returns false if job is queued and can be run now' do
|
39
|
-
@job = SimpleJob.new
|
40
|
-
assert_equal true, @job.queued?
|
41
|
-
assert_equal false, @job.scheduled?
|
42
|
-
end
|
43
|
-
|
44
|
-
it 'returns false if job is running' do
|
45
|
-
@job = SimpleJob.new
|
46
|
-
@job.start
|
47
|
-
assert_equal true, @job.running?
|
48
|
-
assert_equal false, @job.scheduled?
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
describe 'with queued jobs' do
|
53
|
-
before do
|
54
|
-
@job = SimpleJob.create!(description: 'first')
|
55
|
-
@job2 = SimpleJob.create!(description: 'second', run_at: 1.day.from_now)
|
56
|
-
@job3 = SimpleJob.create!(description: 'third', run_at: 2.days.from_now)
|
57
|
-
end
|
58
|
-
|
59
|
-
describe '#scheduled' do
|
60
|
-
it 'returns only scheduled jobs' do
|
61
|
-
count = 0
|
62
|
-
RocketJob::Job.scheduled.each do |job|
|
63
|
-
count += 1
|
64
|
-
assert job.scheduled?
|
65
|
-
end
|
66
|
-
assert 2, count
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
describe '#queued_now' do
|
71
|
-
it 'returns only queued jobs, not scheduled ones' do
|
72
|
-
count = 0
|
73
|
-
RocketJob::Job.queued_now.each do |job|
|
74
|
-
count += 1
|
75
|
-
refute job.scheduled?
|
76
|
-
end
|
77
|
-
assert 1, count
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
describe '#queued' do
|
82
|
-
it 'returns all queued jobs' do
|
83
|
-
count = 0
|
84
|
-
RocketJob::Job.queued.each do |_job|
|
85
|
-
count += 1
|
86
|
-
end
|
87
|
-
assert 3, count
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
@@ -1,94 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class PersistenceTest < Minitest::Test
|
7
|
-
class PersistJob < RocketJob::Job
|
8
|
-
self.priority = 53
|
9
|
-
field :data, type: Hash
|
10
|
-
|
11
|
-
def perform(hash)
|
12
|
-
hash
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
describe RocketJob::Plugins::Job::Persistence do
|
17
|
-
before do
|
18
|
-
RocketJob::Job.destroy_all
|
19
|
-
@description = 'Hello World'
|
20
|
-
@data = {'key' => 'value'}
|
21
|
-
@job = PersistJob.new(
|
22
|
-
description: @description,
|
23
|
-
data: @data,
|
24
|
-
destroy_on_complete: false
|
25
|
-
)
|
26
|
-
end
|
27
|
-
|
28
|
-
after do
|
29
|
-
@job.destroy if @job && !@job.new_record?
|
30
|
-
@job2.destroy if @job2 && !@job2.new_record?
|
31
|
-
@job3.destroy if @job3 && !@job3.new_record?
|
32
|
-
end
|
33
|
-
|
34
|
-
describe '.config' do
|
35
|
-
it 'support multiple databases' do
|
36
|
-
assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
describe '.rocket_job' do
|
41
|
-
it 'sets defaults after initialize' do
|
42
|
-
assert_equal 53, @job.priority
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
describe '#reload' do
|
47
|
-
it 'handle hash' do
|
48
|
-
assert_equal 'value', @job.data['key']
|
49
|
-
@job.worker_name = nil
|
50
|
-
@job.save!
|
51
|
-
@job.worker_name = '123'
|
52
|
-
@job.reload
|
53
|
-
assert @job.data.is_a?(Hash), @job.data.class.ai
|
54
|
-
assert_equal 'value', @job.data['key']
|
55
|
-
assert_nil @job.worker_name
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
describe '#save!' do
|
60
|
-
it 'save a blank job' do
|
61
|
-
@job.save!
|
62
|
-
assert_nil @job.worker_name
|
63
|
-
assert_nil @job.completed_at
|
64
|
-
assert @job.created_at
|
65
|
-
assert_equal @description, @job.description
|
66
|
-
assert_equal false, @job.destroy_on_complete
|
67
|
-
assert_nil @job.expires_at
|
68
|
-
assert_equal @data, @job.data
|
69
|
-
assert_equal 0, @job.percent_complete
|
70
|
-
assert_equal 53, @job.priority
|
71
|
-
assert_equal 0, @job.failure_count
|
72
|
-
assert_nil @job.run_at
|
73
|
-
assert_nil @job.started_at
|
74
|
-
assert_equal :queued, @job.state
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
describe '.counts_by_state' do
|
79
|
-
it 'returns states as symbols' do
|
80
|
-
@job.start!
|
81
|
-
@job2 = PersistJob.create!(data: {key: 'value'})
|
82
|
-
@job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
|
83
|
-
counts = RocketJob::Job.counts_by_state
|
84
|
-
assert_equal 4, counts.size, counts.ai
|
85
|
-
assert_equal 1, counts[:running]
|
86
|
-
assert_equal 2, counts[:queued]
|
87
|
-
assert_equal 1, counts[:queued_now]
|
88
|
-
assert_equal 1, counts[:scheduled]
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|