rocketjob 3.5.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +63 -1
- data/bin/rocketjob +1 -0
- data/bin/rocketjob_batch_perf +11 -0
- data/lib/rocket_job/batch.rb +32 -0
- data/lib/rocket_job/batch/callbacks.rb +40 -0
- data/lib/rocket_job/batch/io.rb +154 -0
- data/lib/rocket_job/batch/logger.rb +57 -0
- data/lib/rocket_job/batch/lower_priority.rb +54 -0
- data/lib/rocket_job/batch/model.rb +157 -0
- data/lib/rocket_job/batch/performance.rb +99 -0
- data/lib/rocket_job/batch/result.rb +8 -0
- data/lib/rocket_job/batch/results.rb +9 -0
- data/lib/rocket_job/batch/state_machine.rb +102 -0
- data/lib/rocket_job/batch/statistics.rb +88 -0
- data/lib/rocket_job/batch/tabular.rb +56 -0
- data/lib/rocket_job/batch/tabular/input.rb +123 -0
- data/lib/rocket_job/batch/tabular/output.rb +59 -0
- data/lib/rocket_job/batch/throttle.rb +91 -0
- data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
- data/lib/rocket_job/batch/worker.rb +288 -0
- data/lib/rocket_job/cli.rb +29 -7
- data/lib/rocket_job/config.rb +1 -1
- data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
- data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
- data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
- data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
- data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
- data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
- data/lib/rocket_job/jobs/performance_job.rb +18 -0
- data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
- data/lib/rocket_job/plugins/document.rb +2 -8
- data/lib/rocket_job/plugins/job/persistence.rb +6 -4
- data/lib/rocket_job/plugins/job/throttle.rb +3 -6
- data/lib/rocket_job/plugins/job/worker.rb +2 -2
- data/lib/rocket_job/server.rb +14 -3
- data/lib/rocket_job/sliced/input.rb +336 -0
- data/lib/rocket_job/sliced/output.rb +99 -0
- data/lib/rocket_job/sliced/slice.rb +166 -0
- data/lib/rocket_job/sliced/slices.rb +166 -0
- data/lib/rocket_job/sliced/writer/input.rb +60 -0
- data/lib/rocket_job/sliced/writer/output.rb +82 -0
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +2 -2
- data/lib/rocketjob.rb +28 -0
- metadata +51 -62
- data/test/config/database.yml +0 -5
- data/test/config/mongoid.yml +0 -88
- data/test/config_test.rb +0 -10
- data/test/dirmon_entry_test.rb +0 -313
- data/test/dirmon_job_test.rb +0 -216
- data/test/files/text.txt +0 -3
- data/test/job_test.rb +0 -71
- data/test/jobs/housekeeping_job_test.rb +0 -102
- data/test/jobs/on_demand_job_test.rb +0 -59
- data/test/jobs/upload_file_job_test.rb +0 -107
- data/test/plugins/cron_test.rb +0 -166
- data/test/plugins/job/callbacks_test.rb +0 -166
- data/test/plugins/job/defaults_test.rb +0 -53
- data/test/plugins/job/logger_test.rb +0 -56
- data/test/plugins/job/model_test.rb +0 -94
- data/test/plugins/job/persistence_test.rb +0 -94
- data/test/plugins/job/state_machine_test.rb +0 -116
- data/test/plugins/job/throttle_test.rb +0 -111
- data/test/plugins/job/worker_test.rb +0 -199
- data/test/plugins/processing_window_test.rb +0 -109
- data/test/plugins/restart_test.rb +0 -193
- data/test/plugins/retry_test.rb +0 -88
- data/test/plugins/singleton_test.rb +0 -92
- data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
- data/test/plugins/state_machine_test.rb +0 -67
- data/test/plugins/transaction_test.rb +0 -84
- data/test/test_db.sqlite3 +0 -0
- data/test/test_helper.rb +0 -17
@@ -1,166 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class CallbacksTest < Minitest::Test
|
7
|
-
# This job adds each callback as they run into an array
|
8
|
-
class BeforePerformJob < RocketJob::Job
|
9
|
-
field :call_list, type: Array, default: []
|
10
|
-
|
11
|
-
before_perform do
|
12
|
-
call_list << 'before_perform_block'
|
13
|
-
end
|
14
|
-
|
15
|
-
before_perform :before_perform_method
|
16
|
-
|
17
|
-
def perform
|
18
|
-
call_list << 'perform'
|
19
|
-
end
|
20
|
-
|
21
|
-
private
|
22
|
-
|
23
|
-
def before_perform_method
|
24
|
-
call_list << 'before_perform_method'
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
# This job adds each callback as they run into an array
|
29
|
-
class AfterPerformJob < RocketJob::Job
|
30
|
-
field :call_list, type: Array, default: []
|
31
|
-
|
32
|
-
after_perform do
|
33
|
-
call_list << 'after_perform_block'
|
34
|
-
end
|
35
|
-
|
36
|
-
after_perform :after_perform_method
|
37
|
-
|
38
|
-
def perform
|
39
|
-
call_list << 'perform'
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
def after_perform_method
|
45
|
-
call_list << 'after_perform_method'
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
# This job adds each callback as they run into an array
|
50
|
-
class AroundPerformJob < RocketJob::Job
|
51
|
-
field :call_list, type: Array, default: []
|
52
|
-
|
53
|
-
around_perform do |_job, block|
|
54
|
-
call_list << 'around_perform_block_before'
|
55
|
-
block.call
|
56
|
-
call_list << 'around_perform_block_after'
|
57
|
-
end
|
58
|
-
|
59
|
-
around_perform :around_perform_method
|
60
|
-
|
61
|
-
def perform
|
62
|
-
call_list << 'perform'
|
63
|
-
end
|
64
|
-
|
65
|
-
private
|
66
|
-
|
67
|
-
def around_perform_method
|
68
|
-
call_list << 'around_perform_method_before'
|
69
|
-
yield
|
70
|
-
call_list << 'around_perform_method_after'
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
# This job adds each callback as they run into an array
|
75
|
-
class CombinedPerformJob < RocketJob::Job
|
76
|
-
field :call_list, type: Array, default: []
|
77
|
-
|
78
|
-
before_perform do
|
79
|
-
call_list << 'before_perform_block'
|
80
|
-
end
|
81
|
-
|
82
|
-
after_perform do
|
83
|
-
call_list << 'after_perform_block'
|
84
|
-
end
|
85
|
-
|
86
|
-
around_perform do |_job, block|
|
87
|
-
call_list << 'around_perform_block_before'
|
88
|
-
block.call
|
89
|
-
call_list << 'around_perform_block_after'
|
90
|
-
end
|
91
|
-
|
92
|
-
before_perform :before_perform_method
|
93
|
-
|
94
|
-
around_perform :around_perform_method
|
95
|
-
|
96
|
-
after_perform :after_perform_method
|
97
|
-
|
98
|
-
def perform
|
99
|
-
call_list << 'perform'
|
100
|
-
end
|
101
|
-
|
102
|
-
private
|
103
|
-
|
104
|
-
def before_perform_method
|
105
|
-
call_list << 'before_perform_method'
|
106
|
-
end
|
107
|
-
|
108
|
-
def around_perform_method
|
109
|
-
call_list << 'around_perform_method_before'
|
110
|
-
yield
|
111
|
-
call_list << 'around_perform_method_after'
|
112
|
-
end
|
113
|
-
|
114
|
-
def after_perform_method
|
115
|
-
call_list << 'after_perform_method'
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
describe RocketJob::Plugins::Job::Callbacks do
|
120
|
-
after do
|
121
|
-
@job.destroy if @job && !@job.new_record?
|
122
|
-
end
|
123
|
-
|
124
|
-
describe '#before_perform' do
|
125
|
-
it 'runs blocks and functions' do
|
126
|
-
@job = BeforePerformJob.new
|
127
|
-
@job.perform_now
|
128
|
-
assert @job.completed?, @job.attributes.ai
|
129
|
-
expected = %w[before_perform_block before_perform_method perform]
|
130
|
-
assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
describe '#after_perform' do
|
135
|
-
it 'runs blocks and functions' do
|
136
|
-
@job = AfterPerformJob.new
|
137
|
-
@job.perform_now
|
138
|
-
assert @job.completed?, @job.attributes.ai
|
139
|
-
expected = %w[perform after_perform_method after_perform_block]
|
140
|
-
assert_equal expected, @job.call_list, 'Sequence of after_perform callbacks is incorrect'
|
141
|
-
end
|
142
|
-
end
|
143
|
-
|
144
|
-
describe '#around_perform' do
|
145
|
-
it 'runs blocks and functions' do
|
146
|
-
@job = AroundPerformJob.new
|
147
|
-
@job.perform_now
|
148
|
-
assert @job.completed?, @job.attributes.ai
|
149
|
-
expected = %w[around_perform_block_before around_perform_method_before perform around_perform_method_after around_perform_block_after]
|
150
|
-
assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
|
151
|
-
end
|
152
|
-
end
|
153
|
-
|
154
|
-
describe 'all callbacks' do
|
155
|
-
it 'runs them in the right order' do
|
156
|
-
@job = CombinedPerformJob.new
|
157
|
-
@job.perform_now
|
158
|
-
assert @job.completed?, @job.attributes.ai
|
159
|
-
expected = %w[before_perform_block around_perform_block_before before_perform_method around_perform_method_before perform after_perform_method around_perform_method_after around_perform_block_after after_perform_block]
|
160
|
-
assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
|
161
|
-
end
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
165
|
-
end
|
166
|
-
end
|
@@ -1,53 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class DefaultsTest < Minitest::Test
|
7
|
-
class ParentJob < RocketJob::Job
|
8
|
-
self.priority = 53
|
9
|
-
self.description = 'Hello'
|
10
|
-
|
11
|
-
def perform
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
class ChildJob < ParentJob
|
16
|
-
self.priority = 72
|
17
|
-
|
18
|
-
def perform
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
describe RocketJob::Plugins::Job do
|
23
|
-
after do
|
24
|
-
@job.destroy if @job && !@job.new_record?
|
25
|
-
end
|
26
|
-
|
27
|
-
describe '.rocket_job' do
|
28
|
-
it 'sets defaults after initialize' do
|
29
|
-
@job = ParentJob.new
|
30
|
-
assert_equal 53, @job.priority
|
31
|
-
assert_equal 'Hello', @job.description
|
32
|
-
end
|
33
|
-
|
34
|
-
it 'can override defaults on initialize' do
|
35
|
-
@job = ParentJob.new(priority: 72, description: 'More')
|
36
|
-
assert_equal 72, @job.priority
|
37
|
-
assert_equal 'More', @job.description
|
38
|
-
end
|
39
|
-
|
40
|
-
it 'allows a child to override parent defaults' do
|
41
|
-
@job = ChildJob.new
|
42
|
-
assert_equal 72, @job.priority
|
43
|
-
end
|
44
|
-
|
45
|
-
it 'passes down parent defaults' do
|
46
|
-
@job = ChildJob.new
|
47
|
-
assert_equal 'Hello', @job.description
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
@@ -1,56 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
# Unit Test for RocketJob::Job
|
4
|
-
module Plugins
|
5
|
-
module Job
|
6
|
-
class LoggerTest < Minitest::Test
|
7
|
-
class LoggerJob < RocketJob::Job
|
8
|
-
def perform
|
9
|
-
logger.debug('DONE', value: 123, other_value: 'HI')
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
describe RocketJob::Plugins::Job::Logger do
|
14
|
-
before do
|
15
|
-
LoggerJob.delete_all
|
16
|
-
end
|
17
|
-
|
18
|
-
after do
|
19
|
-
@job.destroy if @job && !@job.new_record?
|
20
|
-
SemanticLogger.flush
|
21
|
-
end
|
22
|
-
|
23
|
-
describe '#logger' do
|
24
|
-
it 'uses semantic logger' do
|
25
|
-
@job = LoggerJob.new
|
26
|
-
assert_kind_of SemanticLogger::Logger, @job.logger, @job.logger.ai
|
27
|
-
assert_equal @job.class.name, @job.logger.name, @job.logger.ai
|
28
|
-
end
|
29
|
-
|
30
|
-
it 'allows perform to log its own data' do
|
31
|
-
@job = LoggerJob.new
|
32
|
-
@job.perform_now
|
33
|
-
end
|
34
|
-
|
35
|
-
it 'adds start logging' do
|
36
|
-
@job = LoggerJob.new
|
37
|
-
info_called = false
|
38
|
-
@job.logger.stub(:info, ->(description) { info_called = true if description == 'Start #perform' }) do
|
39
|
-
@job.perform_now
|
40
|
-
end
|
41
|
-
assert info_called, "In Plugins::Job::Logger.around_perform logger.info('Start #perform') not called"
|
42
|
-
end
|
43
|
-
|
44
|
-
it 'adds completed logging' do
|
45
|
-
@job = LoggerJob.new
|
46
|
-
measure_called = false
|
47
|
-
@job.logger.stub(:measure_info, ->(description, *_args) { measure_called = true if description == 'Completed #perform' }) do
|
48
|
-
@job.perform_now
|
49
|
-
end
|
50
|
-
assert measure_called, "In Plugins::Job::Logger.around_perform logger.measure_info('Completed #perform') not called"
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,94 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class ModelTest < Minitest::Test
|
7
|
-
class SimpleJob < RocketJob::Job
|
8
|
-
def perform
|
9
|
-
10
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
class TwoArgumentJob < RocketJob::Job
|
14
|
-
self.priority = 53
|
15
|
-
|
16
|
-
def perform(a, b)
|
17
|
-
a + b
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
describe RocketJob::Plugins::Job::Model do
|
22
|
-
after do
|
23
|
-
@job.destroy if @job && !@job.new_record?
|
24
|
-
@job2.destroy if @job2 && !@job2.new_record?
|
25
|
-
@job3.destroy if @job3 && !@job3.new_record?
|
26
|
-
end
|
27
|
-
|
28
|
-
describe '#scheduled?' do
|
29
|
-
it 'returns true if job is queued to run in the future' do
|
30
|
-
@job = SimpleJob.new(run_at: 1.day.from_now)
|
31
|
-
assert_equal true, @job.queued?
|
32
|
-
assert_equal true, @job.scheduled?
|
33
|
-
@job.start
|
34
|
-
assert_equal true, @job.running?
|
35
|
-
assert_equal false, @job.scheduled?
|
36
|
-
end
|
37
|
-
|
38
|
-
it 'returns false if job is queued and can be run now' do
|
39
|
-
@job = SimpleJob.new
|
40
|
-
assert_equal true, @job.queued?
|
41
|
-
assert_equal false, @job.scheduled?
|
42
|
-
end
|
43
|
-
|
44
|
-
it 'returns false if job is running' do
|
45
|
-
@job = SimpleJob.new
|
46
|
-
@job.start
|
47
|
-
assert_equal true, @job.running?
|
48
|
-
assert_equal false, @job.scheduled?
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
describe 'with queued jobs' do
|
53
|
-
before do
|
54
|
-
@job = SimpleJob.create!(description: 'first')
|
55
|
-
@job2 = SimpleJob.create!(description: 'second', run_at: 1.day.from_now)
|
56
|
-
@job3 = SimpleJob.create!(description: 'third', run_at: 2.days.from_now)
|
57
|
-
end
|
58
|
-
|
59
|
-
describe '#scheduled' do
|
60
|
-
it 'returns only scheduled jobs' do
|
61
|
-
count = 0
|
62
|
-
RocketJob::Job.scheduled.each do |job|
|
63
|
-
count += 1
|
64
|
-
assert job.scheduled?
|
65
|
-
end
|
66
|
-
assert 2, count
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
describe '#queued_now' do
|
71
|
-
it 'returns only queued jobs, not scheduled ones' do
|
72
|
-
count = 0
|
73
|
-
RocketJob::Job.queued_now.each do |job|
|
74
|
-
count += 1
|
75
|
-
refute job.scheduled?
|
76
|
-
end
|
77
|
-
assert 1, count
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
describe '#queued' do
|
82
|
-
it 'returns all queued jobs' do
|
83
|
-
count = 0
|
84
|
-
RocketJob::Job.queued.each do |_job|
|
85
|
-
count += 1
|
86
|
-
end
|
87
|
-
assert 3, count
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
@@ -1,94 +0,0 @@
|
|
1
|
-
require_relative '../../test_helper'
|
2
|
-
|
3
|
-
module Plugins
|
4
|
-
module Job
|
5
|
-
# Unit Test for RocketJob::Job
|
6
|
-
class PersistenceTest < Minitest::Test
|
7
|
-
class PersistJob < RocketJob::Job
|
8
|
-
self.priority = 53
|
9
|
-
field :data, type: Hash
|
10
|
-
|
11
|
-
def perform(hash)
|
12
|
-
hash
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
describe RocketJob::Plugins::Job::Persistence do
|
17
|
-
before do
|
18
|
-
RocketJob::Job.destroy_all
|
19
|
-
@description = 'Hello World'
|
20
|
-
@data = {'key' => 'value'}
|
21
|
-
@job = PersistJob.new(
|
22
|
-
description: @description,
|
23
|
-
data: @data,
|
24
|
-
destroy_on_complete: false
|
25
|
-
)
|
26
|
-
end
|
27
|
-
|
28
|
-
after do
|
29
|
-
@job.destroy if @job && !@job.new_record?
|
30
|
-
@job2.destroy if @job2 && !@job2.new_record?
|
31
|
-
@job3.destroy if @job3 && !@job3.new_record?
|
32
|
-
end
|
33
|
-
|
34
|
-
describe '.config' do
|
35
|
-
it 'support multiple databases' do
|
36
|
-
assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
describe '.rocket_job' do
|
41
|
-
it 'sets defaults after initialize' do
|
42
|
-
assert_equal 53, @job.priority
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
describe '#reload' do
|
47
|
-
it 'handle hash' do
|
48
|
-
assert_equal 'value', @job.data['key']
|
49
|
-
@job.worker_name = nil
|
50
|
-
@job.save!
|
51
|
-
@job.worker_name = '123'
|
52
|
-
@job.reload
|
53
|
-
assert @job.data.is_a?(Hash), @job.data.class.ai
|
54
|
-
assert_equal 'value', @job.data['key']
|
55
|
-
assert_nil @job.worker_name
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
describe '#save!' do
|
60
|
-
it 'save a blank job' do
|
61
|
-
@job.save!
|
62
|
-
assert_nil @job.worker_name
|
63
|
-
assert_nil @job.completed_at
|
64
|
-
assert @job.created_at
|
65
|
-
assert_equal @description, @job.description
|
66
|
-
assert_equal false, @job.destroy_on_complete
|
67
|
-
assert_nil @job.expires_at
|
68
|
-
assert_equal @data, @job.data
|
69
|
-
assert_equal 0, @job.percent_complete
|
70
|
-
assert_equal 53, @job.priority
|
71
|
-
assert_equal 0, @job.failure_count
|
72
|
-
assert_nil @job.run_at
|
73
|
-
assert_nil @job.started_at
|
74
|
-
assert_equal :queued, @job.state
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
describe '.counts_by_state' do
|
79
|
-
it 'returns states as symbols' do
|
80
|
-
@job.start!
|
81
|
-
@job2 = PersistJob.create!(data: {key: 'value'})
|
82
|
-
@job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
|
83
|
-
counts = RocketJob::Job.counts_by_state
|
84
|
-
assert_equal 4, counts.size, counts.ai
|
85
|
-
assert_equal 1, counts[:running]
|
86
|
-
assert_equal 2, counts[:queued]
|
87
|
-
assert_equal 1, counts[:queued_now]
|
88
|
-
assert_equal 1, counts[:scheduled]
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|