rocketjob 2.1.3 → 3.0.0.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +36 -0
- data/lib/rocket_job/active_server.rb +48 -0
- data/lib/rocket_job/cli.rb +29 -17
- data/lib/rocket_job/config.rb +19 -31
- data/lib/rocket_job/dirmon_entry.rb +15 -45
- data/lib/rocket_job/extensions/mongo/logging.rb +26 -0
- data/lib/rocket_job/extensions/rocket_job_adapter.rb +3 -5
- data/lib/rocket_job/heartbeat.rb +18 -23
- data/lib/rocket_job/job.rb +0 -1
- data/lib/rocket_job/job_exception.rb +11 -13
- data/lib/rocket_job/jobs/dirmon_job.rb +8 -8
- data/lib/rocket_job/jobs/housekeeping_job.rb +13 -15
- data/lib/rocket_job/performance.rb +5 -5
- data/lib/rocket_job/plugins/cron.rb +3 -10
- data/lib/rocket_job/plugins/document.rb +58 -33
- data/lib/rocket_job/plugins/job/model.rb +43 -71
- data/lib/rocket_job/plugins/job/persistence.rb +7 -63
- data/lib/rocket_job/plugins/job/worker.rb +24 -26
- data/lib/rocket_job/plugins/processing_window.rb +6 -9
- data/lib/rocket_job/plugins/retry.rb +3 -8
- data/lib/rocket_job/plugins/singleton.rb +1 -1
- data/lib/rocket_job/plugins/state_machine.rb +1 -7
- data/lib/rocket_job/server.rb +352 -0
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +46 -336
- data/lib/rocketjob.rb +5 -4
- data/test/config/mongoid.yml +88 -0
- data/test/config_test.rb +1 -1
- data/test/dirmon_entry_test.rb +15 -79
- data/test/dirmon_job_test.rb +6 -6
- data/test/job_test.rb +2 -2
- data/test/plugins/job/callbacks_test.rb +40 -32
- data/test/plugins/job/defaults_test.rb +10 -8
- data/test/plugins/job/model_test.rb +1 -3
- data/test/plugins/job/persistence_test.rb +11 -13
- data/test/plugins/job/worker_test.rb +45 -26
- data/test/plugins/processing_window_test.rb +4 -4
- data/test/plugins/restart_test.rb +11 -12
- data/test/plugins/state_machine_event_callbacks_test.rb +20 -18
- data/test/plugins/state_machine_test.rb +5 -5
- data/test/test_helper.rb +4 -1
- metadata +15 -29
- data/lib/rocket_job/extensions/mongo.rb +0 -23
- data/lib/rocket_job/extensions/mongo_mapper.rb +0 -30
- data/lib/rocket_job/plugins/job/defaults.rb +0 -40
- data/test/config/mongo.yml +0 -46
@@ -6,25 +6,21 @@ module Plugins
|
|
6
6
|
class DefaultsTest < Minitest::Test
|
7
7
|
|
8
8
|
class ParentJob < RocketJob::Job
|
9
|
-
|
10
|
-
|
11
|
-
job.description = 'Hello'
|
12
|
-
end
|
9
|
+
self.priority = 53
|
10
|
+
self.description = 'Hello'
|
13
11
|
|
14
12
|
def perform
|
15
13
|
end
|
16
14
|
end
|
17
15
|
|
18
16
|
class ChildJob < ParentJob
|
19
|
-
|
20
|
-
job.priority = 72
|
21
|
-
end
|
17
|
+
self.priority = 72
|
22
18
|
|
23
19
|
def perform
|
24
20
|
end
|
25
21
|
end
|
26
22
|
|
27
|
-
describe RocketJob::Plugins::Job
|
23
|
+
describe RocketJob::Plugins::Job do
|
28
24
|
after do
|
29
25
|
@job.destroy if @job && !@job.new_record?
|
30
26
|
end
|
@@ -36,6 +32,12 @@ module Plugins
|
|
36
32
|
assert_equal 'Hello', @job.description
|
37
33
|
end
|
38
34
|
|
35
|
+
it 'can override defaults on initialize' do
|
36
|
+
@job = ParentJob.new(priority: 72, description: 'More')
|
37
|
+
assert_equal 72, @job.priority
|
38
|
+
assert_equal 'More', @job.description
|
39
|
+
end
|
40
|
+
|
39
41
|
it 'allows a child to override parent defaults' do
|
40
42
|
@job = ChildJob.new
|
41
43
|
assert_equal 72, @job.priority
|
@@ -6,9 +6,8 @@ module Plugins
|
|
6
6
|
class PersistenceTest < Minitest::Test
|
7
7
|
|
8
8
|
class PersistJob < RocketJob::Job
|
9
|
-
|
10
|
-
|
11
|
-
end
|
9
|
+
self.priority = 53
|
10
|
+
field :data, type: Hash
|
12
11
|
|
13
12
|
def perform(hash)
|
14
13
|
hash
|
@@ -19,10 +18,10 @@ module Plugins
|
|
19
18
|
before do
|
20
19
|
RocketJob::Job.destroy_all
|
21
20
|
@description = 'Hello World'
|
22
|
-
@
|
21
|
+
@data = {'key' => 'value'}
|
23
22
|
@job = PersistJob.new(
|
24
23
|
description: @description,
|
25
|
-
|
24
|
+
data: @data,
|
26
25
|
destroy_on_complete: false
|
27
26
|
)
|
28
27
|
end
|
@@ -35,7 +34,7 @@ module Plugins
|
|
35
34
|
|
36
35
|
describe '.config' do
|
37
36
|
it 'support multiple databases' do
|
38
|
-
assert_equal '
|
37
|
+
assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
|
39
38
|
end
|
40
39
|
end
|
41
40
|
|
@@ -47,14 +46,13 @@ module Plugins
|
|
47
46
|
|
48
47
|
describe '#reload' do
|
49
48
|
it 'handle hash' do
|
50
|
-
assert_equal 'value', @job.
|
49
|
+
assert_equal 'value', @job.data['key']
|
51
50
|
@job.worker_name = nil
|
52
51
|
@job.save!
|
53
52
|
@job.worker_name = '123'
|
54
53
|
@job.reload
|
55
|
-
assert @job.
|
56
|
-
assert_equal 'value', @job.
|
57
|
-
assert_equal 'value', @job.arguments.first[:key]
|
54
|
+
assert @job.data.is_a?(Hash), @job.data.class.ai
|
55
|
+
assert_equal 'value', @job.data['key']
|
58
56
|
assert_equal nil, @job.worker_name
|
59
57
|
end
|
60
58
|
end
|
@@ -68,7 +66,7 @@ module Plugins
|
|
68
66
|
assert_equal @description, @job.description
|
69
67
|
assert_equal false, @job.destroy_on_complete
|
70
68
|
assert_nil @job.expires_at
|
71
|
-
assert_equal @
|
69
|
+
assert_equal @data, @job.data
|
72
70
|
assert_equal 0, @job.percent_complete
|
73
71
|
assert_equal 53, @job.priority
|
74
72
|
assert_equal 0, @job.failure_count
|
@@ -81,8 +79,8 @@ module Plugins
|
|
81
79
|
describe '.counts_by_state' do
|
82
80
|
it 'returns states as symbols' do
|
83
81
|
@job.start!
|
84
|
-
@job2
|
85
|
-
@job3
|
82
|
+
@job2 = PersistJob.create!(data: {key: 'value'})
|
83
|
+
@job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
|
86
84
|
counts = RocketJob::Job.counts_by_state
|
87
85
|
assert_equal 4, counts.size, counts.ai
|
88
86
|
assert_equal 1, counts[:running]
|
@@ -19,20 +19,22 @@ module Plugins
|
|
19
19
|
end
|
20
20
|
|
21
21
|
class SumJob < RocketJob::Job
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
22
|
+
self.destroy_on_complete = false
|
23
|
+
self.collect_output = true
|
24
|
+
self.priority = 51
|
25
|
+
|
26
|
+
field :first, type: Integer
|
27
|
+
field :second, type: Integer
|
27
28
|
|
28
|
-
def perform
|
29
|
+
def perform
|
29
30
|
first + second
|
30
31
|
end
|
31
32
|
end
|
32
33
|
|
33
34
|
describe RocketJob::Plugins::Job::Worker do
|
34
35
|
before do
|
35
|
-
RocketJob::Job.
|
36
|
+
RocketJob::Job.delete_all
|
37
|
+
RocketJob::Server.delete_all
|
36
38
|
end
|
37
39
|
|
38
40
|
after do
|
@@ -42,7 +44,7 @@ module Plugins
|
|
42
44
|
describe '.rocket_job_next_job' do
|
43
45
|
before do
|
44
46
|
@job = QuietJob.new
|
45
|
-
@worker = RocketJob::
|
47
|
+
@worker = RocketJob::Server.new(name: 'worker:123')
|
46
48
|
end
|
47
49
|
|
48
50
|
it 'return nil when no jobs available' do
|
@@ -79,25 +81,17 @@ module Plugins
|
|
79
81
|
|
80
82
|
describe '#perform_now' do
|
81
83
|
it 'calls perform method' do
|
82
|
-
@job = SumJob.new(
|
84
|
+
@job = SumJob.new(first: 10, second: 5)
|
83
85
|
assert_equal 15, @job.perform_now['result']
|
84
86
|
assert @job.completed?, @job.attributes.ai
|
85
87
|
assert_equal 15, @job.result['result']
|
86
88
|
end
|
87
89
|
|
88
|
-
it '
|
89
|
-
@job = SumJob.new(
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
assert @job.exception.backtrace
|
94
|
-
assert_equal 'TypeError', @job.exception.class_name
|
95
|
-
if RUBY_VERSION.to_f < 2.0
|
96
|
-
assert_equal "can't convert Fixnum into String", @job.exception.message
|
97
|
-
else
|
98
|
-
assert_equal 'no implicit conversion of Fixnum into String', @job.exception.message
|
99
|
-
end
|
100
|
-
assert_equal 'inline', @job.exception.worker_name
|
90
|
+
it 'converts type' do
|
91
|
+
@job = SumJob.new(first: '10', second: 5)
|
92
|
+
assert_equal 15, @job.perform_now['result']
|
93
|
+
assert @job.completed?, @job.attributes.ai
|
94
|
+
assert_equal 15, @job.result['result']
|
101
95
|
end
|
102
96
|
|
103
97
|
it 'silence logging when log_level is set' do
|
@@ -127,14 +121,14 @@ module Plugins
|
|
127
121
|
describe '.perform_later' do
|
128
122
|
it 'queues the job for processing' do
|
129
123
|
RocketJob::Config.stub(:inline_mode, false) do
|
130
|
-
@job = SumJob.perform_later(1, 23)
|
124
|
+
@job = SumJob.perform_later(first: 1, second: 23)
|
131
125
|
end
|
132
126
|
assert @job.queued?
|
133
127
|
|
134
128
|
# Manually run the job
|
135
129
|
@job.perform_now
|
136
130
|
assert @job.completed?, @job.attributes.ai
|
137
|
-
assert_equal 24, @job.result['result']
|
131
|
+
assert_equal 24, @job.result['result'], -> { @job.result.ai }
|
138
132
|
|
139
133
|
assert_nil @job.worker_name
|
140
134
|
assert @job.completed_at
|
@@ -150,7 +144,7 @@ module Plugins
|
|
150
144
|
|
151
145
|
it 'runs the job immediately when inline_mode = true' do
|
152
146
|
RocketJob::Config.stub(:inline_mode, true) do
|
153
|
-
@job = SumJob.perform_later(1, 23)
|
147
|
+
@job = SumJob.perform_later(first: 1, second: 23)
|
154
148
|
end
|
155
149
|
|
156
150
|
assert @job.completed?, @job.attributes.ai
|
@@ -171,12 +165,37 @@ module Plugins
|
|
171
165
|
|
172
166
|
describe '.perform_now' do
|
173
167
|
it 'run the job immediately' do
|
174
|
-
@job = SumJob.perform_now(1, 5)
|
168
|
+
@job = SumJob.perform_now(first: 1, second: 5)
|
175
169
|
assert_equal true, @job.completed?
|
176
170
|
assert_equal 6, @job.result['result']
|
177
171
|
end
|
178
172
|
end
|
179
173
|
|
174
|
+
describe '#rocket_job_active_servers' do
|
175
|
+
before do
|
176
|
+
@job = QuietJob.create!
|
177
|
+
@worker = RocketJob::Server.create!(name: 'worker:123')
|
178
|
+
end
|
179
|
+
|
180
|
+
it 'should return empty hash for no active jobs' do
|
181
|
+
assert_equal({}, @job.rocket_job_active_servers)
|
182
|
+
end
|
183
|
+
|
184
|
+
it 'should return active servers' do
|
185
|
+
assert job = RocketJob::Job.rocket_job_next_job(@worker.name)
|
186
|
+
assert active = job.rocket_job_active_servers
|
187
|
+
assert_equal 1, active.size
|
188
|
+
assert active_servers = active[@worker.name]
|
189
|
+
assert_equal 1, active_servers.size
|
190
|
+
assert active_server = active_servers.first
|
191
|
+
assert_equal @job.id, active_server.job.id
|
192
|
+
assert_equal @worker.name, active_server.name
|
193
|
+
assert_equal job.started_at, active_server.started_at
|
194
|
+
assert active_server.duration_s
|
195
|
+
assert active_server.duration
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
180
199
|
end
|
181
200
|
end
|
182
201
|
end
|
@@ -55,8 +55,8 @@ module Plugins
|
|
55
55
|
|
56
56
|
describe '#rocket_job_processing_window_active?' do
|
57
57
|
it 'returns true when in the processing window' do
|
58
|
-
time
|
59
|
-
@job
|
58
|
+
time = Time.parse('2015-12-09 17:50:05 +0000')
|
59
|
+
@job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
|
60
60
|
result = Time.stub(:now, time) do
|
61
61
|
@job.rocket_job_processing_window_active?
|
62
62
|
end
|
@@ -64,8 +64,8 @@ module Plugins
|
|
64
64
|
end
|
65
65
|
|
66
66
|
it 'returns false when not in the processing window' do
|
67
|
-
time
|
68
|
-
@job
|
67
|
+
time = Time.parse('2015-12-09 16:50:05 +0000')
|
68
|
+
@job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
|
69
69
|
result = Time.stub(:now, time) do
|
70
70
|
@job.rocket_job_processing_window_active?
|
71
71
|
end
|
@@ -7,10 +7,10 @@ module Plugins
|
|
7
7
|
include RocketJob::Plugins::Restart
|
8
8
|
|
9
9
|
# Ensure a new start_at and end_at is generated every time this job is restarted
|
10
|
-
self.rocket_job_restart_excludes
|
10
|
+
self.rocket_job_restart_excludes += %w(start_at end_at)
|
11
11
|
|
12
|
-
|
13
|
-
|
12
|
+
field :start_at, type: Date
|
13
|
+
field :end_at, type: Date
|
14
14
|
|
15
15
|
def perform
|
16
16
|
self.start_at = Date.today
|
@@ -28,7 +28,6 @@ module Plugins
|
|
28
28
|
|
29
29
|
after do
|
30
30
|
@job.delete if @job && !@job.new_record?
|
31
|
-
RestartableJob.delete_all
|
32
31
|
end
|
33
32
|
|
34
33
|
describe '#create!' do
|
@@ -53,7 +52,7 @@ module Plugins
|
|
53
52
|
@job = RestartableJob.create!
|
54
53
|
@job.abort!
|
55
54
|
assert_equal 2, RestartableJob.count
|
56
|
-
assert other = RestartableJob.where(id
|
55
|
+
assert other = RestartableJob.where(:id.ne => @job.id).first
|
57
56
|
refute_equal @job.id, other.id
|
58
57
|
assert other.queued?
|
59
58
|
end
|
@@ -63,7 +62,7 @@ module Plugins
|
|
63
62
|
assert @job.expired?
|
64
63
|
@job.abort!
|
65
64
|
assert_equal 1, RestartableJob.count
|
66
|
-
assert_equal nil, RestartableJob.where(id
|
65
|
+
assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
|
67
66
|
end
|
68
67
|
end
|
69
68
|
|
@@ -133,7 +132,7 @@ module Plugins
|
|
133
132
|
assert @job.running?
|
134
133
|
assert @job.expired?
|
135
134
|
assert_equal 1, RestartableJob.count
|
136
|
-
assert_equal nil, RestartableJob.where(id
|
135
|
+
assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
|
137
136
|
end
|
138
137
|
end
|
139
138
|
|
@@ -142,7 +141,7 @@ module Plugins
|
|
142
141
|
@job = RestartableJob.create!(destroy_on_complete: true)
|
143
142
|
@job.perform_now
|
144
143
|
assert_equal 1, RestartableJob.count
|
145
|
-
assert job2 = RestartableJob.where(id
|
144
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
146
145
|
assert job2.queued?, job2.attributes.ai
|
147
146
|
end
|
148
147
|
|
@@ -151,7 +150,7 @@ module Plugins
|
|
151
150
|
refute @job.expired?
|
152
151
|
@job.perform_now
|
153
152
|
assert_equal 1, RestartableJob.count
|
154
|
-
assert job2 = RestartableJob.where(id
|
153
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
155
154
|
assert job2.queued?, job2.attributes.ai
|
156
155
|
|
157
156
|
# Copy across all attributes, except
|
@@ -171,14 +170,14 @@ module Plugins
|
|
171
170
|
assert_equal nil, job2.worker_name
|
172
171
|
assert_equal 0, job2.percent_complete
|
173
172
|
assert_equal nil, job2.exception
|
174
|
-
|
173
|
+
refute job2.result
|
175
174
|
end
|
176
175
|
|
177
176
|
it 'copies run_at when it is in the future' do
|
178
177
|
@job = RestartableJob.create!(run_at: Time.now + 1.day, destroy_on_complete: true)
|
179
178
|
@job.perform_now
|
180
179
|
assert_equal 1, RestartableJob.count
|
181
|
-
assert job2 = RestartableJob.where(id
|
180
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
182
181
|
assert job2.run_at, job2.attributes.ai
|
183
182
|
end
|
184
183
|
|
@@ -186,7 +185,7 @@ module Plugins
|
|
186
185
|
@job = RestartableJob.create!(run_at: Time.now - 1.day, destroy_on_complete: true)
|
187
186
|
@job.perform_now
|
188
187
|
assert_equal 1, RestartableJob.count
|
189
|
-
assert job2 = RestartableJob.where(id
|
188
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
190
189
|
assert_equal nil, job2.run_at
|
191
190
|
end
|
192
191
|
end
|
@@ -6,78 +6,80 @@ module Plugins
|
|
6
6
|
# This job adds each event callback as they run into an array
|
7
7
|
# [:start, :complete, :fail, :retry, :pause, :resume, :abort, :requeue]
|
8
8
|
class PositivePathJob < RocketJob::Job
|
9
|
+
field :call_list, type: Array, default: []
|
10
|
+
|
9
11
|
before_complete do
|
10
|
-
|
12
|
+
call_list << 'before_complete_block'
|
11
13
|
end
|
12
14
|
|
13
15
|
after_complete do
|
14
|
-
|
16
|
+
call_list << 'after_complete_block'
|
15
17
|
end
|
16
18
|
|
17
19
|
before_complete :before_complete_method
|
18
20
|
|
19
21
|
before_start do
|
20
|
-
|
22
|
+
call_list << 'before_start_block'
|
21
23
|
end
|
22
24
|
|
23
25
|
before_start :before_start_method
|
24
26
|
|
25
27
|
before_start do
|
26
|
-
|
28
|
+
call_list << 'before_start2_block'
|
27
29
|
end
|
28
30
|
|
29
31
|
after_start :after_start_method
|
30
32
|
after_complete :after_complete_method
|
31
33
|
|
32
34
|
before_complete do
|
33
|
-
|
35
|
+
call_list << 'before_complete2_block'
|
34
36
|
end
|
35
37
|
|
36
38
|
after_start do
|
37
|
-
|
39
|
+
call_list << 'after_start_block'
|
38
40
|
end
|
39
41
|
|
40
42
|
after_complete do
|
41
|
-
|
43
|
+
call_list << 'after_complete2_block'
|
42
44
|
end
|
43
45
|
|
44
46
|
after_start :after_start_method
|
45
47
|
|
46
48
|
after_start do
|
47
|
-
|
49
|
+
call_list << 'after_start2_block'
|
48
50
|
end
|
49
51
|
|
50
52
|
before_start :before_start_method2
|
51
53
|
before_complete :before_complete_method2
|
52
54
|
|
53
|
-
def perform
|
54
|
-
|
55
|
+
def perform
|
56
|
+
call_list << 'perform'
|
55
57
|
end
|
56
58
|
|
57
59
|
private
|
58
60
|
|
59
61
|
def before_start_method
|
60
|
-
|
62
|
+
call_list << 'before_start_method'
|
61
63
|
end
|
62
64
|
|
63
65
|
def before_start_method2
|
64
|
-
|
66
|
+
call_list << 'before_start_method2'
|
65
67
|
end
|
66
68
|
|
67
69
|
def after_start_method
|
68
|
-
|
70
|
+
call_list << 'after_start_method'
|
69
71
|
end
|
70
72
|
|
71
73
|
def before_complete_method
|
72
|
-
|
74
|
+
call_list << 'before_complete_method'
|
73
75
|
end
|
74
76
|
|
75
77
|
def before_complete_method2
|
76
|
-
|
78
|
+
call_list << 'before_complete_method2'
|
77
79
|
end
|
78
80
|
|
79
81
|
def after_complete_method
|
80
|
-
|
82
|
+
call_list << 'after_complete_method'
|
81
83
|
end
|
82
84
|
end
|
83
85
|
|
@@ -88,11 +90,11 @@ module Plugins
|
|
88
90
|
|
89
91
|
describe 'before_start after_start & before_complete after_complete' do
|
90
92
|
it 'runs blocks and functions' do
|
91
|
-
@job = PositivePathJob.new
|
93
|
+
@job = PositivePathJob.new
|
92
94
|
@job.perform_now
|
93
95
|
assert @job.completed?, @job.attributes.ai
|
94
96
|
expected = %w(before_start_block before_start_method before_start2_block before_start_method2 after_start2_block after_start_method after_start_block perform before_complete_block before_complete_method before_complete2_block before_complete_method2 after_complete2_block after_complete_method after_complete_block)
|
95
|
-
assert_equal expected, @job.
|
97
|
+
assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
|
96
98
|
end
|
97
99
|
end
|
98
100
|
|