rocketjob 2.1.3 → 3.0.0.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +36 -0
- data/lib/rocket_job/active_server.rb +48 -0
- data/lib/rocket_job/cli.rb +29 -17
- data/lib/rocket_job/config.rb +19 -31
- data/lib/rocket_job/dirmon_entry.rb +15 -45
- data/lib/rocket_job/extensions/mongo/logging.rb +26 -0
- data/lib/rocket_job/extensions/rocket_job_adapter.rb +3 -5
- data/lib/rocket_job/heartbeat.rb +18 -23
- data/lib/rocket_job/job.rb +0 -1
- data/lib/rocket_job/job_exception.rb +11 -13
- data/lib/rocket_job/jobs/dirmon_job.rb +8 -8
- data/lib/rocket_job/jobs/housekeeping_job.rb +13 -15
- data/lib/rocket_job/performance.rb +5 -5
- data/lib/rocket_job/plugins/cron.rb +3 -10
- data/lib/rocket_job/plugins/document.rb +58 -33
- data/lib/rocket_job/plugins/job/model.rb +43 -71
- data/lib/rocket_job/plugins/job/persistence.rb +7 -63
- data/lib/rocket_job/plugins/job/worker.rb +24 -26
- data/lib/rocket_job/plugins/processing_window.rb +6 -9
- data/lib/rocket_job/plugins/retry.rb +3 -8
- data/lib/rocket_job/plugins/singleton.rb +1 -1
- data/lib/rocket_job/plugins/state_machine.rb +1 -7
- data/lib/rocket_job/server.rb +352 -0
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +46 -336
- data/lib/rocketjob.rb +5 -4
- data/test/config/mongoid.yml +88 -0
- data/test/config_test.rb +1 -1
- data/test/dirmon_entry_test.rb +15 -79
- data/test/dirmon_job_test.rb +6 -6
- data/test/job_test.rb +2 -2
- data/test/plugins/job/callbacks_test.rb +40 -32
- data/test/plugins/job/defaults_test.rb +10 -8
- data/test/plugins/job/model_test.rb +1 -3
- data/test/plugins/job/persistence_test.rb +11 -13
- data/test/plugins/job/worker_test.rb +45 -26
- data/test/plugins/processing_window_test.rb +4 -4
- data/test/plugins/restart_test.rb +11 -12
- data/test/plugins/state_machine_event_callbacks_test.rb +20 -18
- data/test/plugins/state_machine_test.rb +5 -5
- data/test/test_helper.rb +4 -1
- metadata +15 -29
- data/lib/rocket_job/extensions/mongo.rb +0 -23
- data/lib/rocket_job/extensions/mongo_mapper.rb +0 -30
- data/lib/rocket_job/plugins/job/defaults.rb +0 -40
- data/test/config/mongo.yml +0 -46
@@ -6,25 +6,21 @@ module Plugins
|
|
6
6
|
class DefaultsTest < Minitest::Test
|
7
7
|
|
8
8
|
class ParentJob < RocketJob::Job
|
9
|
-
|
10
|
-
|
11
|
-
job.description = 'Hello'
|
12
|
-
end
|
9
|
+
self.priority = 53
|
10
|
+
self.description = 'Hello'
|
13
11
|
|
14
12
|
def perform
|
15
13
|
end
|
16
14
|
end
|
17
15
|
|
18
16
|
class ChildJob < ParentJob
|
19
|
-
|
20
|
-
job.priority = 72
|
21
|
-
end
|
17
|
+
self.priority = 72
|
22
18
|
|
23
19
|
def perform
|
24
20
|
end
|
25
21
|
end
|
26
22
|
|
27
|
-
describe RocketJob::Plugins::Job
|
23
|
+
describe RocketJob::Plugins::Job do
|
28
24
|
after do
|
29
25
|
@job.destroy if @job && !@job.new_record?
|
30
26
|
end
|
@@ -36,6 +32,12 @@ module Plugins
|
|
36
32
|
assert_equal 'Hello', @job.description
|
37
33
|
end
|
38
34
|
|
35
|
+
it 'can override defaults on initialize' do
|
36
|
+
@job = ParentJob.new(priority: 72, description: 'More')
|
37
|
+
assert_equal 72, @job.priority
|
38
|
+
assert_equal 'More', @job.description
|
39
|
+
end
|
40
|
+
|
39
41
|
it 'allows a child to override parent defaults' do
|
40
42
|
@job = ChildJob.new
|
41
43
|
assert_equal 72, @job.priority
|
@@ -6,9 +6,8 @@ module Plugins
|
|
6
6
|
class PersistenceTest < Minitest::Test
|
7
7
|
|
8
8
|
class PersistJob < RocketJob::Job
|
9
|
-
|
10
|
-
|
11
|
-
end
|
9
|
+
self.priority = 53
|
10
|
+
field :data, type: Hash
|
12
11
|
|
13
12
|
def perform(hash)
|
14
13
|
hash
|
@@ -19,10 +18,10 @@ module Plugins
|
|
19
18
|
before do
|
20
19
|
RocketJob::Job.destroy_all
|
21
20
|
@description = 'Hello World'
|
22
|
-
@
|
21
|
+
@data = {'key' => 'value'}
|
23
22
|
@job = PersistJob.new(
|
24
23
|
description: @description,
|
25
|
-
|
24
|
+
data: @data,
|
26
25
|
destroy_on_complete: false
|
27
26
|
)
|
28
27
|
end
|
@@ -35,7 +34,7 @@ module Plugins
|
|
35
34
|
|
36
35
|
describe '.config' do
|
37
36
|
it 'support multiple databases' do
|
38
|
-
assert_equal '
|
37
|
+
assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
|
39
38
|
end
|
40
39
|
end
|
41
40
|
|
@@ -47,14 +46,13 @@ module Plugins
|
|
47
46
|
|
48
47
|
describe '#reload' do
|
49
48
|
it 'handle hash' do
|
50
|
-
assert_equal 'value', @job.
|
49
|
+
assert_equal 'value', @job.data['key']
|
51
50
|
@job.worker_name = nil
|
52
51
|
@job.save!
|
53
52
|
@job.worker_name = '123'
|
54
53
|
@job.reload
|
55
|
-
assert @job.
|
56
|
-
assert_equal 'value', @job.
|
57
|
-
assert_equal 'value', @job.arguments.first[:key]
|
54
|
+
assert @job.data.is_a?(Hash), @job.data.class.ai
|
55
|
+
assert_equal 'value', @job.data['key']
|
58
56
|
assert_equal nil, @job.worker_name
|
59
57
|
end
|
60
58
|
end
|
@@ -68,7 +66,7 @@ module Plugins
|
|
68
66
|
assert_equal @description, @job.description
|
69
67
|
assert_equal false, @job.destroy_on_complete
|
70
68
|
assert_nil @job.expires_at
|
71
|
-
assert_equal @
|
69
|
+
assert_equal @data, @job.data
|
72
70
|
assert_equal 0, @job.percent_complete
|
73
71
|
assert_equal 53, @job.priority
|
74
72
|
assert_equal 0, @job.failure_count
|
@@ -81,8 +79,8 @@ module Plugins
|
|
81
79
|
describe '.counts_by_state' do
|
82
80
|
it 'returns states as symbols' do
|
83
81
|
@job.start!
|
84
|
-
@job2
|
85
|
-
@job3
|
82
|
+
@job2 = PersistJob.create!(data: {key: 'value'})
|
83
|
+
@job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
|
86
84
|
counts = RocketJob::Job.counts_by_state
|
87
85
|
assert_equal 4, counts.size, counts.ai
|
88
86
|
assert_equal 1, counts[:running]
|
@@ -19,20 +19,22 @@ module Plugins
|
|
19
19
|
end
|
20
20
|
|
21
21
|
class SumJob < RocketJob::Job
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
22
|
+
self.destroy_on_complete = false
|
23
|
+
self.collect_output = true
|
24
|
+
self.priority = 51
|
25
|
+
|
26
|
+
field :first, type: Integer
|
27
|
+
field :second, type: Integer
|
27
28
|
|
28
|
-
def perform
|
29
|
+
def perform
|
29
30
|
first + second
|
30
31
|
end
|
31
32
|
end
|
32
33
|
|
33
34
|
describe RocketJob::Plugins::Job::Worker do
|
34
35
|
before do
|
35
|
-
RocketJob::Job.
|
36
|
+
RocketJob::Job.delete_all
|
37
|
+
RocketJob::Server.delete_all
|
36
38
|
end
|
37
39
|
|
38
40
|
after do
|
@@ -42,7 +44,7 @@ module Plugins
|
|
42
44
|
describe '.rocket_job_next_job' do
|
43
45
|
before do
|
44
46
|
@job = QuietJob.new
|
45
|
-
@worker = RocketJob::
|
47
|
+
@worker = RocketJob::Server.new(name: 'worker:123')
|
46
48
|
end
|
47
49
|
|
48
50
|
it 'return nil when no jobs available' do
|
@@ -79,25 +81,17 @@ module Plugins
|
|
79
81
|
|
80
82
|
describe '#perform_now' do
|
81
83
|
it 'calls perform method' do
|
82
|
-
@job = SumJob.new(
|
84
|
+
@job = SumJob.new(first: 10, second: 5)
|
83
85
|
assert_equal 15, @job.perform_now['result']
|
84
86
|
assert @job.completed?, @job.attributes.ai
|
85
87
|
assert_equal 15, @job.result['result']
|
86
88
|
end
|
87
89
|
|
88
|
-
it '
|
89
|
-
@job = SumJob.new(
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
assert @job.exception.backtrace
|
94
|
-
assert_equal 'TypeError', @job.exception.class_name
|
95
|
-
if RUBY_VERSION.to_f < 2.0
|
96
|
-
assert_equal "can't convert Fixnum into String", @job.exception.message
|
97
|
-
else
|
98
|
-
assert_equal 'no implicit conversion of Fixnum into String', @job.exception.message
|
99
|
-
end
|
100
|
-
assert_equal 'inline', @job.exception.worker_name
|
90
|
+
it 'converts type' do
|
91
|
+
@job = SumJob.new(first: '10', second: 5)
|
92
|
+
assert_equal 15, @job.perform_now['result']
|
93
|
+
assert @job.completed?, @job.attributes.ai
|
94
|
+
assert_equal 15, @job.result['result']
|
101
95
|
end
|
102
96
|
|
103
97
|
it 'silence logging when log_level is set' do
|
@@ -127,14 +121,14 @@ module Plugins
|
|
127
121
|
describe '.perform_later' do
|
128
122
|
it 'queues the job for processing' do
|
129
123
|
RocketJob::Config.stub(:inline_mode, false) do
|
130
|
-
@job = SumJob.perform_later(1, 23)
|
124
|
+
@job = SumJob.perform_later(first: 1, second: 23)
|
131
125
|
end
|
132
126
|
assert @job.queued?
|
133
127
|
|
134
128
|
# Manually run the job
|
135
129
|
@job.perform_now
|
136
130
|
assert @job.completed?, @job.attributes.ai
|
137
|
-
assert_equal 24, @job.result['result']
|
131
|
+
assert_equal 24, @job.result['result'], -> { @job.result.ai }
|
138
132
|
|
139
133
|
assert_nil @job.worker_name
|
140
134
|
assert @job.completed_at
|
@@ -150,7 +144,7 @@ module Plugins
|
|
150
144
|
|
151
145
|
it 'runs the job immediately when inline_mode = true' do
|
152
146
|
RocketJob::Config.stub(:inline_mode, true) do
|
153
|
-
@job = SumJob.perform_later(1, 23)
|
147
|
+
@job = SumJob.perform_later(first: 1, second: 23)
|
154
148
|
end
|
155
149
|
|
156
150
|
assert @job.completed?, @job.attributes.ai
|
@@ -171,12 +165,37 @@ module Plugins
|
|
171
165
|
|
172
166
|
describe '.perform_now' do
|
173
167
|
it 'run the job immediately' do
|
174
|
-
@job = SumJob.perform_now(1, 5)
|
168
|
+
@job = SumJob.perform_now(first: 1, second: 5)
|
175
169
|
assert_equal true, @job.completed?
|
176
170
|
assert_equal 6, @job.result['result']
|
177
171
|
end
|
178
172
|
end
|
179
173
|
|
174
|
+
describe '#rocket_job_active_servers' do
|
175
|
+
before do
|
176
|
+
@job = QuietJob.create!
|
177
|
+
@worker = RocketJob::Server.create!(name: 'worker:123')
|
178
|
+
end
|
179
|
+
|
180
|
+
it 'should return empty hash for no active jobs' do
|
181
|
+
assert_equal({}, @job.rocket_job_active_servers)
|
182
|
+
end
|
183
|
+
|
184
|
+
it 'should return active servers' do
|
185
|
+
assert job = RocketJob::Job.rocket_job_next_job(@worker.name)
|
186
|
+
assert active = job.rocket_job_active_servers
|
187
|
+
assert_equal 1, active.size
|
188
|
+
assert active_servers = active[@worker.name]
|
189
|
+
assert_equal 1, active_servers.size
|
190
|
+
assert active_server = active_servers.first
|
191
|
+
assert_equal @job.id, active_server.job.id
|
192
|
+
assert_equal @worker.name, active_server.name
|
193
|
+
assert_equal job.started_at, active_server.started_at
|
194
|
+
assert active_server.duration_s
|
195
|
+
assert active_server.duration
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
180
199
|
end
|
181
200
|
end
|
182
201
|
end
|
@@ -55,8 +55,8 @@ module Plugins
|
|
55
55
|
|
56
56
|
describe '#rocket_job_processing_window_active?' do
|
57
57
|
it 'returns true when in the processing window' do
|
58
|
-
time
|
59
|
-
@job
|
58
|
+
time = Time.parse('2015-12-09 17:50:05 +0000')
|
59
|
+
@job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
|
60
60
|
result = Time.stub(:now, time) do
|
61
61
|
@job.rocket_job_processing_window_active?
|
62
62
|
end
|
@@ -64,8 +64,8 @@ module Plugins
|
|
64
64
|
end
|
65
65
|
|
66
66
|
it 'returns false when not in the processing window' do
|
67
|
-
time
|
68
|
-
@job
|
67
|
+
time = Time.parse('2015-12-09 16:50:05 +0000')
|
68
|
+
@job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
|
69
69
|
result = Time.stub(:now, time) do
|
70
70
|
@job.rocket_job_processing_window_active?
|
71
71
|
end
|
@@ -7,10 +7,10 @@ module Plugins
|
|
7
7
|
include RocketJob::Plugins::Restart
|
8
8
|
|
9
9
|
# Ensure a new start_at and end_at is generated every time this job is restarted
|
10
|
-
self.rocket_job_restart_excludes
|
10
|
+
self.rocket_job_restart_excludes += %w(start_at end_at)
|
11
11
|
|
12
|
-
|
13
|
-
|
12
|
+
field :start_at, type: Date
|
13
|
+
field :end_at, type: Date
|
14
14
|
|
15
15
|
def perform
|
16
16
|
self.start_at = Date.today
|
@@ -28,7 +28,6 @@ module Plugins
|
|
28
28
|
|
29
29
|
after do
|
30
30
|
@job.delete if @job && !@job.new_record?
|
31
|
-
RestartableJob.delete_all
|
32
31
|
end
|
33
32
|
|
34
33
|
describe '#create!' do
|
@@ -53,7 +52,7 @@ module Plugins
|
|
53
52
|
@job = RestartableJob.create!
|
54
53
|
@job.abort!
|
55
54
|
assert_equal 2, RestartableJob.count
|
56
|
-
assert other = RestartableJob.where(id
|
55
|
+
assert other = RestartableJob.where(:id.ne => @job.id).first
|
57
56
|
refute_equal @job.id, other.id
|
58
57
|
assert other.queued?
|
59
58
|
end
|
@@ -63,7 +62,7 @@ module Plugins
|
|
63
62
|
assert @job.expired?
|
64
63
|
@job.abort!
|
65
64
|
assert_equal 1, RestartableJob.count
|
66
|
-
assert_equal nil, RestartableJob.where(id
|
65
|
+
assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
|
67
66
|
end
|
68
67
|
end
|
69
68
|
|
@@ -133,7 +132,7 @@ module Plugins
|
|
133
132
|
assert @job.running?
|
134
133
|
assert @job.expired?
|
135
134
|
assert_equal 1, RestartableJob.count
|
136
|
-
assert_equal nil, RestartableJob.where(id
|
135
|
+
assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
|
137
136
|
end
|
138
137
|
end
|
139
138
|
|
@@ -142,7 +141,7 @@ module Plugins
|
|
142
141
|
@job = RestartableJob.create!(destroy_on_complete: true)
|
143
142
|
@job.perform_now
|
144
143
|
assert_equal 1, RestartableJob.count
|
145
|
-
assert job2 = RestartableJob.where(id
|
144
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
146
145
|
assert job2.queued?, job2.attributes.ai
|
147
146
|
end
|
148
147
|
|
@@ -151,7 +150,7 @@ module Plugins
|
|
151
150
|
refute @job.expired?
|
152
151
|
@job.perform_now
|
153
152
|
assert_equal 1, RestartableJob.count
|
154
|
-
assert job2 = RestartableJob.where(id
|
153
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
155
154
|
assert job2.queued?, job2.attributes.ai
|
156
155
|
|
157
156
|
# Copy across all attributes, except
|
@@ -171,14 +170,14 @@ module Plugins
|
|
171
170
|
assert_equal nil, job2.worker_name
|
172
171
|
assert_equal 0, job2.percent_complete
|
173
172
|
assert_equal nil, job2.exception
|
174
|
-
|
173
|
+
refute job2.result
|
175
174
|
end
|
176
175
|
|
177
176
|
it 'copies run_at when it is in the future' do
|
178
177
|
@job = RestartableJob.create!(run_at: Time.now + 1.day, destroy_on_complete: true)
|
179
178
|
@job.perform_now
|
180
179
|
assert_equal 1, RestartableJob.count
|
181
|
-
assert job2 = RestartableJob.where(id
|
180
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
182
181
|
assert job2.run_at, job2.attributes.ai
|
183
182
|
end
|
184
183
|
|
@@ -186,7 +185,7 @@ module Plugins
|
|
186
185
|
@job = RestartableJob.create!(run_at: Time.now - 1.day, destroy_on_complete: true)
|
187
186
|
@job.perform_now
|
188
187
|
assert_equal 1, RestartableJob.count
|
189
|
-
assert job2 = RestartableJob.where(id
|
188
|
+
assert job2 = RestartableJob.where(:id.ne => @job.id).first
|
190
189
|
assert_equal nil, job2.run_at
|
191
190
|
end
|
192
191
|
end
|
@@ -6,78 +6,80 @@ module Plugins
|
|
6
6
|
# This job adds each event callback as they run into an array
|
7
7
|
# [:start, :complete, :fail, :retry, :pause, :resume, :abort, :requeue]
|
8
8
|
class PositivePathJob < RocketJob::Job
|
9
|
+
field :call_list, type: Array, default: []
|
10
|
+
|
9
11
|
before_complete do
|
10
|
-
|
12
|
+
call_list << 'before_complete_block'
|
11
13
|
end
|
12
14
|
|
13
15
|
after_complete do
|
14
|
-
|
16
|
+
call_list << 'after_complete_block'
|
15
17
|
end
|
16
18
|
|
17
19
|
before_complete :before_complete_method
|
18
20
|
|
19
21
|
before_start do
|
20
|
-
|
22
|
+
call_list << 'before_start_block'
|
21
23
|
end
|
22
24
|
|
23
25
|
before_start :before_start_method
|
24
26
|
|
25
27
|
before_start do
|
26
|
-
|
28
|
+
call_list << 'before_start2_block'
|
27
29
|
end
|
28
30
|
|
29
31
|
after_start :after_start_method
|
30
32
|
after_complete :after_complete_method
|
31
33
|
|
32
34
|
before_complete do
|
33
|
-
|
35
|
+
call_list << 'before_complete2_block'
|
34
36
|
end
|
35
37
|
|
36
38
|
after_start do
|
37
|
-
|
39
|
+
call_list << 'after_start_block'
|
38
40
|
end
|
39
41
|
|
40
42
|
after_complete do
|
41
|
-
|
43
|
+
call_list << 'after_complete2_block'
|
42
44
|
end
|
43
45
|
|
44
46
|
after_start :after_start_method
|
45
47
|
|
46
48
|
after_start do
|
47
|
-
|
49
|
+
call_list << 'after_start2_block'
|
48
50
|
end
|
49
51
|
|
50
52
|
before_start :before_start_method2
|
51
53
|
before_complete :before_complete_method2
|
52
54
|
|
53
|
-
def perform
|
54
|
-
|
55
|
+
def perform
|
56
|
+
call_list << 'perform'
|
55
57
|
end
|
56
58
|
|
57
59
|
private
|
58
60
|
|
59
61
|
def before_start_method
|
60
|
-
|
62
|
+
call_list << 'before_start_method'
|
61
63
|
end
|
62
64
|
|
63
65
|
def before_start_method2
|
64
|
-
|
66
|
+
call_list << 'before_start_method2'
|
65
67
|
end
|
66
68
|
|
67
69
|
def after_start_method
|
68
|
-
|
70
|
+
call_list << 'after_start_method'
|
69
71
|
end
|
70
72
|
|
71
73
|
def before_complete_method
|
72
|
-
|
74
|
+
call_list << 'before_complete_method'
|
73
75
|
end
|
74
76
|
|
75
77
|
def before_complete_method2
|
76
|
-
|
78
|
+
call_list << 'before_complete_method2'
|
77
79
|
end
|
78
80
|
|
79
81
|
def after_complete_method
|
80
|
-
|
82
|
+
call_list << 'after_complete_method'
|
81
83
|
end
|
82
84
|
end
|
83
85
|
|
@@ -88,11 +90,11 @@ module Plugins
|
|
88
90
|
|
89
91
|
describe 'before_start after_start & before_complete after_complete' do
|
90
92
|
it 'runs blocks and functions' do
|
91
|
-
@job = PositivePathJob.new
|
93
|
+
@job = PositivePathJob.new
|
92
94
|
@job.perform_now
|
93
95
|
assert @job.completed?, @job.attributes.ai
|
94
96
|
expected = %w(before_start_block before_start_method before_start2_block before_start_method2 after_start2_block after_start_method after_start_block perform before_complete_block before_complete_method before_complete2_block before_complete_method2 after_complete2_block after_complete_method after_complete_block)
|
95
|
-
assert_equal expected, @job.
|
97
|
+
assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
|
96
98
|
end
|
97
99
|
end
|
98
100
|
|