rocketjob 2.1.3 → 3.0.0.alpha

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +36 -0
  3. data/lib/rocket_job/active_server.rb +48 -0
  4. data/lib/rocket_job/cli.rb +29 -17
  5. data/lib/rocket_job/config.rb +19 -31
  6. data/lib/rocket_job/dirmon_entry.rb +15 -45
  7. data/lib/rocket_job/extensions/mongo/logging.rb +26 -0
  8. data/lib/rocket_job/extensions/rocket_job_adapter.rb +3 -5
  9. data/lib/rocket_job/heartbeat.rb +18 -23
  10. data/lib/rocket_job/job.rb +0 -1
  11. data/lib/rocket_job/job_exception.rb +11 -13
  12. data/lib/rocket_job/jobs/dirmon_job.rb +8 -8
  13. data/lib/rocket_job/jobs/housekeeping_job.rb +13 -15
  14. data/lib/rocket_job/performance.rb +5 -5
  15. data/lib/rocket_job/plugins/cron.rb +3 -10
  16. data/lib/rocket_job/plugins/document.rb +58 -33
  17. data/lib/rocket_job/plugins/job/model.rb +43 -71
  18. data/lib/rocket_job/plugins/job/persistence.rb +7 -63
  19. data/lib/rocket_job/plugins/job/worker.rb +24 -26
  20. data/lib/rocket_job/plugins/processing_window.rb +6 -9
  21. data/lib/rocket_job/plugins/retry.rb +3 -8
  22. data/lib/rocket_job/plugins/singleton.rb +1 -1
  23. data/lib/rocket_job/plugins/state_machine.rb +1 -7
  24. data/lib/rocket_job/server.rb +352 -0
  25. data/lib/rocket_job/version.rb +1 -1
  26. data/lib/rocket_job/worker.rb +46 -336
  27. data/lib/rocketjob.rb +5 -4
  28. data/test/config/mongoid.yml +88 -0
  29. data/test/config_test.rb +1 -1
  30. data/test/dirmon_entry_test.rb +15 -79
  31. data/test/dirmon_job_test.rb +6 -6
  32. data/test/job_test.rb +2 -2
  33. data/test/plugins/job/callbacks_test.rb +40 -32
  34. data/test/plugins/job/defaults_test.rb +10 -8
  35. data/test/plugins/job/model_test.rb +1 -3
  36. data/test/plugins/job/persistence_test.rb +11 -13
  37. data/test/plugins/job/worker_test.rb +45 -26
  38. data/test/plugins/processing_window_test.rb +4 -4
  39. data/test/plugins/restart_test.rb +11 -12
  40. data/test/plugins/state_machine_event_callbacks_test.rb +20 -18
  41. data/test/plugins/state_machine_test.rb +5 -5
  42. data/test/test_helper.rb +4 -1
  43. metadata +15 -29
  44. data/lib/rocket_job/extensions/mongo.rb +0 -23
  45. data/lib/rocket_job/extensions/mongo_mapper.rb +0 -30
  46. data/lib/rocket_job/plugins/job/defaults.rb +0 -40
  47. data/test/config/mongo.yml +0 -46
@@ -6,25 +6,21 @@ module Plugins
6
6
  class DefaultsTest < Minitest::Test
7
7
 
8
8
  class ParentJob < RocketJob::Job
9
- rocket_job do |job|
10
- job.priority = 53
11
- job.description = 'Hello'
12
- end
9
+ self.priority = 53
10
+ self.description = 'Hello'
13
11
 
14
12
  def perform
15
13
  end
16
14
  end
17
15
 
18
16
  class ChildJob < ParentJob
19
- rocket_job do |job|
20
- job.priority = 72
21
- end
17
+ self.priority = 72
22
18
 
23
19
  def perform
24
20
  end
25
21
  end
26
22
 
27
- describe RocketJob::Plugins::Job::Defaults do
23
+ describe RocketJob::Plugins::Job do
28
24
  after do
29
25
  @job.destroy if @job && !@job.new_record?
30
26
  end
@@ -36,6 +32,12 @@ module Plugins
36
32
  assert_equal 'Hello', @job.description
37
33
  end
38
34
 
35
+ it 'can override defaults on initialize' do
36
+ @job = ParentJob.new(priority: 72, description: 'More')
37
+ assert_equal 72, @job.priority
38
+ assert_equal 'More', @job.description
39
+ end
40
+
39
41
  it 'allows a child to override parent defaults' do
40
42
  @job = ChildJob.new
41
43
  assert_equal 72, @job.priority
@@ -11,9 +11,7 @@ module Plugins
11
11
  end
12
12
 
13
13
  class TwoArgumentJob < RocketJob::Job
14
- rocket_job do |job|
15
- job.priority = 53
16
- end
14
+ self.priority = 53
17
15
 
18
16
  def perform(a, b)
19
17
  a + b
@@ -6,9 +6,8 @@ module Plugins
6
6
  class PersistenceTest < Minitest::Test
7
7
 
8
8
  class PersistJob < RocketJob::Job
9
- rocket_job do |job|
10
- job.priority = 53
11
- end
9
+ self.priority = 53
10
+ field :data, type: Hash
12
11
 
13
12
  def perform(hash)
14
13
  hash
@@ -19,10 +18,10 @@ module Plugins
19
18
  before do
20
19
  RocketJob::Job.destroy_all
21
20
  @description = 'Hello World'
22
- @arguments = [{key: 'value'}]
21
+ @data = {'key' => 'value'}
23
22
  @job = PersistJob.new(
24
23
  description: @description,
25
- arguments: [{key: 'value'}],
24
+ data: @data,
26
25
  destroy_on_complete: false
27
26
  )
28
27
  end
@@ -35,7 +34,7 @@ module Plugins
35
34
 
36
35
  describe '.config' do
37
36
  it 'support multiple databases' do
38
- assert_equal 'test_rocketjob', RocketJob::Job.collection.db.name
37
+ assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
39
38
  end
40
39
  end
41
40
 
@@ -47,14 +46,13 @@ module Plugins
47
46
 
48
47
  describe '#reload' do
49
48
  it 'handle hash' do
50
- assert_equal 'value', @job.arguments.first[:key]
49
+ assert_equal 'value', @job.data['key']
51
50
  @job.worker_name = nil
52
51
  @job.save!
53
52
  @job.worker_name = '123'
54
53
  @job.reload
55
- assert @job.arguments.first.is_a?(ActiveSupport::HashWithIndifferentAccess), @job.arguments.first.class.ai
56
- assert_equal 'value', @job.arguments.first['key']
57
- assert_equal 'value', @job.arguments.first[:key]
54
+ assert @job.data.is_a?(Hash), @job.data.class.ai
55
+ assert_equal 'value', @job.data['key']
58
56
  assert_equal nil, @job.worker_name
59
57
  end
60
58
  end
@@ -68,7 +66,7 @@ module Plugins
68
66
  assert_equal @description, @job.description
69
67
  assert_equal false, @job.destroy_on_complete
70
68
  assert_nil @job.expires_at
71
- assert_equal @arguments, @job.arguments
69
+ assert_equal @data, @job.data
72
70
  assert_equal 0, @job.percent_complete
73
71
  assert_equal 53, @job.priority
74
72
  assert_equal 0, @job.failure_count
@@ -81,8 +79,8 @@ module Plugins
81
79
  describe '.counts_by_state' do
82
80
  it 'returns states as symbols' do
83
81
  @job.start!
84
- @job2 = PersistJob.create!(arguments: [{key: 'value'}])
85
- @job3 = PersistJob.create!(arguments: [{key: 'value'}], run_at: 1.day.from_now)
82
+ @job2 = PersistJob.create!(data: {key: 'value'})
83
+ @job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
86
84
  counts = RocketJob::Job.counts_by_state
87
85
  assert_equal 4, counts.size, counts.ai
88
86
  assert_equal 1, counts[:running]
@@ -19,20 +19,22 @@ module Plugins
19
19
  end
20
20
 
21
21
  class SumJob < RocketJob::Job
22
- rocket_job do |job|
23
- job.destroy_on_complete = false
24
- job.collect_output = true
25
- job.priority = 51
26
- end
22
+ self.destroy_on_complete = false
23
+ self.collect_output = true
24
+ self.priority = 51
25
+
26
+ field :first, type: Integer
27
+ field :second, type: Integer
27
28
 
28
- def perform(first, second)
29
+ def perform
29
30
  first + second
30
31
  end
31
32
  end
32
33
 
33
34
  describe RocketJob::Plugins::Job::Worker do
34
35
  before do
35
- RocketJob::Job.destroy_all
36
+ RocketJob::Job.delete_all
37
+ RocketJob::Server.delete_all
36
38
  end
37
39
 
38
40
  after do
@@ -42,7 +44,7 @@ module Plugins
42
44
  describe '.rocket_job_next_job' do
43
45
  before do
44
46
  @job = QuietJob.new
45
- @worker = RocketJob::Worker.new(name: 'worker:123')
47
+ @worker = RocketJob::Server.new(name: 'worker:123')
46
48
  end
47
49
 
48
50
  it 'return nil when no jobs available' do
@@ -79,25 +81,17 @@ module Plugins
79
81
 
80
82
  describe '#perform_now' do
81
83
  it 'calls perform method' do
82
- @job = SumJob.new(arguments: [10, 5])
84
+ @job = SumJob.new(first: 10, second: 5)
83
85
  assert_equal 15, @job.perform_now['result']
84
86
  assert @job.completed?, @job.attributes.ai
85
87
  assert_equal 15, @job.result['result']
86
88
  end
87
89
 
88
- it 'saves exception' do
89
- @job = SumJob.new(arguments: ['10', 5])
90
- assert_raises TypeError do
91
- @job.perform_now
92
- end
93
- assert @job.exception.backtrace
94
- assert_equal 'TypeError', @job.exception.class_name
95
- if RUBY_VERSION.to_f < 2.0
96
- assert_equal "can't convert Fixnum into String", @job.exception.message
97
- else
98
- assert_equal 'no implicit conversion of Fixnum into String', @job.exception.message
99
- end
100
- assert_equal 'inline', @job.exception.worker_name
90
+ it 'converts type' do
91
+ @job = SumJob.new(first: '10', second: 5)
92
+ assert_equal 15, @job.perform_now['result']
93
+ assert @job.completed?, @job.attributes.ai
94
+ assert_equal 15, @job.result['result']
101
95
  end
102
96
 
103
97
  it 'silence logging when log_level is set' do
@@ -127,14 +121,14 @@ module Plugins
127
121
  describe '.perform_later' do
128
122
  it 'queues the job for processing' do
129
123
  RocketJob::Config.stub(:inline_mode, false) do
130
- @job = SumJob.perform_later(1, 23)
124
+ @job = SumJob.perform_later(first: 1, second: 23)
131
125
  end
132
126
  assert @job.queued?
133
127
 
134
128
  # Manually run the job
135
129
  @job.perform_now
136
130
  assert @job.completed?, @job.attributes.ai
137
- assert_equal 24, @job.result['result']
131
+ assert_equal 24, @job.result['result'], -> { @job.result.ai }
138
132
 
139
133
  assert_nil @job.worker_name
140
134
  assert @job.completed_at
@@ -150,7 +144,7 @@ module Plugins
150
144
 
151
145
  it 'runs the job immediately when inline_mode = true' do
152
146
  RocketJob::Config.stub(:inline_mode, true) do
153
- @job = SumJob.perform_later(1, 23)
147
+ @job = SumJob.perform_later(first: 1, second: 23)
154
148
  end
155
149
 
156
150
  assert @job.completed?, @job.attributes.ai
@@ -171,12 +165,37 @@ module Plugins
171
165
 
172
166
  describe '.perform_now' do
173
167
  it 'run the job immediately' do
174
- @job = SumJob.perform_now(1, 5)
168
+ @job = SumJob.perform_now(first: 1, second: 5)
175
169
  assert_equal true, @job.completed?
176
170
  assert_equal 6, @job.result['result']
177
171
  end
178
172
  end
179
173
 
174
+ describe '#rocket_job_active_servers' do
175
+ before do
176
+ @job = QuietJob.create!
177
+ @worker = RocketJob::Server.create!(name: 'worker:123')
178
+ end
179
+
180
+ it 'should return empty hash for no active jobs' do
181
+ assert_equal({}, @job.rocket_job_active_servers)
182
+ end
183
+
184
+ it 'should return active servers' do
185
+ assert job = RocketJob::Job.rocket_job_next_job(@worker.name)
186
+ assert active = job.rocket_job_active_servers
187
+ assert_equal 1, active.size
188
+ assert active_servers = active[@worker.name]
189
+ assert_equal 1, active_servers.size
190
+ assert active_server = active_servers.first
191
+ assert_equal @job.id, active_server.job.id
192
+ assert_equal @worker.name, active_server.name
193
+ assert_equal job.started_at, active_server.started_at
194
+ assert active_server.duration_s
195
+ assert active_server.duration
196
+ end
197
+ end
198
+
180
199
  end
181
200
  end
182
201
  end
@@ -55,8 +55,8 @@ module Plugins
55
55
 
56
56
  describe '#rocket_job_processing_window_active?' do
57
57
  it 'returns true when in the processing window' do
58
- time = Time.parse('2015-12-09 17:50:05 +0000')
59
- @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
58
+ time = Time.parse('2015-12-09 17:50:05 +0000')
59
+ @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
60
60
  result = Time.stub(:now, time) do
61
61
  @job.rocket_job_processing_window_active?
62
62
  end
@@ -64,8 +64,8 @@ module Plugins
64
64
  end
65
65
 
66
66
  it 'returns false when not in the processing window' do
67
- time = Time.parse('2015-12-09 16:50:05 +0000')
68
- @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
67
+ time = Time.parse('2015-12-09 16:50:05 +0000')
68
+ @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
69
69
  result = Time.stub(:now, time) do
70
70
  @job.rocket_job_processing_window_active?
71
71
  end
@@ -7,10 +7,10 @@ module Plugins
7
7
  include RocketJob::Plugins::Restart
8
8
 
9
9
  # Ensure a new start_at and end_at is generated every time this job is restarted
10
- self.rocket_job_restart_excludes = self.rocket_job_restart_excludes + %w(start_at end_at)
10
+ self.rocket_job_restart_excludes += %w(start_at end_at)
11
11
 
12
- key :start_at, Date
13
- key :end_at, Date
12
+ field :start_at, type: Date
13
+ field :end_at, type: Date
14
14
 
15
15
  def perform
16
16
  self.start_at = Date.today
@@ -28,7 +28,6 @@ module Plugins
28
28
 
29
29
  after do
30
30
  @job.delete if @job && !@job.new_record?
31
- RestartableJob.delete_all
32
31
  end
33
32
 
34
33
  describe '#create!' do
@@ -53,7 +52,7 @@ module Plugins
53
52
  @job = RestartableJob.create!
54
53
  @job.abort!
55
54
  assert_equal 2, RestartableJob.count
56
- assert other = RestartableJob.where(id: {'$ne' => @job.id}).first
55
+ assert other = RestartableJob.where(:id.ne => @job.id).first
57
56
  refute_equal @job.id, other.id
58
57
  assert other.queued?
59
58
  end
@@ -63,7 +62,7 @@ module Plugins
63
62
  assert @job.expired?
64
63
  @job.abort!
65
64
  assert_equal 1, RestartableJob.count
66
- assert_equal nil, RestartableJob.where(id: {'$ne' => @job.id}).first
65
+ assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
67
66
  end
68
67
  end
69
68
 
@@ -133,7 +132,7 @@ module Plugins
133
132
  assert @job.running?
134
133
  assert @job.expired?
135
134
  assert_equal 1, RestartableJob.count
136
- assert_equal nil, RestartableJob.where(id: {'$ne' => @job.id}).first
135
+ assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
137
136
  end
138
137
  end
139
138
 
@@ -142,7 +141,7 @@ module Plugins
142
141
  @job = RestartableJob.create!(destroy_on_complete: true)
143
142
  @job.perform_now
144
143
  assert_equal 1, RestartableJob.count
145
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
144
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
146
145
  assert job2.queued?, job2.attributes.ai
147
146
  end
148
147
 
@@ -151,7 +150,7 @@ module Plugins
151
150
  refute @job.expired?
152
151
  @job.perform_now
153
152
  assert_equal 1, RestartableJob.count
154
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
153
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
155
154
  assert job2.queued?, job2.attributes.ai
156
155
 
157
156
  # Copy across all attributes, except
@@ -171,14 +170,14 @@ module Plugins
171
170
  assert_equal nil, job2.worker_name
172
171
  assert_equal 0, job2.percent_complete
173
172
  assert_equal nil, job2.exception
174
- assert_equal({}, job2.result)
173
+ refute job2.result
175
174
  end
176
175
 
177
176
  it 'copies run_at when it is in the future' do
178
177
  @job = RestartableJob.create!(run_at: Time.now + 1.day, destroy_on_complete: true)
179
178
  @job.perform_now
180
179
  assert_equal 1, RestartableJob.count
181
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
180
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
182
181
  assert job2.run_at, job2.attributes.ai
183
182
  end
184
183
 
@@ -186,7 +185,7 @@ module Plugins
186
185
  @job = RestartableJob.create!(run_at: Time.now - 1.day, destroy_on_complete: true)
187
186
  @job.perform_now
188
187
  assert_equal 1, RestartableJob.count
189
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
188
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
190
189
  assert_equal nil, job2.run_at
191
190
  end
192
191
  end
@@ -6,78 +6,80 @@ module Plugins
6
6
  # This job adds each event callback as they run into an array
7
7
  # [:start, :complete, :fail, :retry, :pause, :resume, :abort, :requeue]
8
8
  class PositivePathJob < RocketJob::Job
9
+ field :call_list, type: Array, default: []
10
+
9
11
  before_complete do
10
- arguments.first << 'before_complete_block'
12
+ call_list << 'before_complete_block'
11
13
  end
12
14
 
13
15
  after_complete do
14
- arguments.first << 'after_complete_block'
16
+ call_list << 'after_complete_block'
15
17
  end
16
18
 
17
19
  before_complete :before_complete_method
18
20
 
19
21
  before_start do
20
- arguments.first << 'before_start_block'
22
+ call_list << 'before_start_block'
21
23
  end
22
24
 
23
25
  before_start :before_start_method
24
26
 
25
27
  before_start do
26
- arguments.first << 'before_start2_block'
28
+ call_list << 'before_start2_block'
27
29
  end
28
30
 
29
31
  after_start :after_start_method
30
32
  after_complete :after_complete_method
31
33
 
32
34
  before_complete do
33
- arguments.first << 'before_complete2_block'
35
+ call_list << 'before_complete2_block'
34
36
  end
35
37
 
36
38
  after_start do
37
- arguments.first << 'after_start_block'
39
+ call_list << 'after_start_block'
38
40
  end
39
41
 
40
42
  after_complete do
41
- arguments.first << 'after_complete2_block'
43
+ call_list << 'after_complete2_block'
42
44
  end
43
45
 
44
46
  after_start :after_start_method
45
47
 
46
48
  after_start do
47
- arguments.first << 'after_start2_block'
49
+ call_list << 'after_start2_block'
48
50
  end
49
51
 
50
52
  before_start :before_start_method2
51
53
  before_complete :before_complete_method2
52
54
 
53
- def perform(list)
54
- list << 'perform'
55
+ def perform
56
+ call_list << 'perform'
55
57
  end
56
58
 
57
59
  private
58
60
 
59
61
  def before_start_method
60
- arguments.first << 'before_start_method'
62
+ call_list << 'before_start_method'
61
63
  end
62
64
 
63
65
  def before_start_method2
64
- arguments.first << 'before_start_method2'
66
+ call_list << 'before_start_method2'
65
67
  end
66
68
 
67
69
  def after_start_method
68
- arguments.first << 'after_start_method'
70
+ call_list << 'after_start_method'
69
71
  end
70
72
 
71
73
  def before_complete_method
72
- arguments.first << 'before_complete_method'
74
+ call_list << 'before_complete_method'
73
75
  end
74
76
 
75
77
  def before_complete_method2
76
- arguments.first << 'before_complete_method2'
78
+ call_list << 'before_complete_method2'
77
79
  end
78
80
 
79
81
  def after_complete_method
80
- arguments.first << 'after_complete_method'
82
+ call_list << 'after_complete_method'
81
83
  end
82
84
  end
83
85
 
@@ -88,11 +90,11 @@ module Plugins
88
90
 
89
91
  describe 'before_start after_start & before_complete after_complete' do
90
92
  it 'runs blocks and functions' do
91
- @job = PositivePathJob.new(arguments: [[]])
93
+ @job = PositivePathJob.new
92
94
  @job.perform_now
93
95
  assert @job.completed?, @job.attributes.ai
94
96
  expected = %w(before_start_block before_start_method before_start2_block before_start_method2 after_start2_block after_start_method after_start_block perform before_complete_block before_complete_method before_complete2_block before_complete_method2 after_complete2_block after_complete_method after_complete_block)
95
- assert_equal expected, @job.arguments.first, 'Sequence of before_perform callbacks is incorrect'
97
+ assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
96
98
  end
97
99
  end
98
100