rocketjob 2.1.3 → 3.0.0.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +36 -0
  3. data/lib/rocket_job/active_server.rb +48 -0
  4. data/lib/rocket_job/cli.rb +29 -17
  5. data/lib/rocket_job/config.rb +19 -31
  6. data/lib/rocket_job/dirmon_entry.rb +15 -45
  7. data/lib/rocket_job/extensions/mongo/logging.rb +26 -0
  8. data/lib/rocket_job/extensions/rocket_job_adapter.rb +3 -5
  9. data/lib/rocket_job/heartbeat.rb +18 -23
  10. data/lib/rocket_job/job.rb +0 -1
  11. data/lib/rocket_job/job_exception.rb +11 -13
  12. data/lib/rocket_job/jobs/dirmon_job.rb +8 -8
  13. data/lib/rocket_job/jobs/housekeeping_job.rb +13 -15
  14. data/lib/rocket_job/performance.rb +5 -5
  15. data/lib/rocket_job/plugins/cron.rb +3 -10
  16. data/lib/rocket_job/plugins/document.rb +58 -33
  17. data/lib/rocket_job/plugins/job/model.rb +43 -71
  18. data/lib/rocket_job/plugins/job/persistence.rb +7 -63
  19. data/lib/rocket_job/plugins/job/worker.rb +24 -26
  20. data/lib/rocket_job/plugins/processing_window.rb +6 -9
  21. data/lib/rocket_job/plugins/retry.rb +3 -8
  22. data/lib/rocket_job/plugins/singleton.rb +1 -1
  23. data/lib/rocket_job/plugins/state_machine.rb +1 -7
  24. data/lib/rocket_job/server.rb +352 -0
  25. data/lib/rocket_job/version.rb +1 -1
  26. data/lib/rocket_job/worker.rb +46 -336
  27. data/lib/rocketjob.rb +5 -4
  28. data/test/config/mongoid.yml +88 -0
  29. data/test/config_test.rb +1 -1
  30. data/test/dirmon_entry_test.rb +15 -79
  31. data/test/dirmon_job_test.rb +6 -6
  32. data/test/job_test.rb +2 -2
  33. data/test/plugins/job/callbacks_test.rb +40 -32
  34. data/test/plugins/job/defaults_test.rb +10 -8
  35. data/test/plugins/job/model_test.rb +1 -3
  36. data/test/plugins/job/persistence_test.rb +11 -13
  37. data/test/plugins/job/worker_test.rb +45 -26
  38. data/test/plugins/processing_window_test.rb +4 -4
  39. data/test/plugins/restart_test.rb +11 -12
  40. data/test/plugins/state_machine_event_callbacks_test.rb +20 -18
  41. data/test/plugins/state_machine_test.rb +5 -5
  42. data/test/test_helper.rb +4 -1
  43. metadata +15 -29
  44. data/lib/rocket_job/extensions/mongo.rb +0 -23
  45. data/lib/rocket_job/extensions/mongo_mapper.rb +0 -30
  46. data/lib/rocket_job/plugins/job/defaults.rb +0 -40
  47. data/test/config/mongo.yml +0 -46
@@ -6,25 +6,21 @@ module Plugins
6
6
  class DefaultsTest < Minitest::Test
7
7
 
8
8
  class ParentJob < RocketJob::Job
9
- rocket_job do |job|
10
- job.priority = 53
11
- job.description = 'Hello'
12
- end
9
+ self.priority = 53
10
+ self.description = 'Hello'
13
11
 
14
12
  def perform
15
13
  end
16
14
  end
17
15
 
18
16
  class ChildJob < ParentJob
19
- rocket_job do |job|
20
- job.priority = 72
21
- end
17
+ self.priority = 72
22
18
 
23
19
  def perform
24
20
  end
25
21
  end
26
22
 
27
- describe RocketJob::Plugins::Job::Defaults do
23
+ describe RocketJob::Plugins::Job do
28
24
  after do
29
25
  @job.destroy if @job && !@job.new_record?
30
26
  end
@@ -36,6 +32,12 @@ module Plugins
36
32
  assert_equal 'Hello', @job.description
37
33
  end
38
34
 
35
+ it 'can override defaults on initialize' do
36
+ @job = ParentJob.new(priority: 72, description: 'More')
37
+ assert_equal 72, @job.priority
38
+ assert_equal 'More', @job.description
39
+ end
40
+
39
41
  it 'allows a child to override parent defaults' do
40
42
  @job = ChildJob.new
41
43
  assert_equal 72, @job.priority
@@ -11,9 +11,7 @@ module Plugins
11
11
  end
12
12
 
13
13
  class TwoArgumentJob < RocketJob::Job
14
- rocket_job do |job|
15
- job.priority = 53
16
- end
14
+ self.priority = 53
17
15
 
18
16
  def perform(a, b)
19
17
  a + b
@@ -6,9 +6,8 @@ module Plugins
6
6
  class PersistenceTest < Minitest::Test
7
7
 
8
8
  class PersistJob < RocketJob::Job
9
- rocket_job do |job|
10
- job.priority = 53
11
- end
9
+ self.priority = 53
10
+ field :data, type: Hash
12
11
 
13
12
  def perform(hash)
14
13
  hash
@@ -19,10 +18,10 @@ module Plugins
19
18
  before do
20
19
  RocketJob::Job.destroy_all
21
20
  @description = 'Hello World'
22
- @arguments = [{key: 'value'}]
21
+ @data = {'key' => 'value'}
23
22
  @job = PersistJob.new(
24
23
  description: @description,
25
- arguments: [{key: 'value'}],
24
+ data: @data,
26
25
  destroy_on_complete: false
27
26
  )
28
27
  end
@@ -35,7 +34,7 @@ module Plugins
35
34
 
36
35
  describe '.config' do
37
36
  it 'support multiple databases' do
38
- assert_equal 'test_rocketjob', RocketJob::Job.collection.db.name
37
+ assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
39
38
  end
40
39
  end
41
40
 
@@ -47,14 +46,13 @@ module Plugins
47
46
 
48
47
  describe '#reload' do
49
48
  it 'handle hash' do
50
- assert_equal 'value', @job.arguments.first[:key]
49
+ assert_equal 'value', @job.data['key']
51
50
  @job.worker_name = nil
52
51
  @job.save!
53
52
  @job.worker_name = '123'
54
53
  @job.reload
55
- assert @job.arguments.first.is_a?(ActiveSupport::HashWithIndifferentAccess), @job.arguments.first.class.ai
56
- assert_equal 'value', @job.arguments.first['key']
57
- assert_equal 'value', @job.arguments.first[:key]
54
+ assert @job.data.is_a?(Hash), @job.data.class.ai
55
+ assert_equal 'value', @job.data['key']
58
56
  assert_equal nil, @job.worker_name
59
57
  end
60
58
  end
@@ -68,7 +66,7 @@ module Plugins
68
66
  assert_equal @description, @job.description
69
67
  assert_equal false, @job.destroy_on_complete
70
68
  assert_nil @job.expires_at
71
- assert_equal @arguments, @job.arguments
69
+ assert_equal @data, @job.data
72
70
  assert_equal 0, @job.percent_complete
73
71
  assert_equal 53, @job.priority
74
72
  assert_equal 0, @job.failure_count
@@ -81,8 +79,8 @@ module Plugins
81
79
  describe '.counts_by_state' do
82
80
  it 'returns states as symbols' do
83
81
  @job.start!
84
- @job2 = PersistJob.create!(arguments: [{key: 'value'}])
85
- @job3 = PersistJob.create!(arguments: [{key: 'value'}], run_at: 1.day.from_now)
82
+ @job2 = PersistJob.create!(data: {key: 'value'})
83
+ @job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
86
84
  counts = RocketJob::Job.counts_by_state
87
85
  assert_equal 4, counts.size, counts.ai
88
86
  assert_equal 1, counts[:running]
@@ -19,20 +19,22 @@ module Plugins
19
19
  end
20
20
 
21
21
  class SumJob < RocketJob::Job
22
- rocket_job do |job|
23
- job.destroy_on_complete = false
24
- job.collect_output = true
25
- job.priority = 51
26
- end
22
+ self.destroy_on_complete = false
23
+ self.collect_output = true
24
+ self.priority = 51
25
+
26
+ field :first, type: Integer
27
+ field :second, type: Integer
27
28
 
28
- def perform(first, second)
29
+ def perform
29
30
  first + second
30
31
  end
31
32
  end
32
33
 
33
34
  describe RocketJob::Plugins::Job::Worker do
34
35
  before do
35
- RocketJob::Job.destroy_all
36
+ RocketJob::Job.delete_all
37
+ RocketJob::Server.delete_all
36
38
  end
37
39
 
38
40
  after do
@@ -42,7 +44,7 @@ module Plugins
42
44
  describe '.rocket_job_next_job' do
43
45
  before do
44
46
  @job = QuietJob.new
45
- @worker = RocketJob::Worker.new(name: 'worker:123')
47
+ @worker = RocketJob::Server.new(name: 'worker:123')
46
48
  end
47
49
 
48
50
  it 'return nil when no jobs available' do
@@ -79,25 +81,17 @@ module Plugins
79
81
 
80
82
  describe '#perform_now' do
81
83
  it 'calls perform method' do
82
- @job = SumJob.new(arguments: [10, 5])
84
+ @job = SumJob.new(first: 10, second: 5)
83
85
  assert_equal 15, @job.perform_now['result']
84
86
  assert @job.completed?, @job.attributes.ai
85
87
  assert_equal 15, @job.result['result']
86
88
  end
87
89
 
88
- it 'saves exception' do
89
- @job = SumJob.new(arguments: ['10', 5])
90
- assert_raises TypeError do
91
- @job.perform_now
92
- end
93
- assert @job.exception.backtrace
94
- assert_equal 'TypeError', @job.exception.class_name
95
- if RUBY_VERSION.to_f < 2.0
96
- assert_equal "can't convert Fixnum into String", @job.exception.message
97
- else
98
- assert_equal 'no implicit conversion of Fixnum into String', @job.exception.message
99
- end
100
- assert_equal 'inline', @job.exception.worker_name
90
+ it 'converts type' do
91
+ @job = SumJob.new(first: '10', second: 5)
92
+ assert_equal 15, @job.perform_now['result']
93
+ assert @job.completed?, @job.attributes.ai
94
+ assert_equal 15, @job.result['result']
101
95
  end
102
96
 
103
97
  it 'silence logging when log_level is set' do
@@ -127,14 +121,14 @@ module Plugins
127
121
  describe '.perform_later' do
128
122
  it 'queues the job for processing' do
129
123
  RocketJob::Config.stub(:inline_mode, false) do
130
- @job = SumJob.perform_later(1, 23)
124
+ @job = SumJob.perform_later(first: 1, second: 23)
131
125
  end
132
126
  assert @job.queued?
133
127
 
134
128
  # Manually run the job
135
129
  @job.perform_now
136
130
  assert @job.completed?, @job.attributes.ai
137
- assert_equal 24, @job.result['result']
131
+ assert_equal 24, @job.result['result'], -> { @job.result.ai }
138
132
 
139
133
  assert_nil @job.worker_name
140
134
  assert @job.completed_at
@@ -150,7 +144,7 @@ module Plugins
150
144
 
151
145
  it 'runs the job immediately when inline_mode = true' do
152
146
  RocketJob::Config.stub(:inline_mode, true) do
153
- @job = SumJob.perform_later(1, 23)
147
+ @job = SumJob.perform_later(first: 1, second: 23)
154
148
  end
155
149
 
156
150
  assert @job.completed?, @job.attributes.ai
@@ -171,12 +165,37 @@ module Plugins
171
165
 
172
166
  describe '.perform_now' do
173
167
  it 'run the job immediately' do
174
- @job = SumJob.perform_now(1, 5)
168
+ @job = SumJob.perform_now(first: 1, second: 5)
175
169
  assert_equal true, @job.completed?
176
170
  assert_equal 6, @job.result['result']
177
171
  end
178
172
  end
179
173
 
174
+ describe '#rocket_job_active_servers' do
175
+ before do
176
+ @job = QuietJob.create!
177
+ @worker = RocketJob::Server.create!(name: 'worker:123')
178
+ end
179
+
180
+ it 'should return empty hash for no active jobs' do
181
+ assert_equal({}, @job.rocket_job_active_servers)
182
+ end
183
+
184
+ it 'should return active servers' do
185
+ assert job = RocketJob::Job.rocket_job_next_job(@worker.name)
186
+ assert active = job.rocket_job_active_servers
187
+ assert_equal 1, active.size
188
+ assert active_servers = active[@worker.name]
189
+ assert_equal 1, active_servers.size
190
+ assert active_server = active_servers.first
191
+ assert_equal @job.id, active_server.job.id
192
+ assert_equal @worker.name, active_server.name
193
+ assert_equal job.started_at, active_server.started_at
194
+ assert active_server.duration_s
195
+ assert active_server.duration
196
+ end
197
+ end
198
+
180
199
  end
181
200
  end
182
201
  end
@@ -55,8 +55,8 @@ module Plugins
55
55
 
56
56
  describe '#rocket_job_processing_window_active?' do
57
57
  it 'returns true when in the processing window' do
58
- time = Time.parse('2015-12-09 17:50:05 +0000')
59
- @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
58
+ time = Time.parse('2015-12-09 17:50:05 +0000')
59
+ @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
60
60
  result = Time.stub(:now, time) do
61
61
  @job.rocket_job_processing_window_active?
62
62
  end
@@ -64,8 +64,8 @@ module Plugins
64
64
  end
65
65
 
66
66
  it 'returns false when not in the processing window' do
67
- time = Time.parse('2015-12-09 16:50:05 +0000')
68
- @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
67
+ time = Time.parse('2015-12-09 16:50:05 +0000')
68
+ @job = ProcessingWindowJob.new(processing_schedule: '* 17 * * * UTC', processing_duration: 1.hour)
69
69
  result = Time.stub(:now, time) do
70
70
  @job.rocket_job_processing_window_active?
71
71
  end
@@ -7,10 +7,10 @@ module Plugins
7
7
  include RocketJob::Plugins::Restart
8
8
 
9
9
  # Ensure a new start_at and end_at is generated every time this job is restarted
10
- self.rocket_job_restart_excludes = self.rocket_job_restart_excludes + %w(start_at end_at)
10
+ self.rocket_job_restart_excludes += %w(start_at end_at)
11
11
 
12
- key :start_at, Date
13
- key :end_at, Date
12
+ field :start_at, type: Date
13
+ field :end_at, type: Date
14
14
 
15
15
  def perform
16
16
  self.start_at = Date.today
@@ -28,7 +28,6 @@ module Plugins
28
28
 
29
29
  after do
30
30
  @job.delete if @job && !@job.new_record?
31
- RestartableJob.delete_all
32
31
  end
33
32
 
34
33
  describe '#create!' do
@@ -53,7 +52,7 @@ module Plugins
53
52
  @job = RestartableJob.create!
54
53
  @job.abort!
55
54
  assert_equal 2, RestartableJob.count
56
- assert other = RestartableJob.where(id: {'$ne' => @job.id}).first
55
+ assert other = RestartableJob.where(:id.ne => @job.id).first
57
56
  refute_equal @job.id, other.id
58
57
  assert other.queued?
59
58
  end
@@ -63,7 +62,7 @@ module Plugins
63
62
  assert @job.expired?
64
63
  @job.abort!
65
64
  assert_equal 1, RestartableJob.count
66
- assert_equal nil, RestartableJob.where(id: {'$ne' => @job.id}).first
65
+ assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
67
66
  end
68
67
  end
69
68
 
@@ -133,7 +132,7 @@ module Plugins
133
132
  assert @job.running?
134
133
  assert @job.expired?
135
134
  assert_equal 1, RestartableJob.count
136
- assert_equal nil, RestartableJob.where(id: {'$ne' => @job.id}).first
135
+ assert_equal nil, RestartableJob.where(:id.ne => @job.id).first
137
136
  end
138
137
  end
139
138
 
@@ -142,7 +141,7 @@ module Plugins
142
141
  @job = RestartableJob.create!(destroy_on_complete: true)
143
142
  @job.perform_now
144
143
  assert_equal 1, RestartableJob.count
145
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
144
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
146
145
  assert job2.queued?, job2.attributes.ai
147
146
  end
148
147
 
@@ -151,7 +150,7 @@ module Plugins
151
150
  refute @job.expired?
152
151
  @job.perform_now
153
152
  assert_equal 1, RestartableJob.count
154
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
153
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
155
154
  assert job2.queued?, job2.attributes.ai
156
155
 
157
156
  # Copy across all attributes, except
@@ -171,14 +170,14 @@ module Plugins
171
170
  assert_equal nil, job2.worker_name
172
171
  assert_equal 0, job2.percent_complete
173
172
  assert_equal nil, job2.exception
174
- assert_equal({}, job2.result)
173
+ refute job2.result
175
174
  end
176
175
 
177
176
  it 'copies run_at when it is in the future' do
178
177
  @job = RestartableJob.create!(run_at: Time.now + 1.day, destroy_on_complete: true)
179
178
  @job.perform_now
180
179
  assert_equal 1, RestartableJob.count
181
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
180
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
182
181
  assert job2.run_at, job2.attributes.ai
183
182
  end
184
183
 
@@ -186,7 +185,7 @@ module Plugins
186
185
  @job = RestartableJob.create!(run_at: Time.now - 1.day, destroy_on_complete: true)
187
186
  @job.perform_now
188
187
  assert_equal 1, RestartableJob.count
189
- assert job2 = RestartableJob.where(id: {'$ne' => @job.id}).first
188
+ assert job2 = RestartableJob.where(:id.ne => @job.id).first
190
189
  assert_equal nil, job2.run_at
191
190
  end
192
191
  end
@@ -6,78 +6,80 @@ module Plugins
6
6
  # This job adds each event callback as they run into an array
7
7
  # [:start, :complete, :fail, :retry, :pause, :resume, :abort, :requeue]
8
8
  class PositivePathJob < RocketJob::Job
9
+ field :call_list, type: Array, default: []
10
+
9
11
  before_complete do
10
- arguments.first << 'before_complete_block'
12
+ call_list << 'before_complete_block'
11
13
  end
12
14
 
13
15
  after_complete do
14
- arguments.first << 'after_complete_block'
16
+ call_list << 'after_complete_block'
15
17
  end
16
18
 
17
19
  before_complete :before_complete_method
18
20
 
19
21
  before_start do
20
- arguments.first << 'before_start_block'
22
+ call_list << 'before_start_block'
21
23
  end
22
24
 
23
25
  before_start :before_start_method
24
26
 
25
27
  before_start do
26
- arguments.first << 'before_start2_block'
28
+ call_list << 'before_start2_block'
27
29
  end
28
30
 
29
31
  after_start :after_start_method
30
32
  after_complete :after_complete_method
31
33
 
32
34
  before_complete do
33
- arguments.first << 'before_complete2_block'
35
+ call_list << 'before_complete2_block'
34
36
  end
35
37
 
36
38
  after_start do
37
- arguments.first << 'after_start_block'
39
+ call_list << 'after_start_block'
38
40
  end
39
41
 
40
42
  after_complete do
41
- arguments.first << 'after_complete2_block'
43
+ call_list << 'after_complete2_block'
42
44
  end
43
45
 
44
46
  after_start :after_start_method
45
47
 
46
48
  after_start do
47
- arguments.first << 'after_start2_block'
49
+ call_list << 'after_start2_block'
48
50
  end
49
51
 
50
52
  before_start :before_start_method2
51
53
  before_complete :before_complete_method2
52
54
 
53
- def perform(list)
54
- list << 'perform'
55
+ def perform
56
+ call_list << 'perform'
55
57
  end
56
58
 
57
59
  private
58
60
 
59
61
  def before_start_method
60
- arguments.first << 'before_start_method'
62
+ call_list << 'before_start_method'
61
63
  end
62
64
 
63
65
  def before_start_method2
64
- arguments.first << 'before_start_method2'
66
+ call_list << 'before_start_method2'
65
67
  end
66
68
 
67
69
  def after_start_method
68
- arguments.first << 'after_start_method'
70
+ call_list << 'after_start_method'
69
71
  end
70
72
 
71
73
  def before_complete_method
72
- arguments.first << 'before_complete_method'
74
+ call_list << 'before_complete_method'
73
75
  end
74
76
 
75
77
  def before_complete_method2
76
- arguments.first << 'before_complete_method2'
78
+ call_list << 'before_complete_method2'
77
79
  end
78
80
 
79
81
  def after_complete_method
80
- arguments.first << 'after_complete_method'
82
+ call_list << 'after_complete_method'
81
83
  end
82
84
  end
83
85
 
@@ -88,11 +90,11 @@ module Plugins
88
90
 
89
91
  describe 'before_start after_start & before_complete after_complete' do
90
92
  it 'runs blocks and functions' do
91
- @job = PositivePathJob.new(arguments: [[]])
93
+ @job = PositivePathJob.new
92
94
  @job.perform_now
93
95
  assert @job.completed?, @job.attributes.ai
94
96
  expected = %w(before_start_block before_start_method before_start2_block before_start_method2 after_start2_block after_start_method after_start_block perform before_complete_block before_complete_method before_complete2_block before_complete_method2 after_complete2_block after_complete_method after_complete_block)
95
- assert_equal expected, @job.arguments.first, 'Sequence of before_perform callbacks is incorrect'
97
+ assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
96
98
  end
97
99
  end
98
100