rocketjob 3.4.3 → 3.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (64) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +27 -0
  3. data/bin/rocketjob +1 -1
  4. data/lib/rocket_job/active_worker.rb +4 -3
  5. data/lib/rocket_job/cli.rb +13 -12
  6. data/lib/rocket_job/config.rb +17 -13
  7. data/lib/rocket_job/dirmon_entry.rb +88 -91
  8. data/lib/rocket_job/extensions/mongo/logging.rb +8 -4
  9. data/lib/rocket_job/extensions/mongoid/factory.rb +8 -6
  10. data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -4
  11. data/lib/rocket_job/heartbeat.rb +7 -8
  12. data/lib/rocket_job/job_exception.rb +6 -5
  13. data/lib/rocket_job/jobs/dirmon_job.rb +5 -7
  14. data/lib/rocket_job/jobs/housekeeping_job.rb +3 -2
  15. data/lib/rocket_job/jobs/on_demand_job.rb +104 -0
  16. data/lib/rocket_job/jobs/simple_job.rb +0 -2
  17. data/lib/rocket_job/jobs/upload_file_job.rb +100 -0
  18. data/lib/rocket_job/performance.rb +15 -10
  19. data/lib/rocket_job/plugins/cron.rb +7 -124
  20. data/lib/rocket_job/plugins/document.rb +8 -10
  21. data/lib/rocket_job/plugins/job/callbacks.rb +0 -1
  22. data/lib/rocket_job/plugins/job/logger.rb +0 -1
  23. data/lib/rocket_job/plugins/job/model.rb +15 -20
  24. data/lib/rocket_job/plugins/job/persistence.rb +3 -13
  25. data/lib/rocket_job/plugins/job/state_machine.rb +1 -2
  26. data/lib/rocket_job/plugins/job/throttle.rb +16 -12
  27. data/lib/rocket_job/plugins/job/worker.rb +15 -19
  28. data/lib/rocket_job/plugins/processing_window.rb +2 -2
  29. data/lib/rocket_job/plugins/restart.rb +3 -4
  30. data/lib/rocket_job/plugins/retry.rb +2 -3
  31. data/lib/rocket_job/plugins/singleton.rb +2 -3
  32. data/lib/rocket_job/plugins/state_machine.rb +19 -23
  33. data/lib/rocket_job/rocket_job.rb +4 -5
  34. data/lib/rocket_job/server.rb +35 -41
  35. data/lib/rocket_job/version.rb +2 -2
  36. data/lib/rocket_job/worker.rb +22 -21
  37. data/lib/rocketjob.rb +2 -0
  38. data/test/config/mongoid.yml +2 -2
  39. data/test/config_test.rb +0 -2
  40. data/test/dirmon_entry_test.rb +161 -134
  41. data/test/dirmon_job_test.rb +80 -78
  42. data/test/job_test.rb +0 -2
  43. data/test/jobs/housekeeping_job_test.rb +0 -1
  44. data/test/jobs/on_demand_job_test.rb +59 -0
  45. data/test/jobs/upload_file_job_test.rb +99 -0
  46. data/test/plugins/cron_test.rb +1 -3
  47. data/test/plugins/job/callbacks_test.rb +8 -13
  48. data/test/plugins/job/defaults_test.rb +0 -1
  49. data/test/plugins/job/logger_test.rb +2 -4
  50. data/test/plugins/job/model_test.rb +1 -2
  51. data/test/plugins/job/persistence_test.rb +0 -2
  52. data/test/plugins/job/state_machine_test.rb +0 -2
  53. data/test/plugins/job/throttle_test.rb +6 -8
  54. data/test/plugins/job/worker_test.rb +1 -2
  55. data/test/plugins/processing_window_test.rb +0 -2
  56. data/test/plugins/restart_test.rb +0 -1
  57. data/test/plugins/retry_test.rb +1 -2
  58. data/test/plugins/singleton_test.rb +0 -2
  59. data/test/plugins/state_machine_event_callbacks_test.rb +1 -2
  60. data/test/plugins/state_machine_test.rb +0 -2
  61. data/test/plugins/transaction_test.rb +5 -7
  62. data/test/test_db.sqlite3 +0 -0
  63. data/test/test_helper.rb +2 -1
  64. metadata +42 -36
data/test/job_test.rb CHANGED
@@ -2,7 +2,6 @@ require_relative 'test_helper'
2
2
 
3
3
  # Unit Test for RocketJob::Job
4
4
  class JobTest < Minitest::Test
5
-
6
5
  class SimpleJob < RocketJob::Job
7
6
  def perform
8
7
  end
@@ -68,6 +67,5 @@ class JobTest < Minitest::Test
68
67
  assert_equal worker_name2, @job2.worker_name
69
68
  end
70
69
  end
71
-
72
70
  end
73
71
  end
@@ -98,6 +98,5 @@ class HousekeepingJobTest < Minitest::Test
98
98
  assert_equal 1, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
99
99
  end
100
100
  end
101
-
102
101
  end
103
102
  end
@@ -0,0 +1,59 @@
1
+ require_relative '../test_helper'
2
+
3
+ class OnDemandJobTest < Minitest::Test
4
+ describe RocketJob::Jobs::OnDemandJob do
5
+ before do
6
+ RocketJob::Jobs::OnDemandJob.delete_all
7
+ end
8
+
9
+ describe '#perform' do
10
+ it 'hello world' do
11
+ code = <<~CODE
12
+ logger.info 'Hello World'
13
+ CODE
14
+
15
+ job = RocketJob::Jobs::OnDemandJob.new(code: code)
16
+ job.perform_now
17
+ end
18
+
19
+ it 'retain output' do
20
+ code = <<~CODE
21
+ {'value' => 'h' * 24}
22
+ CODE
23
+
24
+ job = RocketJob::Jobs::OnDemandJob.new(
25
+ code: code,
26
+ collect_output: true
27
+ )
28
+ job.perform_now
29
+ assert_equal 'h' * 24, job.result['value']
30
+ end
31
+
32
+ it 'accepts input data' do
33
+ code = <<~CODE
34
+ {'value' => data['a'] * data['b']}
35
+ CODE
36
+
37
+ job = RocketJob::Jobs::OnDemandJob.new(
38
+ code: code,
39
+ collect_output: true,
40
+ data: {'a' => 10, 'b' => 2}
41
+ )
42
+ job.perform_now
43
+ assert_equal 20, job.result['value']
44
+ end
45
+
46
+ it 'validates code' do
47
+ code = <<~CODE
48
+ def bad code
49
+ CODE
50
+
51
+ job = RocketJob::Jobs::OnDemandJob.new(code: code)
52
+ refute job.valid?
53
+ assert_raises Mongoid::Errors::Validations do
54
+ job.perform_now
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,99 @@
1
+ require_relative '../test_helper'
2
+
3
+ class UploadFileJobTest < Minitest::Test
4
+ class TestJob < RocketJob::Job
5
+ field :upload_file_name, type: String
6
+
7
+ def upload(file_name)
8
+ self.upload_file_name = file_name
9
+ end
10
+
11
+ def perform
12
+ end
13
+ end
14
+
15
+ class BatchTestJob < RocketJob::Job
16
+ field :upload_file_name, type: String
17
+ field :saved_streams, type: Array
18
+
19
+ def upload(file_name, streams: nil)
20
+ self.upload_file_name = file_name
21
+ self.saved_streams = streams
22
+ end
23
+
24
+ def perform
25
+ end
26
+ end
27
+
28
+ class BadJob < RocketJob::Job
29
+ def perform
30
+ end
31
+ end
32
+
33
+ describe RocketJob::Jobs::UploadFileJob do
34
+ before do
35
+ RocketJob::Job.delete_all
36
+ end
37
+
38
+ describe '#valid?' do
39
+ it 'validates upload_file_name' do
40
+ job = RocketJob::Jobs::UploadFileJob.new(job_class_name: UploadFileJobTest::TestJob.to_s)
41
+ refute job.valid?
42
+ assert_equal ["can't be blank"], job.errors.messages[:upload_file_name]
43
+ end
44
+
45
+ it 'validates file does not exist' do
46
+ job = RocketJob::Jobs::UploadFileJob.new(
47
+ job_class_name: UploadFileJobTest::TestJob.to_s,
48
+ upload_file_name: '/tmp/blah'
49
+ )
50
+ refute job.valid?
51
+ assert_equal ['Upload file: /tmp/blah does not exist.'], job.errors.messages[:upload_file_name]
52
+ end
53
+
54
+ it 'validates job_class_name' do
55
+ job = RocketJob::Jobs::UploadFileJob.new(job_class_name: UploadFileJobTest::BadJob.to_s)
56
+ refute job.valid?
57
+ assert_equal ['UploadFileJobTest::BadJob must implement any one of: :upload :upload_file_name= :full_file_name= instance methods'], job.errors.messages[:job_class_name]
58
+ end
59
+
60
+ it 'with valid job and upload_file_name' do
61
+ job = RocketJob::Jobs::UploadFileJob.new(
62
+ job_class_name: UploadFileJobTest::TestJob.to_s,
63
+ upload_file_name: __FILE__
64
+ )
65
+ assert job.valid?
66
+ end
67
+ end
68
+
69
+ describe '#perform' do
70
+ let :job do
71
+ RocketJob::Jobs::UploadFileJob.new(
72
+ job_class_name: UploadFileJobTest::TestJob.name,
73
+ upload_file_name: __FILE__
74
+ )
75
+ end
76
+
77
+ it 'creates the job' do
78
+ job.perform_now
79
+ assert_equal 1, UploadFileJobTest::TestJob.count
80
+ assert job = UploadFileJobTest::TestJob.first
81
+ end
82
+
83
+ it 'calls upload' do
84
+ job.perform_now
85
+ assert job = UploadFileJobTest::TestJob.first
86
+ assert_equal __FILE__, job.upload_file_name
87
+ end
88
+
89
+ it 'calls upload with original_file_name' do
90
+ job.job_class_name = BatchTestJob.name
91
+ job.original_file_name = 'file.zip'
92
+ job.perform_now
93
+ assert created_job = UploadFileJobTest::BatchTestJob.first
94
+ assert_equal __FILE__, created_job.upload_file_name
95
+ assert_equal %i[file zip], created_job.saved_streams
96
+ end
97
+ end
98
+ end
99
+ end
@@ -15,7 +15,7 @@ module Plugins
15
15
  include RocketJob::Plugins::Cron
16
16
 
17
17
  def perform
18
- raise 'oh no' if failure_count == 0
18
+ raise 'oh no' if failure_count.zero?
19
19
  end
20
20
  end
21
21
 
@@ -60,7 +60,6 @@ module Plugins
60
60
  assert_equal Time.parse('2015-12-10 06:00:00 UTC'), @job.run_at
61
61
  end
62
62
  end
63
-
64
63
  end
65
64
 
66
65
  describe '#save' do
@@ -162,7 +161,6 @@ module Plugins
162
161
  end
163
162
  end
164
163
  end
165
-
166
164
  end
167
165
  end
168
166
  end
@@ -23,7 +23,6 @@ module Plugins
23
23
  def before_perform_method
24
24
  call_list << 'before_perform_method'
25
25
  end
26
-
27
26
  end
28
27
 
29
28
  # This job adds each callback as they run into an array
@@ -45,14 +44,13 @@ module Plugins
45
44
  def after_perform_method
46
45
  call_list << 'after_perform_method'
47
46
  end
48
-
49
47
  end
50
48
 
51
49
  # This job adds each callback as they run into an array
52
50
  class AroundPerformJob < RocketJob::Job
53
51
  field :call_list, type: Array, default: []
54
52
 
55
- around_perform do |job, block|
53
+ around_perform do |_job, block|
56
54
  call_list << 'around_perform_block_before'
57
55
  block.call
58
56
  call_list << 'around_perform_block_after'
@@ -71,7 +69,6 @@ module Plugins
71
69
  yield
72
70
  call_list << 'around_perform_method_after'
73
71
  end
74
-
75
72
  end
76
73
 
77
74
  # This job adds each callback as they run into an array
@@ -86,7 +83,7 @@ module Plugins
86
83
  call_list << 'after_perform_block'
87
84
  end
88
85
 
89
- around_perform do |job, block|
86
+ around_perform do |_job, block|
90
87
  call_list << 'around_perform_block_before'
91
88
  block.call
92
89
  call_list << 'around_perform_block_after'
@@ -108,16 +105,15 @@ module Plugins
108
105
  call_list << 'before_perform_method'
109
106
  end
110
107
 
111
- def around_perform_method(&block)
108
+ def around_perform_method
112
109
  call_list << 'around_perform_method_before'
113
- block.call
110
+ yield
114
111
  call_list << 'around_perform_method_after'
115
112
  end
116
113
 
117
114
  def after_perform_method
118
115
  call_list << 'after_perform_method'
119
116
  end
120
-
121
117
  end
122
118
 
123
119
  describe RocketJob::Plugins::Job::Callbacks do
@@ -130,7 +126,7 @@ module Plugins
130
126
  @job = BeforePerformJob.new
131
127
  @job.perform_now
132
128
  assert @job.completed?, @job.attributes.ai
133
- expected = %w(before_perform_block before_perform_method perform)
129
+ expected = %w[before_perform_block before_perform_method perform]
134
130
  assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
135
131
  end
136
132
  end
@@ -140,7 +136,7 @@ module Plugins
140
136
  @job = AfterPerformJob.new
141
137
  @job.perform_now
142
138
  assert @job.completed?, @job.attributes.ai
143
- expected = %w(perform after_perform_method after_perform_block)
139
+ expected = %w[perform after_perform_method after_perform_block]
144
140
  assert_equal expected, @job.call_list, 'Sequence of after_perform callbacks is incorrect'
145
141
  end
146
142
  end
@@ -150,7 +146,7 @@ module Plugins
150
146
  @job = AroundPerformJob.new
151
147
  @job.perform_now
152
148
  assert @job.completed?, @job.attributes.ai
153
- expected = %w(around_perform_block_before around_perform_method_before perform around_perform_method_after around_perform_block_after)
149
+ expected = %w[around_perform_block_before around_perform_method_before perform around_perform_method_after around_perform_block_after]
154
150
  assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
155
151
  end
156
152
  end
@@ -160,11 +156,10 @@ module Plugins
160
156
  @job = CombinedPerformJob.new
161
157
  @job.perform_now
162
158
  assert @job.completed?, @job.attributes.ai
163
- expected = %w(before_perform_block around_perform_block_before before_perform_method around_perform_method_before perform after_perform_method around_perform_method_after around_perform_block_after after_perform_block)
159
+ expected = %w[before_perform_block around_perform_block_before before_perform_method around_perform_method_before perform after_perform_method around_perform_method_after around_perform_block_after after_perform_block]
164
160
  assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
165
161
  end
166
162
  end
167
-
168
163
  end
169
164
  end
170
165
  end
@@ -4,7 +4,6 @@ module Plugins
4
4
  module Job
5
5
  # Unit Test for RocketJob::Job
6
6
  class DefaultsTest < Minitest::Test
7
-
8
7
  class ParentJob < RocketJob::Job
9
8
  self.priority = 53
10
9
  self.description = 'Hello'
@@ -4,7 +4,6 @@ require_relative '../../test_helper'
4
4
  module Plugins
5
5
  module Job
6
6
  class LoggerTest < Minitest::Test
7
-
8
7
  class LoggerJob < RocketJob::Job
9
8
  def perform
10
9
  logger.debug('DONE', value: 123, other_value: 'HI')
@@ -36,7 +35,7 @@ module Plugins
36
35
  it 'adds start logging' do
37
36
  @job = LoggerJob.new
38
37
  info_called = false
39
- @job.logger.stub(:info, -> description { info_called = true if description == 'Start #perform' }) do
38
+ @job.logger.stub(:info, ->(description) { info_called = true if description == 'Start #perform' }) do
40
39
  @job.perform_now
41
40
  end
42
41
  assert info_called, "In Plugins::Job::Logger.around_perform logger.info('Start #perform') not called"
@@ -45,13 +44,12 @@ module Plugins
45
44
  it 'adds completed logging' do
46
45
  @job = LoggerJob.new
47
46
  measure_called = false
48
- @job.logger.stub(:measure_info, -> description, *args { measure_called = true if description == 'Completed #perform' }) do
47
+ @job.logger.stub(:measure_info, ->(description, *_args) { measure_called = true if description == 'Completed #perform' }) do
49
48
  @job.perform_now
50
49
  end
51
50
  assert measure_called, "In Plugins::Job::Logger.around_perform logger.measure_info('Completed #perform') not called"
52
51
  end
53
52
  end
54
-
55
53
  end
56
54
  end
57
55
  end
@@ -81,14 +81,13 @@ module Plugins
81
81
  describe '#queued' do
82
82
  it 'returns all queued jobs' do
83
83
  count = 0
84
- RocketJob::Job.queued.each do |job|
84
+ RocketJob::Job.queued.each do |_job|
85
85
  count += 1
86
86
  end
87
87
  assert 3, count
88
88
  end
89
89
  end
90
90
  end
91
-
92
91
  end
93
92
  end
94
93
  end
@@ -4,7 +4,6 @@ module Plugins
4
4
  module Job
5
5
  # Unit Test for RocketJob::Job
6
6
  class PersistenceTest < Minitest::Test
7
-
8
7
  class PersistJob < RocketJob::Job
9
8
  self.priority = 53
10
9
  field :data, type: Hash
@@ -89,7 +88,6 @@ module Plugins
89
88
  assert_equal 1, counts[:scheduled]
90
89
  end
91
90
  end
92
-
93
91
  end
94
92
  end
95
93
  end
@@ -3,7 +3,6 @@ module Plugins
3
3
  module Job
4
4
  # Unit Test for RocketJob::Job
5
5
  class StateMachineTest < Minitest::Test
6
-
7
6
  class StateMachineJob < RocketJob::Job
8
7
  def perform
9
8
  end
@@ -111,7 +110,6 @@ module Plugins
111
110
  assert_nil @job.exception
112
111
  end
113
112
  end
114
-
115
113
  end
116
114
  end
117
115
  end
@@ -4,7 +4,6 @@ require_relative '../../test_helper'
4
4
  module Plugins
5
5
  module Job
6
6
  class ThrottleTest < Minitest::Test
7
-
8
7
  class ThrottleJob < RocketJob::Job
9
8
  # Only allow one to be processed at a time
10
9
  self.throttle_running_jobs = 1
@@ -20,20 +19,20 @@ module Plugins
20
19
  RocketJob::Job.delete_all
21
20
  end
22
21
 
23
- describe '.has_throttle?' do
22
+ describe '.throttle?' do
24
23
  it 'defines the running jobs throttle' do
25
- assert ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
26
- refute ThrottleJob.has_throttle?(:blah?), ThrottleJob.rocket_job_throttles
24
+ assert ThrottleJob.throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
25
+ refute ThrottleJob.throttle?(:blah?), ThrottleJob.rocket_job_throttles
27
26
  end
28
27
  end
29
28
 
30
29
  describe '.undefine_throttle' do
31
30
  it 'undefines the running jobs throttle' do
32
- assert ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
31
+ assert ThrottleJob.throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
33
32
  ThrottleJob.undefine_throttle(:throttle_running_jobs_exceeded?)
34
- refute ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
33
+ refute ThrottleJob.throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
35
34
  ThrottleJob.define_throttle(:throttle_running_jobs_exceeded?)
36
- assert ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
35
+ assert ThrottleJob.throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
37
36
  end
38
37
  end
39
38
 
@@ -106,7 +105,6 @@ module Plugins
106
105
  assert_equal 1, filter.size
107
106
  end
108
107
  end
109
-
110
108
  end
111
109
  end
112
110
  end
@@ -98,7 +98,7 @@ module Plugins
98
98
  @job = NoisyJob.new
99
99
  @job.log_level = :warn
100
100
  logged = false
101
- @job.logger.stub(:log_internal, -> level, index, message, payload, exception { logged = true if message.include?('some very noisy logging') }) do
101
+ @job.logger.stub(:log_internal, ->(_level, _index, message, _payload, _exception) { logged = true if message.include?('some very noisy logging') }) do
102
102
  @job.perform_now
103
103
  end
104
104
  assert_equal false, logged
@@ -193,7 +193,6 @@ module Plugins
193
193
  assert active_worker.duration
194
194
  end
195
195
  end
196
-
197
196
  end
198
197
  end
199
198
  end