rocketjob 3.5.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/bin/rocketjob +1 -0
  4. data/bin/rocketjob_batch_perf +11 -0
  5. data/lib/rocket_job/batch.rb +32 -0
  6. data/lib/rocket_job/batch/callbacks.rb +40 -0
  7. data/lib/rocket_job/batch/io.rb +154 -0
  8. data/lib/rocket_job/batch/logger.rb +57 -0
  9. data/lib/rocket_job/batch/lower_priority.rb +54 -0
  10. data/lib/rocket_job/batch/model.rb +157 -0
  11. data/lib/rocket_job/batch/performance.rb +99 -0
  12. data/lib/rocket_job/batch/result.rb +8 -0
  13. data/lib/rocket_job/batch/results.rb +9 -0
  14. data/lib/rocket_job/batch/state_machine.rb +102 -0
  15. data/lib/rocket_job/batch/statistics.rb +88 -0
  16. data/lib/rocket_job/batch/tabular.rb +56 -0
  17. data/lib/rocket_job/batch/tabular/input.rb +123 -0
  18. data/lib/rocket_job/batch/tabular/output.rb +59 -0
  19. data/lib/rocket_job/batch/throttle.rb +91 -0
  20. data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
  21. data/lib/rocket_job/batch/worker.rb +288 -0
  22. data/lib/rocket_job/cli.rb +29 -7
  23. data/lib/rocket_job/config.rb +1 -1
  24. data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
  25. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
  26. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
  27. data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
  28. data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
  29. data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
  31. data/lib/rocket_job/jobs/performance_job.rb +18 -0
  32. data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
  33. data/lib/rocket_job/plugins/document.rb +2 -8
  34. data/lib/rocket_job/plugins/job/persistence.rb +6 -4
  35. data/lib/rocket_job/plugins/job/throttle.rb +3 -6
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -2
  37. data/lib/rocket_job/server.rb +14 -3
  38. data/lib/rocket_job/sliced/input.rb +336 -0
  39. data/lib/rocket_job/sliced/output.rb +99 -0
  40. data/lib/rocket_job/sliced/slice.rb +166 -0
  41. data/lib/rocket_job/sliced/slices.rb +166 -0
  42. data/lib/rocket_job/sliced/writer/input.rb +60 -0
  43. data/lib/rocket_job/sliced/writer/output.rb +82 -0
  44. data/lib/rocket_job/version.rb +1 -1
  45. data/lib/rocket_job/worker.rb +2 -2
  46. data/lib/rocketjob.rb +28 -0
  47. metadata +51 -62
  48. data/test/config/database.yml +0 -5
  49. data/test/config/mongoid.yml +0 -88
  50. data/test/config_test.rb +0 -10
  51. data/test/dirmon_entry_test.rb +0 -313
  52. data/test/dirmon_job_test.rb +0 -216
  53. data/test/files/text.txt +0 -3
  54. data/test/job_test.rb +0 -71
  55. data/test/jobs/housekeeping_job_test.rb +0 -102
  56. data/test/jobs/on_demand_job_test.rb +0 -59
  57. data/test/jobs/upload_file_job_test.rb +0 -107
  58. data/test/plugins/cron_test.rb +0 -166
  59. data/test/plugins/job/callbacks_test.rb +0 -166
  60. data/test/plugins/job/defaults_test.rb +0 -53
  61. data/test/plugins/job/logger_test.rb +0 -56
  62. data/test/plugins/job/model_test.rb +0 -94
  63. data/test/plugins/job/persistence_test.rb +0 -94
  64. data/test/plugins/job/state_machine_test.rb +0 -116
  65. data/test/plugins/job/throttle_test.rb +0 -111
  66. data/test/plugins/job/worker_test.rb +0 -199
  67. data/test/plugins/processing_window_test.rb +0 -109
  68. data/test/plugins/restart_test.rb +0 -193
  69. data/test/plugins/retry_test.rb +0 -88
  70. data/test/plugins/singleton_test.rb +0 -92
  71. data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
  72. data/test/plugins/state_machine_test.rb +0 -67
  73. data/test/plugins/transaction_test.rb +0 -84
  74. data/test/test_db.sqlite3 +0 -0
  75. data/test/test_helper.rb +0 -17
@@ -1,166 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class CallbacksTest < Minitest::Test
7
- # This job adds each callback as they run into an array
8
- class BeforePerformJob < RocketJob::Job
9
- field :call_list, type: Array, default: []
10
-
11
- before_perform do
12
- call_list << 'before_perform_block'
13
- end
14
-
15
- before_perform :before_perform_method
16
-
17
- def perform
18
- call_list << 'perform'
19
- end
20
-
21
- private
22
-
23
- def before_perform_method
24
- call_list << 'before_perform_method'
25
- end
26
- end
27
-
28
- # This job adds each callback as they run into an array
29
- class AfterPerformJob < RocketJob::Job
30
- field :call_list, type: Array, default: []
31
-
32
- after_perform do
33
- call_list << 'after_perform_block'
34
- end
35
-
36
- after_perform :after_perform_method
37
-
38
- def perform
39
- call_list << 'perform'
40
- end
41
-
42
- private
43
-
44
- def after_perform_method
45
- call_list << 'after_perform_method'
46
- end
47
- end
48
-
49
- # This job adds each callback as they run into an array
50
- class AroundPerformJob < RocketJob::Job
51
- field :call_list, type: Array, default: []
52
-
53
- around_perform do |_job, block|
54
- call_list << 'around_perform_block_before'
55
- block.call
56
- call_list << 'around_perform_block_after'
57
- end
58
-
59
- around_perform :around_perform_method
60
-
61
- def perform
62
- call_list << 'perform'
63
- end
64
-
65
- private
66
-
67
- def around_perform_method
68
- call_list << 'around_perform_method_before'
69
- yield
70
- call_list << 'around_perform_method_after'
71
- end
72
- end
73
-
74
- # This job adds each callback as they run into an array
75
- class CombinedPerformJob < RocketJob::Job
76
- field :call_list, type: Array, default: []
77
-
78
- before_perform do
79
- call_list << 'before_perform_block'
80
- end
81
-
82
- after_perform do
83
- call_list << 'after_perform_block'
84
- end
85
-
86
- around_perform do |_job, block|
87
- call_list << 'around_perform_block_before'
88
- block.call
89
- call_list << 'around_perform_block_after'
90
- end
91
-
92
- before_perform :before_perform_method
93
-
94
- around_perform :around_perform_method
95
-
96
- after_perform :after_perform_method
97
-
98
- def perform
99
- call_list << 'perform'
100
- end
101
-
102
- private
103
-
104
- def before_perform_method
105
- call_list << 'before_perform_method'
106
- end
107
-
108
- def around_perform_method
109
- call_list << 'around_perform_method_before'
110
- yield
111
- call_list << 'around_perform_method_after'
112
- end
113
-
114
- def after_perform_method
115
- call_list << 'after_perform_method'
116
- end
117
- end
118
-
119
- describe RocketJob::Plugins::Job::Callbacks do
120
- after do
121
- @job.destroy if @job && !@job.new_record?
122
- end
123
-
124
- describe '#before_perform' do
125
- it 'runs blocks and functions' do
126
- @job = BeforePerformJob.new
127
- @job.perform_now
128
- assert @job.completed?, @job.attributes.ai
129
- expected = %w[before_perform_block before_perform_method perform]
130
- assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
131
- end
132
- end
133
-
134
- describe '#after_perform' do
135
- it 'runs blocks and functions' do
136
- @job = AfterPerformJob.new
137
- @job.perform_now
138
- assert @job.completed?, @job.attributes.ai
139
- expected = %w[perform after_perform_method after_perform_block]
140
- assert_equal expected, @job.call_list, 'Sequence of after_perform callbacks is incorrect'
141
- end
142
- end
143
-
144
- describe '#around_perform' do
145
- it 'runs blocks and functions' do
146
- @job = AroundPerformJob.new
147
- @job.perform_now
148
- assert @job.completed?, @job.attributes.ai
149
- expected = %w[around_perform_block_before around_perform_method_before perform around_perform_method_after around_perform_block_after]
150
- assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
151
- end
152
- end
153
-
154
- describe 'all callbacks' do
155
- it 'runs them in the right order' do
156
- @job = CombinedPerformJob.new
157
- @job.perform_now
158
- assert @job.completed?, @job.attributes.ai
159
- expected = %w[before_perform_block around_perform_block_before before_perform_method around_perform_method_before perform after_perform_method around_perform_method_after around_perform_block_after after_perform_block]
160
- assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
161
- end
162
- end
163
- end
164
- end
165
- end
166
- end
@@ -1,53 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class DefaultsTest < Minitest::Test
7
- class ParentJob < RocketJob::Job
8
- self.priority = 53
9
- self.description = 'Hello'
10
-
11
- def perform
12
- end
13
- end
14
-
15
- class ChildJob < ParentJob
16
- self.priority = 72
17
-
18
- def perform
19
- end
20
- end
21
-
22
- describe RocketJob::Plugins::Job do
23
- after do
24
- @job.destroy if @job && !@job.new_record?
25
- end
26
-
27
- describe '.rocket_job' do
28
- it 'sets defaults after initialize' do
29
- @job = ParentJob.new
30
- assert_equal 53, @job.priority
31
- assert_equal 'Hello', @job.description
32
- end
33
-
34
- it 'can override defaults on initialize' do
35
- @job = ParentJob.new(priority: 72, description: 'More')
36
- assert_equal 72, @job.priority
37
- assert_equal 'More', @job.description
38
- end
39
-
40
- it 'allows a child to override parent defaults' do
41
- @job = ChildJob.new
42
- assert_equal 72, @job.priority
43
- end
44
-
45
- it 'passes down parent defaults' do
46
- @job = ChildJob.new
47
- assert_equal 'Hello', @job.description
48
- end
49
- end
50
- end
51
- end
52
- end
53
- end
@@ -1,56 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- module Plugins
5
- module Job
6
- class LoggerTest < Minitest::Test
7
- class LoggerJob < RocketJob::Job
8
- def perform
9
- logger.debug('DONE', value: 123, other_value: 'HI')
10
- end
11
- end
12
-
13
- describe RocketJob::Plugins::Job::Logger do
14
- before do
15
- LoggerJob.delete_all
16
- end
17
-
18
- after do
19
- @job.destroy if @job && !@job.new_record?
20
- SemanticLogger.flush
21
- end
22
-
23
- describe '#logger' do
24
- it 'uses semantic logger' do
25
- @job = LoggerJob.new
26
- assert_kind_of SemanticLogger::Logger, @job.logger, @job.logger.ai
27
- assert_equal @job.class.name, @job.logger.name, @job.logger.ai
28
- end
29
-
30
- it 'allows perform to log its own data' do
31
- @job = LoggerJob.new
32
- @job.perform_now
33
- end
34
-
35
- it 'adds start logging' do
36
- @job = LoggerJob.new
37
- info_called = false
38
- @job.logger.stub(:info, ->(description) { info_called = true if description == 'Start #perform' }) do
39
- @job.perform_now
40
- end
41
- assert info_called, "In Plugins::Job::Logger.around_perform logger.info('Start #perform') not called"
42
- end
43
-
44
- it 'adds completed logging' do
45
- @job = LoggerJob.new
46
- measure_called = false
47
- @job.logger.stub(:measure_info, ->(description, *_args) { measure_called = true if description == 'Completed #perform' }) do
48
- @job.perform_now
49
- end
50
- assert measure_called, "In Plugins::Job::Logger.around_perform logger.measure_info('Completed #perform') not called"
51
- end
52
- end
53
- end
54
- end
55
- end
56
- end
@@ -1,94 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class ModelTest < Minitest::Test
7
- class SimpleJob < RocketJob::Job
8
- def perform
9
- 10
10
- end
11
- end
12
-
13
- class TwoArgumentJob < RocketJob::Job
14
- self.priority = 53
15
-
16
- def perform(a, b)
17
- a + b
18
- end
19
- end
20
-
21
- describe RocketJob::Plugins::Job::Model do
22
- after do
23
- @job.destroy if @job && !@job.new_record?
24
- @job2.destroy if @job2 && !@job2.new_record?
25
- @job3.destroy if @job3 && !@job3.new_record?
26
- end
27
-
28
- describe '#scheduled?' do
29
- it 'returns true if job is queued to run in the future' do
30
- @job = SimpleJob.new(run_at: 1.day.from_now)
31
- assert_equal true, @job.queued?
32
- assert_equal true, @job.scheduled?
33
- @job.start
34
- assert_equal true, @job.running?
35
- assert_equal false, @job.scheduled?
36
- end
37
-
38
- it 'returns false if job is queued and can be run now' do
39
- @job = SimpleJob.new
40
- assert_equal true, @job.queued?
41
- assert_equal false, @job.scheduled?
42
- end
43
-
44
- it 'returns false if job is running' do
45
- @job = SimpleJob.new
46
- @job.start
47
- assert_equal true, @job.running?
48
- assert_equal false, @job.scheduled?
49
- end
50
- end
51
-
52
- describe 'with queued jobs' do
53
- before do
54
- @job = SimpleJob.create!(description: 'first')
55
- @job2 = SimpleJob.create!(description: 'second', run_at: 1.day.from_now)
56
- @job3 = SimpleJob.create!(description: 'third', run_at: 2.days.from_now)
57
- end
58
-
59
- describe '#scheduled' do
60
- it 'returns only scheduled jobs' do
61
- count = 0
62
- RocketJob::Job.scheduled.each do |job|
63
- count += 1
64
- assert job.scheduled?
65
- end
66
- assert 2, count
67
- end
68
- end
69
-
70
- describe '#queued_now' do
71
- it 'returns only queued jobs, not scheduled ones' do
72
- count = 0
73
- RocketJob::Job.queued_now.each do |job|
74
- count += 1
75
- refute job.scheduled?
76
- end
77
- assert 1, count
78
- end
79
- end
80
-
81
- describe '#queued' do
82
- it 'returns all queued jobs' do
83
- count = 0
84
- RocketJob::Job.queued.each do |_job|
85
- count += 1
86
- end
87
- assert 3, count
88
- end
89
- end
90
- end
91
- end
92
- end
93
- end
94
- end
@@ -1,94 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class PersistenceTest < Minitest::Test
7
- class PersistJob < RocketJob::Job
8
- self.priority = 53
9
- field :data, type: Hash
10
-
11
- def perform(hash)
12
- hash
13
- end
14
- end
15
-
16
- describe RocketJob::Plugins::Job::Persistence do
17
- before do
18
- RocketJob::Job.destroy_all
19
- @description = 'Hello World'
20
- @data = {'key' => 'value'}
21
- @job = PersistJob.new(
22
- description: @description,
23
- data: @data,
24
- destroy_on_complete: false
25
- )
26
- end
27
-
28
- after do
29
- @job.destroy if @job && !@job.new_record?
30
- @job2.destroy if @job2 && !@job2.new_record?
31
- @job3.destroy if @job3 && !@job3.new_record?
32
- end
33
-
34
- describe '.config' do
35
- it 'support multiple databases' do
36
- assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
37
- end
38
- end
39
-
40
- describe '.rocket_job' do
41
- it 'sets defaults after initialize' do
42
- assert_equal 53, @job.priority
43
- end
44
- end
45
-
46
- describe '#reload' do
47
- it 'handle hash' do
48
- assert_equal 'value', @job.data['key']
49
- @job.worker_name = nil
50
- @job.save!
51
- @job.worker_name = '123'
52
- @job.reload
53
- assert @job.data.is_a?(Hash), @job.data.class.ai
54
- assert_equal 'value', @job.data['key']
55
- assert_nil @job.worker_name
56
- end
57
- end
58
-
59
- describe '#save!' do
60
- it 'save a blank job' do
61
- @job.save!
62
- assert_nil @job.worker_name
63
- assert_nil @job.completed_at
64
- assert @job.created_at
65
- assert_equal @description, @job.description
66
- assert_equal false, @job.destroy_on_complete
67
- assert_nil @job.expires_at
68
- assert_equal @data, @job.data
69
- assert_equal 0, @job.percent_complete
70
- assert_equal 53, @job.priority
71
- assert_equal 0, @job.failure_count
72
- assert_nil @job.run_at
73
- assert_nil @job.started_at
74
- assert_equal :queued, @job.state
75
- end
76
- end
77
-
78
- describe '.counts_by_state' do
79
- it 'returns states as symbols' do
80
- @job.start!
81
- @job2 = PersistJob.create!(data: {key: 'value'})
82
- @job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
83
- counts = RocketJob::Job.counts_by_state
84
- assert_equal 4, counts.size, counts.ai
85
- assert_equal 1, counts[:running]
86
- assert_equal 2, counts[:queued]
87
- assert_equal 1, counts[:queued_now]
88
- assert_equal 1, counts[:scheduled]
89
- end
90
- end
91
- end
92
- end
93
- end
94
- end