rocketjob 3.5.2 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/bin/rocketjob +1 -0
  4. data/bin/rocketjob_batch_perf +11 -0
  5. data/lib/rocket_job/batch.rb +32 -0
  6. data/lib/rocket_job/batch/callbacks.rb +40 -0
  7. data/lib/rocket_job/batch/io.rb +154 -0
  8. data/lib/rocket_job/batch/logger.rb +57 -0
  9. data/lib/rocket_job/batch/lower_priority.rb +54 -0
  10. data/lib/rocket_job/batch/model.rb +157 -0
  11. data/lib/rocket_job/batch/performance.rb +99 -0
  12. data/lib/rocket_job/batch/result.rb +8 -0
  13. data/lib/rocket_job/batch/results.rb +9 -0
  14. data/lib/rocket_job/batch/state_machine.rb +102 -0
  15. data/lib/rocket_job/batch/statistics.rb +88 -0
  16. data/lib/rocket_job/batch/tabular.rb +56 -0
  17. data/lib/rocket_job/batch/tabular/input.rb +123 -0
  18. data/lib/rocket_job/batch/tabular/output.rb +59 -0
  19. data/lib/rocket_job/batch/throttle.rb +91 -0
  20. data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
  21. data/lib/rocket_job/batch/worker.rb +288 -0
  22. data/lib/rocket_job/cli.rb +29 -7
  23. data/lib/rocket_job/config.rb +1 -1
  24. data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
  25. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
  26. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
  27. data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
  28. data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
  29. data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
  31. data/lib/rocket_job/jobs/performance_job.rb +18 -0
  32. data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
  33. data/lib/rocket_job/plugins/document.rb +2 -8
  34. data/lib/rocket_job/plugins/job/persistence.rb +6 -4
  35. data/lib/rocket_job/plugins/job/throttle.rb +3 -6
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -2
  37. data/lib/rocket_job/server.rb +14 -3
  38. data/lib/rocket_job/sliced/input.rb +336 -0
  39. data/lib/rocket_job/sliced/output.rb +99 -0
  40. data/lib/rocket_job/sliced/slice.rb +166 -0
  41. data/lib/rocket_job/sliced/slices.rb +166 -0
  42. data/lib/rocket_job/sliced/writer/input.rb +60 -0
  43. data/lib/rocket_job/sliced/writer/output.rb +82 -0
  44. data/lib/rocket_job/version.rb +1 -1
  45. data/lib/rocket_job/worker.rb +2 -2
  46. data/lib/rocketjob.rb +28 -0
  47. metadata +51 -62
  48. data/test/config/database.yml +0 -5
  49. data/test/config/mongoid.yml +0 -88
  50. data/test/config_test.rb +0 -10
  51. data/test/dirmon_entry_test.rb +0 -313
  52. data/test/dirmon_job_test.rb +0 -216
  53. data/test/files/text.txt +0 -3
  54. data/test/job_test.rb +0 -71
  55. data/test/jobs/housekeeping_job_test.rb +0 -102
  56. data/test/jobs/on_demand_job_test.rb +0 -59
  57. data/test/jobs/upload_file_job_test.rb +0 -107
  58. data/test/plugins/cron_test.rb +0 -166
  59. data/test/plugins/job/callbacks_test.rb +0 -166
  60. data/test/plugins/job/defaults_test.rb +0 -53
  61. data/test/plugins/job/logger_test.rb +0 -56
  62. data/test/plugins/job/model_test.rb +0 -94
  63. data/test/plugins/job/persistence_test.rb +0 -94
  64. data/test/plugins/job/state_machine_test.rb +0 -116
  65. data/test/plugins/job/throttle_test.rb +0 -111
  66. data/test/plugins/job/worker_test.rb +0 -199
  67. data/test/plugins/processing_window_test.rb +0 -109
  68. data/test/plugins/restart_test.rb +0 -193
  69. data/test/plugins/retry_test.rb +0 -88
  70. data/test/plugins/singleton_test.rb +0 -92
  71. data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
  72. data/test/plugins/state_machine_test.rb +0 -67
  73. data/test/plugins/transaction_test.rb +0 -84
  74. data/test/test_db.sqlite3 +0 -0
  75. data/test/test_helper.rb +0 -17
@@ -1,166 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class CallbacksTest < Minitest::Test
7
- # This job adds each callback as they run into an array
8
- class BeforePerformJob < RocketJob::Job
9
- field :call_list, type: Array, default: []
10
-
11
- before_perform do
12
- call_list << 'before_perform_block'
13
- end
14
-
15
- before_perform :before_perform_method
16
-
17
- def perform
18
- call_list << 'perform'
19
- end
20
-
21
- private
22
-
23
- def before_perform_method
24
- call_list << 'before_perform_method'
25
- end
26
- end
27
-
28
- # This job adds each callback as they run into an array
29
- class AfterPerformJob < RocketJob::Job
30
- field :call_list, type: Array, default: []
31
-
32
- after_perform do
33
- call_list << 'after_perform_block'
34
- end
35
-
36
- after_perform :after_perform_method
37
-
38
- def perform
39
- call_list << 'perform'
40
- end
41
-
42
- private
43
-
44
- def after_perform_method
45
- call_list << 'after_perform_method'
46
- end
47
- end
48
-
49
- # This job adds each callback as they run into an array
50
- class AroundPerformJob < RocketJob::Job
51
- field :call_list, type: Array, default: []
52
-
53
- around_perform do |_job, block|
54
- call_list << 'around_perform_block_before'
55
- block.call
56
- call_list << 'around_perform_block_after'
57
- end
58
-
59
- around_perform :around_perform_method
60
-
61
- def perform
62
- call_list << 'perform'
63
- end
64
-
65
- private
66
-
67
- def around_perform_method
68
- call_list << 'around_perform_method_before'
69
- yield
70
- call_list << 'around_perform_method_after'
71
- end
72
- end
73
-
74
- # This job adds each callback as they run into an array
75
- class CombinedPerformJob < RocketJob::Job
76
- field :call_list, type: Array, default: []
77
-
78
- before_perform do
79
- call_list << 'before_perform_block'
80
- end
81
-
82
- after_perform do
83
- call_list << 'after_perform_block'
84
- end
85
-
86
- around_perform do |_job, block|
87
- call_list << 'around_perform_block_before'
88
- block.call
89
- call_list << 'around_perform_block_after'
90
- end
91
-
92
- before_perform :before_perform_method
93
-
94
- around_perform :around_perform_method
95
-
96
- after_perform :after_perform_method
97
-
98
- def perform
99
- call_list << 'perform'
100
- end
101
-
102
- private
103
-
104
- def before_perform_method
105
- call_list << 'before_perform_method'
106
- end
107
-
108
- def around_perform_method
109
- call_list << 'around_perform_method_before'
110
- yield
111
- call_list << 'around_perform_method_after'
112
- end
113
-
114
- def after_perform_method
115
- call_list << 'after_perform_method'
116
- end
117
- end
118
-
119
- describe RocketJob::Plugins::Job::Callbacks do
120
- after do
121
- @job.destroy if @job && !@job.new_record?
122
- end
123
-
124
- describe '#before_perform' do
125
- it 'runs blocks and functions' do
126
- @job = BeforePerformJob.new
127
- @job.perform_now
128
- assert @job.completed?, @job.attributes.ai
129
- expected = %w[before_perform_block before_perform_method perform]
130
- assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
131
- end
132
- end
133
-
134
- describe '#after_perform' do
135
- it 'runs blocks and functions' do
136
- @job = AfterPerformJob.new
137
- @job.perform_now
138
- assert @job.completed?, @job.attributes.ai
139
- expected = %w[perform after_perform_method after_perform_block]
140
- assert_equal expected, @job.call_list, 'Sequence of after_perform callbacks is incorrect'
141
- end
142
- end
143
-
144
- describe '#around_perform' do
145
- it 'runs blocks and functions' do
146
- @job = AroundPerformJob.new
147
- @job.perform_now
148
- assert @job.completed?, @job.attributes.ai
149
- expected = %w[around_perform_block_before around_perform_method_before perform around_perform_method_after around_perform_block_after]
150
- assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
151
- end
152
- end
153
-
154
- describe 'all callbacks' do
155
- it 'runs them in the right order' do
156
- @job = CombinedPerformJob.new
157
- @job.perform_now
158
- assert @job.completed?, @job.attributes.ai
159
- expected = %w[before_perform_block around_perform_block_before before_perform_method around_perform_method_before perform after_perform_method around_perform_method_after around_perform_block_after after_perform_block]
160
- assert_equal expected, @job.call_list, 'Sequence of around_perform callbacks is incorrect'
161
- end
162
- end
163
- end
164
- end
165
- end
166
- end
@@ -1,53 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class DefaultsTest < Minitest::Test
7
- class ParentJob < RocketJob::Job
8
- self.priority = 53
9
- self.description = 'Hello'
10
-
11
- def perform
12
- end
13
- end
14
-
15
- class ChildJob < ParentJob
16
- self.priority = 72
17
-
18
- def perform
19
- end
20
- end
21
-
22
- describe RocketJob::Plugins::Job do
23
- after do
24
- @job.destroy if @job && !@job.new_record?
25
- end
26
-
27
- describe '.rocket_job' do
28
- it 'sets defaults after initialize' do
29
- @job = ParentJob.new
30
- assert_equal 53, @job.priority
31
- assert_equal 'Hello', @job.description
32
- end
33
-
34
- it 'can override defaults on initialize' do
35
- @job = ParentJob.new(priority: 72, description: 'More')
36
- assert_equal 72, @job.priority
37
- assert_equal 'More', @job.description
38
- end
39
-
40
- it 'allows a child to override parent defaults' do
41
- @job = ChildJob.new
42
- assert_equal 72, @job.priority
43
- end
44
-
45
- it 'passes down parent defaults' do
46
- @job = ChildJob.new
47
- assert_equal 'Hello', @job.description
48
- end
49
- end
50
- end
51
- end
52
- end
53
- end
@@ -1,56 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- module Plugins
5
- module Job
6
- class LoggerTest < Minitest::Test
7
- class LoggerJob < RocketJob::Job
8
- def perform
9
- logger.debug('DONE', value: 123, other_value: 'HI')
10
- end
11
- end
12
-
13
- describe RocketJob::Plugins::Job::Logger do
14
- before do
15
- LoggerJob.delete_all
16
- end
17
-
18
- after do
19
- @job.destroy if @job && !@job.new_record?
20
- SemanticLogger.flush
21
- end
22
-
23
- describe '#logger' do
24
- it 'uses semantic logger' do
25
- @job = LoggerJob.new
26
- assert_kind_of SemanticLogger::Logger, @job.logger, @job.logger.ai
27
- assert_equal @job.class.name, @job.logger.name, @job.logger.ai
28
- end
29
-
30
- it 'allows perform to log its own data' do
31
- @job = LoggerJob.new
32
- @job.perform_now
33
- end
34
-
35
- it 'adds start logging' do
36
- @job = LoggerJob.new
37
- info_called = false
38
- @job.logger.stub(:info, ->(description) { info_called = true if description == 'Start #perform' }) do
39
- @job.perform_now
40
- end
41
- assert info_called, "In Plugins::Job::Logger.around_perform logger.info('Start #perform') not called"
42
- end
43
-
44
- it 'adds completed logging' do
45
- @job = LoggerJob.new
46
- measure_called = false
47
- @job.logger.stub(:measure_info, ->(description, *_args) { measure_called = true if description == 'Completed #perform' }) do
48
- @job.perform_now
49
- end
50
- assert measure_called, "In Plugins::Job::Logger.around_perform logger.measure_info('Completed #perform') not called"
51
- end
52
- end
53
- end
54
- end
55
- end
56
- end
@@ -1,94 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class ModelTest < Minitest::Test
7
- class SimpleJob < RocketJob::Job
8
- def perform
9
- 10
10
- end
11
- end
12
-
13
- class TwoArgumentJob < RocketJob::Job
14
- self.priority = 53
15
-
16
- def perform(a, b)
17
- a + b
18
- end
19
- end
20
-
21
- describe RocketJob::Plugins::Job::Model do
22
- after do
23
- @job.destroy if @job && !@job.new_record?
24
- @job2.destroy if @job2 && !@job2.new_record?
25
- @job3.destroy if @job3 && !@job3.new_record?
26
- end
27
-
28
- describe '#scheduled?' do
29
- it 'returns true if job is queued to run in the future' do
30
- @job = SimpleJob.new(run_at: 1.day.from_now)
31
- assert_equal true, @job.queued?
32
- assert_equal true, @job.scheduled?
33
- @job.start
34
- assert_equal true, @job.running?
35
- assert_equal false, @job.scheduled?
36
- end
37
-
38
- it 'returns false if job is queued and can be run now' do
39
- @job = SimpleJob.new
40
- assert_equal true, @job.queued?
41
- assert_equal false, @job.scheduled?
42
- end
43
-
44
- it 'returns false if job is running' do
45
- @job = SimpleJob.new
46
- @job.start
47
- assert_equal true, @job.running?
48
- assert_equal false, @job.scheduled?
49
- end
50
- end
51
-
52
- describe 'with queued jobs' do
53
- before do
54
- @job = SimpleJob.create!(description: 'first')
55
- @job2 = SimpleJob.create!(description: 'second', run_at: 1.day.from_now)
56
- @job3 = SimpleJob.create!(description: 'third', run_at: 2.days.from_now)
57
- end
58
-
59
- describe '#scheduled' do
60
- it 'returns only scheduled jobs' do
61
- count = 0
62
- RocketJob::Job.scheduled.each do |job|
63
- count += 1
64
- assert job.scheduled?
65
- end
66
- assert 2, count
67
- end
68
- end
69
-
70
- describe '#queued_now' do
71
- it 'returns only queued jobs, not scheduled ones' do
72
- count = 0
73
- RocketJob::Job.queued_now.each do |job|
74
- count += 1
75
- refute job.scheduled?
76
- end
77
- assert 1, count
78
- end
79
- end
80
-
81
- describe '#queued' do
82
- it 'returns all queued jobs' do
83
- count = 0
84
- RocketJob::Job.queued.each do |_job|
85
- count += 1
86
- end
87
- assert 3, count
88
- end
89
- end
90
- end
91
- end
92
- end
93
- end
94
- end
@@ -1,94 +0,0 @@
1
- require_relative '../../test_helper'
2
-
3
- module Plugins
4
- module Job
5
- # Unit Test for RocketJob::Job
6
- class PersistenceTest < Minitest::Test
7
- class PersistJob < RocketJob::Job
8
- self.priority = 53
9
- field :data, type: Hash
10
-
11
- def perform(hash)
12
- hash
13
- end
14
- end
15
-
16
- describe RocketJob::Plugins::Job::Persistence do
17
- before do
18
- RocketJob::Job.destroy_all
19
- @description = 'Hello World'
20
- @data = {'key' => 'value'}
21
- @job = PersistJob.new(
22
- description: @description,
23
- data: @data,
24
- destroy_on_complete: false
25
- )
26
- end
27
-
28
- after do
29
- @job.destroy if @job && !@job.new_record?
30
- @job2.destroy if @job2 && !@job2.new_record?
31
- @job3.destroy if @job3 && !@job3.new_record?
32
- end
33
-
34
- describe '.config' do
35
- it 'support multiple databases' do
36
- assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
37
- end
38
- end
39
-
40
- describe '.rocket_job' do
41
- it 'sets defaults after initialize' do
42
- assert_equal 53, @job.priority
43
- end
44
- end
45
-
46
- describe '#reload' do
47
- it 'handle hash' do
48
- assert_equal 'value', @job.data['key']
49
- @job.worker_name = nil
50
- @job.save!
51
- @job.worker_name = '123'
52
- @job.reload
53
- assert @job.data.is_a?(Hash), @job.data.class.ai
54
- assert_equal 'value', @job.data['key']
55
- assert_nil @job.worker_name
56
- end
57
- end
58
-
59
- describe '#save!' do
60
- it 'save a blank job' do
61
- @job.save!
62
- assert_nil @job.worker_name
63
- assert_nil @job.completed_at
64
- assert @job.created_at
65
- assert_equal @description, @job.description
66
- assert_equal false, @job.destroy_on_complete
67
- assert_nil @job.expires_at
68
- assert_equal @data, @job.data
69
- assert_equal 0, @job.percent_complete
70
- assert_equal 53, @job.priority
71
- assert_equal 0, @job.failure_count
72
- assert_nil @job.run_at
73
- assert_nil @job.started_at
74
- assert_equal :queued, @job.state
75
- end
76
- end
77
-
78
- describe '.counts_by_state' do
79
- it 'returns states as symbols' do
80
- @job.start!
81
- @job2 = PersistJob.create!(data: {key: 'value'})
82
- @job3 = PersistJob.create!(data: {key: 'value'}, run_at: 1.day.from_now)
83
- counts = RocketJob::Job.counts_by_state
84
- assert_equal 4, counts.size, counts.ai
85
- assert_equal 1, counts[:running]
86
- assert_equal 2, counts[:queued]
87
- assert_equal 1, counts[:queued_now]
88
- assert_equal 1, counts[:scheduled]
89
- end
90
- end
91
- end
92
- end
93
- end
94
- end