rocketjob 3.5.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/bin/rocketjob +1 -0
  4. data/bin/rocketjob_batch_perf +11 -0
  5. data/lib/rocket_job/batch.rb +32 -0
  6. data/lib/rocket_job/batch/callbacks.rb +40 -0
  7. data/lib/rocket_job/batch/io.rb +154 -0
  8. data/lib/rocket_job/batch/logger.rb +57 -0
  9. data/lib/rocket_job/batch/lower_priority.rb +54 -0
  10. data/lib/rocket_job/batch/model.rb +157 -0
  11. data/lib/rocket_job/batch/performance.rb +99 -0
  12. data/lib/rocket_job/batch/result.rb +8 -0
  13. data/lib/rocket_job/batch/results.rb +9 -0
  14. data/lib/rocket_job/batch/state_machine.rb +102 -0
  15. data/lib/rocket_job/batch/statistics.rb +88 -0
  16. data/lib/rocket_job/batch/tabular.rb +56 -0
  17. data/lib/rocket_job/batch/tabular/input.rb +123 -0
  18. data/lib/rocket_job/batch/tabular/output.rb +59 -0
  19. data/lib/rocket_job/batch/throttle.rb +91 -0
  20. data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
  21. data/lib/rocket_job/batch/worker.rb +288 -0
  22. data/lib/rocket_job/cli.rb +29 -7
  23. data/lib/rocket_job/config.rb +1 -1
  24. data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
  25. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
  26. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
  27. data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
  28. data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
  29. data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
  31. data/lib/rocket_job/jobs/performance_job.rb +18 -0
  32. data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
  33. data/lib/rocket_job/plugins/document.rb +2 -8
  34. data/lib/rocket_job/plugins/job/persistence.rb +6 -4
  35. data/lib/rocket_job/plugins/job/throttle.rb +3 -6
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -2
  37. data/lib/rocket_job/server.rb +14 -3
  38. data/lib/rocket_job/sliced/input.rb +336 -0
  39. data/lib/rocket_job/sliced/output.rb +99 -0
  40. data/lib/rocket_job/sliced/slice.rb +166 -0
  41. data/lib/rocket_job/sliced/slices.rb +166 -0
  42. data/lib/rocket_job/sliced/writer/input.rb +60 -0
  43. data/lib/rocket_job/sliced/writer/output.rb +82 -0
  44. data/lib/rocket_job/version.rb +1 -1
  45. data/lib/rocket_job/worker.rb +2 -2
  46. data/lib/rocketjob.rb +28 -0
  47. metadata +51 -62
  48. data/test/config/database.yml +0 -5
  49. data/test/config/mongoid.yml +0 -88
  50. data/test/config_test.rb +0 -10
  51. data/test/dirmon_entry_test.rb +0 -313
  52. data/test/dirmon_job_test.rb +0 -216
  53. data/test/files/text.txt +0 -3
  54. data/test/job_test.rb +0 -71
  55. data/test/jobs/housekeeping_job_test.rb +0 -102
  56. data/test/jobs/on_demand_job_test.rb +0 -59
  57. data/test/jobs/upload_file_job_test.rb +0 -107
  58. data/test/plugins/cron_test.rb +0 -166
  59. data/test/plugins/job/callbacks_test.rb +0 -166
  60. data/test/plugins/job/defaults_test.rb +0 -53
  61. data/test/plugins/job/logger_test.rb +0 -56
  62. data/test/plugins/job/model_test.rb +0 -94
  63. data/test/plugins/job/persistence_test.rb +0 -94
  64. data/test/plugins/job/state_machine_test.rb +0 -116
  65. data/test/plugins/job/throttle_test.rb +0 -111
  66. data/test/plugins/job/worker_test.rb +0 -199
  67. data/test/plugins/processing_window_test.rb +0 -109
  68. data/test/plugins/restart_test.rb +0 -193
  69. data/test/plugins/retry_test.rb +0 -88
  70. data/test/plugins/singleton_test.rb +0 -92
  71. data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
  72. data/test/plugins/state_machine_test.rb +0 -67
  73. data/test/plugins/transaction_test.rb +0 -84
  74. data/test/test_db.sqlite3 +0 -0
  75. data/test/test_helper.rb +0 -17
@@ -1,193 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- module Plugins
5
- class RestartTest < Minitest::Test
6
- class RestartableJob < RocketJob::Job
7
- include RocketJob::Plugins::Restart
8
-
9
- field :start_at, type: Date
10
- field :end_at, type: Date
11
-
12
- def perform
13
- self.start_at = Date.today
14
- self.end_at = Date.today
15
- 'DONE'
16
- end
17
- end
18
-
19
- class RestartablePausableJob < RocketJob::Job
20
- include RocketJob::Plugins::Restart
21
-
22
- field :start_at, type: Date
23
- field :end_at, type: Date
24
-
25
- # Job will reload itself during process to check if it was paused.
26
- self.pausable = true
27
-
28
- def perform
29
- self.start_at = Date.today
30
- self.end_at = Date.today
31
- 'DONE'
32
- end
33
- end
34
-
35
- describe RocketJob::Plugins::Restart do
36
- before do
37
- RestartableJob.delete_all
38
- RestartablePausableJob.delete_all
39
- end
40
-
41
- after do
42
- @job.delete if @job && !@job.new_record?
43
- end
44
-
45
- describe '#create!' do
46
- it 'queues a new job' do
47
- @job = RestartableJob.create!
48
- assert @job.valid?
49
- refute @job.new_record?
50
- end
51
- end
52
-
53
- describe '#save!' do
54
- it 'queues a new job' do
55
- @job = RestartableJob.new
56
- @job.save!
57
- assert @job.valid?
58
- refute @job.new_record?
59
- end
60
- end
61
-
62
- describe '#abort!' do
63
- it 'queues a new job on abort' do
64
- @job = RestartableJob.create!
65
- @job.abort!
66
- assert_equal 2, RestartableJob.count
67
- assert other = RestartableJob.where(:id.ne => @job.id).first
68
- refute_equal @job.id, other.id
69
- assert other.queued?
70
- end
71
-
72
- it 'does not queue a new job when expired' do
73
- @job = RestartableJob.create!(expires_at: Time.now - 1.day)
74
- assert @job.expired?
75
- @job.abort!
76
- assert_equal 1, RestartableJob.count
77
- assert_nil RestartableJob.where(:id.ne => @job.id).first
78
- end
79
- end
80
-
81
- describe '#complete' do
82
- it 'queues a new job when destroy_on_complete' do
83
- assert_equal 0, RestartableJob.count
84
- @job = RestartableJob.create!(destroy_on_complete: true)
85
- @job.perform_now
86
- assert @job.completed?, @job.attributes.ai
87
- assert_equal 1, RestartableJob.count, RestartableJob.all.to_a.ai
88
- end
89
-
90
- it 'queues a new job when not destroy_on_complete' do
91
- @job = RestartableJob.create!(destroy_on_complete: false)
92
- @job.perform_now
93
- assert @job.completed?
94
- assert_equal 2, RestartableJob.count
95
- end
96
-
97
- it 'does not queue a new job when expired' do
98
- @job = RestartableJob.create!(expires_at: Time.now - 1.day)
99
- @job.perform_now
100
- assert @job.expired?
101
- assert @job.completed?
102
- assert_equal 0, RestartableJob.count
103
- end
104
- end
105
-
106
- describe '#pause' do
107
- it 'does not enqueue a new job when paused' do
108
- @job = RestartablePausableJob.new
109
- @job.start
110
- @job.pause!
111
- assert @job.paused?
112
- assert_equal 1, RestartablePausableJob.count
113
- end
114
- end
115
-
116
- describe '#fail' do
117
- it 'aborts from queued' do
118
- @job = RestartableJob.new
119
- assert @job.queued?
120
- @job.fail
121
- assert @job.aborted?
122
- end
123
-
124
- it 'aborts from running' do
125
- @job = RestartableJob.new
126
- @job.start
127
- assert @job.running?
128
- @job.fail
129
- assert @job.aborted?
130
- end
131
-
132
- it 'aborts from paused' do
133
- @job = RestartablePausableJob.new
134
- @job.start
135
- @job.pause
136
- assert @job.paused?
137
- @job.fail
138
- assert @job.aborted?
139
- end
140
-
141
- it 'does not queue a new job when expired' do
142
- @job = RestartableJob.new(expires_at: Time.now - 1.day)
143
- @job.start!
144
- assert @job.running?
145
- assert @job.expired?
146
- assert_equal 1, RestartableJob.count
147
- assert_nil RestartableJob.where(:id.ne => @job.id).first
148
- end
149
- end
150
-
151
- describe '#create_new_instance' do
152
- it 'sets job back to queued state' do
153
- @job = RestartableJob.create!(destroy_on_complete: true)
154
- @job.perform_now
155
- assert_equal 1, RestartableJob.count
156
- assert job2 = RestartableJob.where(:id.ne => @job.id).first
157
- assert job2.queued?, job2.attributes.ai
158
- end
159
-
160
- it 'excludes attributes related to running jobs' do
161
- @job = RestartableJob.create!(destroy_on_complete: true, expires_at: Time.now + 1.day)
162
- refute @job.expired?
163
- @job.perform_now
164
- assert_equal 1, RestartableJob.count
165
- assert job2 = RestartableJob.where(:id.ne => @job.id).first
166
- assert job2.queued?, job2.attributes.ai
167
-
168
- assert RestartableJob.rocket_job_restart_attributes.include?(:priority)
169
- assert RestartableJob.rocket_job_restart_attributes.exclude?(:start_at)
170
- assert RestartableJob.rocket_job_restart_attributes.exclude?(:end_at)
171
- assert RestartableJob.rocket_job_restart_attributes.exclude?(:run_at)
172
-
173
- # Copy across all attributes, except
174
- RestartableJob.rocket_job_restart_attributes.each do |key|
175
- assert_equal @job.send(key).to_s, job2.send(key).to_s, "Attributes are supposed to be copied across. For #{key}"
176
- end
177
-
178
- assert_nil job2.start_at
179
- assert_nil job2.end_at
180
- assert_equal :queued, job2.state
181
- assert job2.created_at
182
- assert_nil job2.started_at
183
- assert_nil job2.completed_at
184
- assert_equal 0, job2.failure_count
185
- assert_nil job2.worker_name
186
- assert_equal 0, job2.percent_complete
187
- assert_nil job2.exception
188
- refute job2.result
189
- end
190
- end
191
- end
192
- end
193
- end
@@ -1,88 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- module Plugins
4
- class RetryTest < Minitest::Test
5
- class RetryJob < RocketJob::Job
6
- include RocketJob::Plugins::Retry
7
-
8
- # Fails 5 times before succeeding
9
- def perform
10
- raise 'Oh No' unless rocket_job_failure_count >= 5
11
- 'DONE'
12
- end
13
- end
14
-
15
- describe RocketJob::Plugins::Retry do
16
- before do
17
- RetryJob.delete_all
18
- end
19
-
20
- after do
21
- @job.delete if @job && !@job.new_record?
22
- end
23
-
24
- describe '#perform' do
25
- it 're-queues job on failure' do
26
- @job = RetryJob.create!
27
- assert created_at = @job.created_at
28
- assert_equal 0, @job.failed_at_list.size
29
-
30
- assert_raises RuntimeError do
31
- @job.perform_now
32
- end
33
-
34
- assert @job.queued?, -> { @job.attributes.ai }
35
-
36
- # Includes failure time
37
- assert_equal 1, @job.rocket_job_failure_count
38
- assert failed_at = @job.failed_at_list.first
39
- assert failed_at >= created_at
40
-
41
- assert next_time = @job.run_at
42
- assert next_time > failed_at
43
- end
44
-
45
- it 're-queues until it succeeds' do
46
- @job = RetryJob.create!
47
-
48
- # 5 retries
49
- 5.times do |i|
50
- assert_raises RuntimeError do
51
- @job.perform_now
52
- end
53
- assert @job.queued?, -> { @job.attributes.ai }
54
- assert_equal (i + 1), @job.rocket_job_failure_count
55
- end
56
-
57
- assert_equal 5, @job.rocket_job_failure_count
58
-
59
- # Should succeed on the 6th attempt
60
- @job.perform_now
61
- assert @job.completed?, -> { @job.attributes.ai }
62
- assert_equal 5, @job.rocket_job_failure_count
63
- end
64
-
65
- it 'stops re-queueing after limit is reached' do
66
- @job = RetryJob.create!(retry_limit: 3)
67
-
68
- # 3 attempts are retried
69
- 3.times do |i|
70
- assert_raises RuntimeError do
71
- @job.perform_now
72
- end
73
- assert @job.queued?, -> { @job.attributes.ai }
74
- assert_equal (i + 1), @job.rocket_job_failure_count
75
- end
76
-
77
- # Should fail on the 4th attempt
78
- assert_equal 3, @job.rocket_job_failure_count
79
- assert_raises RuntimeError do
80
- @job.perform_now
81
- end
82
-
83
- assert @job.failed?, -> { @job.attributes.ai }
84
- end
85
- end
86
- end
87
- end
88
- end
@@ -1,92 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- module Plugins
4
- # Unit Test for RocketJob::Job
5
- class SingletonTest < Minitest::Test
6
- class SingletonJob < RocketJob::Job
7
- include RocketJob::Plugins::Singleton
8
-
9
- def perform
10
- end
11
- end
12
-
13
- describe RocketJob::Plugins::Singleton do
14
- before do
15
- SingletonJob.delete_all
16
- end
17
-
18
- after do
19
- @job.destroy if @job && !@job.new_record?
20
- end
21
-
22
- describe '#rocket_job_singleton_active?' do
23
- it 'returns false if no jobs of this class are active' do
24
- @job = SingletonJob.new
25
- assert_equal false, @job.rocket_job_singleton_active?
26
- end
27
-
28
- it 'excludes self when queued from check' do
29
- @job = SingletonJob.create
30
- assert @job.queued?
31
- assert_equal false, @job.rocket_job_singleton_active?
32
- end
33
-
34
- it 'excludes self when started from check' do
35
- @job = SingletonJob.new
36
- @job.start!
37
- assert @job.running?
38
- assert_equal false, @job.rocket_job_singleton_active?
39
- end
40
-
41
- it 'returns true when other jobs of this class are queued' do
42
- @job = SingletonJob.create!
43
- job2 = SingletonJob.new
44
- assert_equal true, job2.rocket_job_singleton_active?
45
- end
46
-
47
- it 'returns true when other jobs of this class are running' do
48
- @job = SingletonJob.new
49
- @job.start!
50
- job2 = SingletonJob.new
51
- assert_equal true, job2.rocket_job_singleton_active?
52
- end
53
-
54
- it 'returns false when other jobs of this class are not active' do
55
- @job = SingletonJob.new
56
- @job.perform_now
57
- @job.save!
58
- assert @job.completed?
59
- job2 = SingletonJob.new
60
- assert_equal false, job2.rocket_job_singleton_active?
61
- end
62
- end
63
-
64
- describe 'validation' do
65
- it 'passes if another job is not active' do
66
- @job = SingletonJob.new
67
- @job.perform_now
68
- @job.save!
69
- job2 = SingletonJob.new
70
- assert_equal true, job2.valid?
71
- end
72
-
73
- it 'fails if another job is active' do
74
- @job = SingletonJob.new
75
- @job.start!
76
- job2 = SingletonJob.new
77
- refute job2.valid?
78
- assert_equal ['Another instance of Plugins::SingletonTest::SingletonJob is already running, queued, or paused'], job2.errors.messages[:state]
79
- end
80
-
81
- it 'passes if another job is active, but this job is not' do
82
- @job = SingletonJob.new
83
- @job.start!
84
- job2 = SingletonJob.new
85
- job2.abort
86
- assert job2.valid?
87
- job2.save!
88
- end
89
- end
90
- end
91
- end
92
- end
@@ -1,102 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- module Plugins
4
- # Unit Test for RocketJob::Job
5
- class StateMachineEventCallbacksTest < Minitest::Test
6
- # This job adds each event callback as they run into an array
7
- # [:start, :complete, :fail, :retry, :pause, :resume, :abort, :requeue]
8
- class PositivePathJob < RocketJob::Job
9
- field :call_list, type: Array, default: []
10
-
11
- before_complete do
12
- call_list << 'before_complete_block'
13
- end
14
-
15
- after_complete do
16
- call_list << 'after_complete_block'
17
- end
18
-
19
- before_complete :before_complete_method
20
-
21
- before_start do
22
- call_list << 'before_start_block'
23
- end
24
-
25
- before_start :before_start_method
26
-
27
- before_start do
28
- call_list << 'before_start2_block'
29
- end
30
-
31
- after_start :after_start_method
32
- after_complete :after_complete_method
33
-
34
- before_complete do
35
- call_list << 'before_complete2_block'
36
- end
37
-
38
- after_start do
39
- call_list << 'after_start_block'
40
- end
41
-
42
- after_complete do
43
- call_list << 'after_complete2_block'
44
- end
45
-
46
- after_start :after_start_method
47
-
48
- after_start do
49
- call_list << 'after_start2_block'
50
- end
51
-
52
- before_start :before_start_method2
53
- before_complete :before_complete_method2
54
-
55
- def perform
56
- call_list << 'perform'
57
- end
58
-
59
- private
60
-
61
- def before_start_method
62
- call_list << 'before_start_method'
63
- end
64
-
65
- def before_start_method2
66
- call_list << 'before_start_method2'
67
- end
68
-
69
- def after_start_method
70
- call_list << 'after_start_method'
71
- end
72
-
73
- def before_complete_method
74
- call_list << 'before_complete_method'
75
- end
76
-
77
- def before_complete_method2
78
- call_list << 'before_complete_method2'
79
- end
80
-
81
- def after_complete_method
82
- call_list << 'after_complete_method'
83
- end
84
- end
85
-
86
- describe RocketJob::Plugins::StateMachine do
87
- after do
88
- @job.destroy if @job && !@job.new_record?
89
- end
90
-
91
- describe 'before_start after_start & before_complete after_complete' do
92
- it 'runs blocks and functions' do
93
- @job = PositivePathJob.new
94
- @job.perform_now
95
- assert @job.completed?, @job.attributes.ai
96
- expected = %w[before_start_block before_start_method before_start2_block before_start_method2 after_start2_block after_start_method after_start_block perform before_complete_block before_complete_method before_complete2_block before_complete_method2 after_complete2_block after_complete_method after_complete_block]
97
- assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
98
- end
99
- end
100
- end
101
- end
102
- end