rocketjob 3.5.2 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/bin/rocketjob +1 -0
  4. data/bin/rocketjob_batch_perf +11 -0
  5. data/lib/rocket_job/batch.rb +32 -0
  6. data/lib/rocket_job/batch/callbacks.rb +40 -0
  7. data/lib/rocket_job/batch/io.rb +154 -0
  8. data/lib/rocket_job/batch/logger.rb +57 -0
  9. data/lib/rocket_job/batch/lower_priority.rb +54 -0
  10. data/lib/rocket_job/batch/model.rb +157 -0
  11. data/lib/rocket_job/batch/performance.rb +99 -0
  12. data/lib/rocket_job/batch/result.rb +8 -0
  13. data/lib/rocket_job/batch/results.rb +9 -0
  14. data/lib/rocket_job/batch/state_machine.rb +102 -0
  15. data/lib/rocket_job/batch/statistics.rb +88 -0
  16. data/lib/rocket_job/batch/tabular.rb +56 -0
  17. data/lib/rocket_job/batch/tabular/input.rb +123 -0
  18. data/lib/rocket_job/batch/tabular/output.rb +59 -0
  19. data/lib/rocket_job/batch/throttle.rb +91 -0
  20. data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
  21. data/lib/rocket_job/batch/worker.rb +288 -0
  22. data/lib/rocket_job/cli.rb +29 -7
  23. data/lib/rocket_job/config.rb +1 -1
  24. data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
  25. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
  26. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
  27. data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
  28. data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
  29. data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
  31. data/lib/rocket_job/jobs/performance_job.rb +18 -0
  32. data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
  33. data/lib/rocket_job/plugins/document.rb +2 -8
  34. data/lib/rocket_job/plugins/job/persistence.rb +6 -4
  35. data/lib/rocket_job/plugins/job/throttle.rb +3 -6
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -2
  37. data/lib/rocket_job/server.rb +14 -3
  38. data/lib/rocket_job/sliced/input.rb +336 -0
  39. data/lib/rocket_job/sliced/output.rb +99 -0
  40. data/lib/rocket_job/sliced/slice.rb +166 -0
  41. data/lib/rocket_job/sliced/slices.rb +166 -0
  42. data/lib/rocket_job/sliced/writer/input.rb +60 -0
  43. data/lib/rocket_job/sliced/writer/output.rb +82 -0
  44. data/lib/rocket_job/version.rb +1 -1
  45. data/lib/rocket_job/worker.rb +2 -2
  46. data/lib/rocketjob.rb +28 -0
  47. metadata +51 -62
  48. data/test/config/database.yml +0 -5
  49. data/test/config/mongoid.yml +0 -88
  50. data/test/config_test.rb +0 -10
  51. data/test/dirmon_entry_test.rb +0 -313
  52. data/test/dirmon_job_test.rb +0 -216
  53. data/test/files/text.txt +0 -3
  54. data/test/job_test.rb +0 -71
  55. data/test/jobs/housekeeping_job_test.rb +0 -102
  56. data/test/jobs/on_demand_job_test.rb +0 -59
  57. data/test/jobs/upload_file_job_test.rb +0 -107
  58. data/test/plugins/cron_test.rb +0 -166
  59. data/test/plugins/job/callbacks_test.rb +0 -166
  60. data/test/plugins/job/defaults_test.rb +0 -53
  61. data/test/plugins/job/logger_test.rb +0 -56
  62. data/test/plugins/job/model_test.rb +0 -94
  63. data/test/plugins/job/persistence_test.rb +0 -94
  64. data/test/plugins/job/state_machine_test.rb +0 -116
  65. data/test/plugins/job/throttle_test.rb +0 -111
  66. data/test/plugins/job/worker_test.rb +0 -199
  67. data/test/plugins/processing_window_test.rb +0 -109
  68. data/test/plugins/restart_test.rb +0 -193
  69. data/test/plugins/retry_test.rb +0 -88
  70. data/test/plugins/singleton_test.rb +0 -92
  71. data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
  72. data/test/plugins/state_machine_test.rb +0 -67
  73. data/test/plugins/transaction_test.rb +0 -84
  74. data/test/test_db.sqlite3 +0 -0
  75. data/test/test_helper.rb +0 -17
@@ -1,193 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- module Plugins
5
- class RestartTest < Minitest::Test
6
- class RestartableJob < RocketJob::Job
7
- include RocketJob::Plugins::Restart
8
-
9
- field :start_at, type: Date
10
- field :end_at, type: Date
11
-
12
- def perform
13
- self.start_at = Date.today
14
- self.end_at = Date.today
15
- 'DONE'
16
- end
17
- end
18
-
19
- class RestartablePausableJob < RocketJob::Job
20
- include RocketJob::Plugins::Restart
21
-
22
- field :start_at, type: Date
23
- field :end_at, type: Date
24
-
25
- # Job will reload itself during process to check if it was paused.
26
- self.pausable = true
27
-
28
- def perform
29
- self.start_at = Date.today
30
- self.end_at = Date.today
31
- 'DONE'
32
- end
33
- end
34
-
35
- describe RocketJob::Plugins::Restart do
36
- before do
37
- RestartableJob.delete_all
38
- RestartablePausableJob.delete_all
39
- end
40
-
41
- after do
42
- @job.delete if @job && !@job.new_record?
43
- end
44
-
45
- describe '#create!' do
46
- it 'queues a new job' do
47
- @job = RestartableJob.create!
48
- assert @job.valid?
49
- refute @job.new_record?
50
- end
51
- end
52
-
53
- describe '#save!' do
54
- it 'queues a new job' do
55
- @job = RestartableJob.new
56
- @job.save!
57
- assert @job.valid?
58
- refute @job.new_record?
59
- end
60
- end
61
-
62
- describe '#abort!' do
63
- it 'queues a new job on abort' do
64
- @job = RestartableJob.create!
65
- @job.abort!
66
- assert_equal 2, RestartableJob.count
67
- assert other = RestartableJob.where(:id.ne => @job.id).first
68
- refute_equal @job.id, other.id
69
- assert other.queued?
70
- end
71
-
72
- it 'does not queue a new job when expired' do
73
- @job = RestartableJob.create!(expires_at: Time.now - 1.day)
74
- assert @job.expired?
75
- @job.abort!
76
- assert_equal 1, RestartableJob.count
77
- assert_nil RestartableJob.where(:id.ne => @job.id).first
78
- end
79
- end
80
-
81
- describe '#complete' do
82
- it 'queues a new job when destroy_on_complete' do
83
- assert_equal 0, RestartableJob.count
84
- @job = RestartableJob.create!(destroy_on_complete: true)
85
- @job.perform_now
86
- assert @job.completed?, @job.attributes.ai
87
- assert_equal 1, RestartableJob.count, RestartableJob.all.to_a.ai
88
- end
89
-
90
- it 'queues a new job when not destroy_on_complete' do
91
- @job = RestartableJob.create!(destroy_on_complete: false)
92
- @job.perform_now
93
- assert @job.completed?
94
- assert_equal 2, RestartableJob.count
95
- end
96
-
97
- it 'does not queue a new job when expired' do
98
- @job = RestartableJob.create!(expires_at: Time.now - 1.day)
99
- @job.perform_now
100
- assert @job.expired?
101
- assert @job.completed?
102
- assert_equal 0, RestartableJob.count
103
- end
104
- end
105
-
106
- describe '#pause' do
107
- it 'does not enqueue a new job when paused' do
108
- @job = RestartablePausableJob.new
109
- @job.start
110
- @job.pause!
111
- assert @job.paused?
112
- assert_equal 1, RestartablePausableJob.count
113
- end
114
- end
115
-
116
- describe '#fail' do
117
- it 'aborts from queued' do
118
- @job = RestartableJob.new
119
- assert @job.queued?
120
- @job.fail
121
- assert @job.aborted?
122
- end
123
-
124
- it 'aborts from running' do
125
- @job = RestartableJob.new
126
- @job.start
127
- assert @job.running?
128
- @job.fail
129
- assert @job.aborted?
130
- end
131
-
132
- it 'aborts from paused' do
133
- @job = RestartablePausableJob.new
134
- @job.start
135
- @job.pause
136
- assert @job.paused?
137
- @job.fail
138
- assert @job.aborted?
139
- end
140
-
141
- it 'does not queue a new job when expired' do
142
- @job = RestartableJob.new(expires_at: Time.now - 1.day)
143
- @job.start!
144
- assert @job.running?
145
- assert @job.expired?
146
- assert_equal 1, RestartableJob.count
147
- assert_nil RestartableJob.where(:id.ne => @job.id).first
148
- end
149
- end
150
-
151
- describe '#create_new_instance' do
152
- it 'sets job back to queued state' do
153
- @job = RestartableJob.create!(destroy_on_complete: true)
154
- @job.perform_now
155
- assert_equal 1, RestartableJob.count
156
- assert job2 = RestartableJob.where(:id.ne => @job.id).first
157
- assert job2.queued?, job2.attributes.ai
158
- end
159
-
160
- it 'excludes attributes related to running jobs' do
161
- @job = RestartableJob.create!(destroy_on_complete: true, expires_at: Time.now + 1.day)
162
- refute @job.expired?
163
- @job.perform_now
164
- assert_equal 1, RestartableJob.count
165
- assert job2 = RestartableJob.where(:id.ne => @job.id).first
166
- assert job2.queued?, job2.attributes.ai
167
-
168
- assert RestartableJob.rocket_job_restart_attributes.include?(:priority)
169
- assert RestartableJob.rocket_job_restart_attributes.exclude?(:start_at)
170
- assert RestartableJob.rocket_job_restart_attributes.exclude?(:end_at)
171
- assert RestartableJob.rocket_job_restart_attributes.exclude?(:run_at)
172
-
173
- # Copy across all attributes, except
174
- RestartableJob.rocket_job_restart_attributes.each do |key|
175
- assert_equal @job.send(key).to_s, job2.send(key).to_s, "Attributes are supposed to be copied across. For #{key}"
176
- end
177
-
178
- assert_nil job2.start_at
179
- assert_nil job2.end_at
180
- assert_equal :queued, job2.state
181
- assert job2.created_at
182
- assert_nil job2.started_at
183
- assert_nil job2.completed_at
184
- assert_equal 0, job2.failure_count
185
- assert_nil job2.worker_name
186
- assert_equal 0, job2.percent_complete
187
- assert_nil job2.exception
188
- refute job2.result
189
- end
190
- end
191
- end
192
- end
193
- end
@@ -1,88 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- module Plugins
4
- class RetryTest < Minitest::Test
5
- class RetryJob < RocketJob::Job
6
- include RocketJob::Plugins::Retry
7
-
8
- # Fails 5 times before succeeding
9
- def perform
10
- raise 'Oh No' unless rocket_job_failure_count >= 5
11
- 'DONE'
12
- end
13
- end
14
-
15
- describe RocketJob::Plugins::Retry do
16
- before do
17
- RetryJob.delete_all
18
- end
19
-
20
- after do
21
- @job.delete if @job && !@job.new_record?
22
- end
23
-
24
- describe '#perform' do
25
- it 're-queues job on failure' do
26
- @job = RetryJob.create!
27
- assert created_at = @job.created_at
28
- assert_equal 0, @job.failed_at_list.size
29
-
30
- assert_raises RuntimeError do
31
- @job.perform_now
32
- end
33
-
34
- assert @job.queued?, -> { @job.attributes.ai }
35
-
36
- # Includes failure time
37
- assert_equal 1, @job.rocket_job_failure_count
38
- assert failed_at = @job.failed_at_list.first
39
- assert failed_at >= created_at
40
-
41
- assert next_time = @job.run_at
42
- assert next_time > failed_at
43
- end
44
-
45
- it 're-queues until it succeeds' do
46
- @job = RetryJob.create!
47
-
48
- # 5 retries
49
- 5.times do |i|
50
- assert_raises RuntimeError do
51
- @job.perform_now
52
- end
53
- assert @job.queued?, -> { @job.attributes.ai }
54
- assert_equal (i + 1), @job.rocket_job_failure_count
55
- end
56
-
57
- assert_equal 5, @job.rocket_job_failure_count
58
-
59
- # Should succeed on the 6th attempt
60
- @job.perform_now
61
- assert @job.completed?, -> { @job.attributes.ai }
62
- assert_equal 5, @job.rocket_job_failure_count
63
- end
64
-
65
- it 'stops re-queueing after limit is reached' do
66
- @job = RetryJob.create!(retry_limit: 3)
67
-
68
- # 3 attempts are retried
69
- 3.times do |i|
70
- assert_raises RuntimeError do
71
- @job.perform_now
72
- end
73
- assert @job.queued?, -> { @job.attributes.ai }
74
- assert_equal (i + 1), @job.rocket_job_failure_count
75
- end
76
-
77
- # Should fail on the 4th attempt
78
- assert_equal 3, @job.rocket_job_failure_count
79
- assert_raises RuntimeError do
80
- @job.perform_now
81
- end
82
-
83
- assert @job.failed?, -> { @job.attributes.ai }
84
- end
85
- end
86
- end
87
- end
88
- end
@@ -1,92 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- module Plugins
4
- # Unit Test for RocketJob::Job
5
- class SingletonTest < Minitest::Test
6
- class SingletonJob < RocketJob::Job
7
- include RocketJob::Plugins::Singleton
8
-
9
- def perform
10
- end
11
- end
12
-
13
- describe RocketJob::Plugins::Singleton do
14
- before do
15
- SingletonJob.delete_all
16
- end
17
-
18
- after do
19
- @job.destroy if @job && !@job.new_record?
20
- end
21
-
22
- describe '#rocket_job_singleton_active?' do
23
- it 'returns false if no jobs of this class are active' do
24
- @job = SingletonJob.new
25
- assert_equal false, @job.rocket_job_singleton_active?
26
- end
27
-
28
- it 'excludes self when queued from check' do
29
- @job = SingletonJob.create
30
- assert @job.queued?
31
- assert_equal false, @job.rocket_job_singleton_active?
32
- end
33
-
34
- it 'excludes self when started from check' do
35
- @job = SingletonJob.new
36
- @job.start!
37
- assert @job.running?
38
- assert_equal false, @job.rocket_job_singleton_active?
39
- end
40
-
41
- it 'returns true when other jobs of this class are queued' do
42
- @job = SingletonJob.create!
43
- job2 = SingletonJob.new
44
- assert_equal true, job2.rocket_job_singleton_active?
45
- end
46
-
47
- it 'returns true when other jobs of this class are running' do
48
- @job = SingletonJob.new
49
- @job.start!
50
- job2 = SingletonJob.new
51
- assert_equal true, job2.rocket_job_singleton_active?
52
- end
53
-
54
- it 'returns false when other jobs of this class are not active' do
55
- @job = SingletonJob.new
56
- @job.perform_now
57
- @job.save!
58
- assert @job.completed?
59
- job2 = SingletonJob.new
60
- assert_equal false, job2.rocket_job_singleton_active?
61
- end
62
- end
63
-
64
- describe 'validation' do
65
- it 'passes if another job is not active' do
66
- @job = SingletonJob.new
67
- @job.perform_now
68
- @job.save!
69
- job2 = SingletonJob.new
70
- assert_equal true, job2.valid?
71
- end
72
-
73
- it 'fails if another job is active' do
74
- @job = SingletonJob.new
75
- @job.start!
76
- job2 = SingletonJob.new
77
- refute job2.valid?
78
- assert_equal ['Another instance of Plugins::SingletonTest::SingletonJob is already running, queued, or paused'], job2.errors.messages[:state]
79
- end
80
-
81
- it 'passes if another job is active, but this job is not' do
82
- @job = SingletonJob.new
83
- @job.start!
84
- job2 = SingletonJob.new
85
- job2.abort
86
- assert job2.valid?
87
- job2.save!
88
- end
89
- end
90
- end
91
- end
92
- end
@@ -1,102 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- module Plugins
4
- # Unit Test for RocketJob::Job
5
- class StateMachineEventCallbacksTest < Minitest::Test
6
- # This job adds each event callback as they run into an array
7
- # [:start, :complete, :fail, :retry, :pause, :resume, :abort, :requeue]
8
- class PositivePathJob < RocketJob::Job
9
- field :call_list, type: Array, default: []
10
-
11
- before_complete do
12
- call_list << 'before_complete_block'
13
- end
14
-
15
- after_complete do
16
- call_list << 'after_complete_block'
17
- end
18
-
19
- before_complete :before_complete_method
20
-
21
- before_start do
22
- call_list << 'before_start_block'
23
- end
24
-
25
- before_start :before_start_method
26
-
27
- before_start do
28
- call_list << 'before_start2_block'
29
- end
30
-
31
- after_start :after_start_method
32
- after_complete :after_complete_method
33
-
34
- before_complete do
35
- call_list << 'before_complete2_block'
36
- end
37
-
38
- after_start do
39
- call_list << 'after_start_block'
40
- end
41
-
42
- after_complete do
43
- call_list << 'after_complete2_block'
44
- end
45
-
46
- after_start :after_start_method
47
-
48
- after_start do
49
- call_list << 'after_start2_block'
50
- end
51
-
52
- before_start :before_start_method2
53
- before_complete :before_complete_method2
54
-
55
- def perform
56
- call_list << 'perform'
57
- end
58
-
59
- private
60
-
61
- def before_start_method
62
- call_list << 'before_start_method'
63
- end
64
-
65
- def before_start_method2
66
- call_list << 'before_start_method2'
67
- end
68
-
69
- def after_start_method
70
- call_list << 'after_start_method'
71
- end
72
-
73
- def before_complete_method
74
- call_list << 'before_complete_method'
75
- end
76
-
77
- def before_complete_method2
78
- call_list << 'before_complete_method2'
79
- end
80
-
81
- def after_complete_method
82
- call_list << 'after_complete_method'
83
- end
84
- end
85
-
86
- describe RocketJob::Plugins::StateMachine do
87
- after do
88
- @job.destroy if @job && !@job.new_record?
89
- end
90
-
91
- describe 'before_start after_start & before_complete after_complete' do
92
- it 'runs blocks and functions' do
93
- @job = PositivePathJob.new
94
- @job.perform_now
95
- assert @job.completed?, @job.attributes.ai
96
- expected = %w[before_start_block before_start_method before_start2_block before_start_method2 after_start2_block after_start_method after_start_block perform before_complete_block before_complete_method before_complete2_block before_complete_method2 after_complete2_block after_complete_method after_complete_block]
97
- assert_equal expected, @job.call_list, 'Sequence of before_perform callbacks is incorrect'
98
- end
99
- end
100
- end
101
- end
102
- end