rocketjob 3.5.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/bin/rocketjob +1 -0
  4. data/bin/rocketjob_batch_perf +11 -0
  5. data/lib/rocket_job/batch.rb +32 -0
  6. data/lib/rocket_job/batch/callbacks.rb +40 -0
  7. data/lib/rocket_job/batch/io.rb +154 -0
  8. data/lib/rocket_job/batch/logger.rb +57 -0
  9. data/lib/rocket_job/batch/lower_priority.rb +54 -0
  10. data/lib/rocket_job/batch/model.rb +157 -0
  11. data/lib/rocket_job/batch/performance.rb +99 -0
  12. data/lib/rocket_job/batch/result.rb +8 -0
  13. data/lib/rocket_job/batch/results.rb +9 -0
  14. data/lib/rocket_job/batch/state_machine.rb +102 -0
  15. data/lib/rocket_job/batch/statistics.rb +88 -0
  16. data/lib/rocket_job/batch/tabular.rb +56 -0
  17. data/lib/rocket_job/batch/tabular/input.rb +123 -0
  18. data/lib/rocket_job/batch/tabular/output.rb +59 -0
  19. data/lib/rocket_job/batch/throttle.rb +91 -0
  20. data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
  21. data/lib/rocket_job/batch/worker.rb +288 -0
  22. data/lib/rocket_job/cli.rb +29 -7
  23. data/lib/rocket_job/config.rb +1 -1
  24. data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
  25. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
  26. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
  27. data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
  28. data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
  29. data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
  31. data/lib/rocket_job/jobs/performance_job.rb +18 -0
  32. data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
  33. data/lib/rocket_job/plugins/document.rb +2 -8
  34. data/lib/rocket_job/plugins/job/persistence.rb +6 -4
  35. data/lib/rocket_job/plugins/job/throttle.rb +3 -6
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -2
  37. data/lib/rocket_job/server.rb +14 -3
  38. data/lib/rocket_job/sliced/input.rb +336 -0
  39. data/lib/rocket_job/sliced/output.rb +99 -0
  40. data/lib/rocket_job/sliced/slice.rb +166 -0
  41. data/lib/rocket_job/sliced/slices.rb +166 -0
  42. data/lib/rocket_job/sliced/writer/input.rb +60 -0
  43. data/lib/rocket_job/sliced/writer/output.rb +82 -0
  44. data/lib/rocket_job/version.rb +1 -1
  45. data/lib/rocket_job/worker.rb +2 -2
  46. data/lib/rocketjob.rb +28 -0
  47. metadata +51 -62
  48. data/test/config/database.yml +0 -5
  49. data/test/config/mongoid.yml +0 -88
  50. data/test/config_test.rb +0 -10
  51. data/test/dirmon_entry_test.rb +0 -313
  52. data/test/dirmon_job_test.rb +0 -216
  53. data/test/files/text.txt +0 -3
  54. data/test/job_test.rb +0 -71
  55. data/test/jobs/housekeeping_job_test.rb +0 -102
  56. data/test/jobs/on_demand_job_test.rb +0 -59
  57. data/test/jobs/upload_file_job_test.rb +0 -107
  58. data/test/plugins/cron_test.rb +0 -166
  59. data/test/plugins/job/callbacks_test.rb +0 -166
  60. data/test/plugins/job/defaults_test.rb +0 -53
  61. data/test/plugins/job/logger_test.rb +0 -56
  62. data/test/plugins/job/model_test.rb +0 -94
  63. data/test/plugins/job/persistence_test.rb +0 -94
  64. data/test/plugins/job/state_machine_test.rb +0 -116
  65. data/test/plugins/job/throttle_test.rb +0 -111
  66. data/test/plugins/job/worker_test.rb +0 -199
  67. data/test/plugins/processing_window_test.rb +0 -109
  68. data/test/plugins/restart_test.rb +0 -193
  69. data/test/plugins/retry_test.rb +0 -88
  70. data/test/plugins/singleton_test.rb +0 -92
  71. data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
  72. data/test/plugins/state_machine_test.rb +0 -67
  73. data/test/plugins/transaction_test.rb +0 -84
  74. data/test/test_db.sqlite3 +0 -0
  75. data/test/test_helper.rb +0 -17
@@ -1,3 +0,0 @@
1
- Hello World
2
- Line2
3
- Line3
@@ -1,71 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- class JobTest < Minitest::Test
5
- class SimpleJob < RocketJob::Job
6
- def perform
7
- end
8
- end
9
-
10
- describe RocketJob::Job do
11
- before do
12
- @description = 'Hello World'
13
- @job = SimpleJob.new(description: @description)
14
- @job2 = SimpleJob.new(description: @description, priority: 52)
15
- end
16
-
17
- after do
18
- @job.destroy if @job && !@job.new_record?
19
- @job2.destroy if @job2 && !@job2.new_record?
20
- end
21
-
22
- describe '#status' do
23
- it 'return status for a queued job' do
24
- assert_equal true, @job.queued?
25
- h = @job.status
26
- assert_equal :queued, h['state']
27
- assert_equal @description, h['description']
28
- end
29
-
30
- it 'return status for a failed job' do
31
- @job.start!
32
- @job.fail!('worker:1234', 'oh no')
33
- assert_equal true, @job.failed?
34
- h = @job.status
35
- assert_equal :failed, h['state']
36
- assert_equal @description, h['description']
37
- assert_equal 'RocketJob::JobException', h['exception']['class_name'], h
38
- assert_equal 'oh no', h['exception']['message'], h
39
- end
40
- end
41
-
42
- describe '.requeue_dead_server' do
43
- it 'requeue jobs from dead workers' do
44
- assert_equal 52, @job2.priority
45
- worker_name = 'server:12345'
46
- @job.worker_name = worker_name
47
- @job.start!
48
- assert @job.running?, @job.state
49
-
50
- worker_name2 = 'server:76467'
51
- @job2.worker_name = worker_name2
52
- @job2.start!
53
- assert_equal true, @job2.valid?
54
- assert @job2.running?, @job2.state
55
- @job2.save!
56
-
57
- RocketJob::Job.requeue_dead_server(worker_name)
58
- @job.reload
59
-
60
- assert @job.queued?
61
- assert_nil @job.worker_name
62
-
63
- assert_equal worker_name2, @job2.worker_name
64
- @job2.reload
65
- assert_equal worker_name2, @job2.worker_name
66
- assert @job2.running?, @job2.state
67
- assert_equal worker_name2, @job2.worker_name
68
- end
69
- end
70
- end
71
- end
@@ -1,102 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- class HousekeepingJobTest < Minitest::Test
5
- class TestJob < RocketJob::Job
6
- def perform
7
- end
8
- end
9
-
10
- describe RocketJob::Jobs::HousekeepingJob do
11
- before do
12
- HousekeepingJobTest::TestJob.delete_all
13
- RocketJob::Jobs::HousekeepingJob.delete_all
14
- RocketJob::Server.delete_all
15
- end
16
-
17
- describe 'job retention' do
18
- before do
19
- job = HousekeepingJobTest::TestJob.new
20
- Time.stub(:now, 2.days.ago) do
21
- job.abort!
22
- end
23
- job = HousekeepingJobTest::TestJob.new
24
- Time.stub(:now, 8.days.ago) do
25
- job.abort!
26
- end
27
-
28
- job = HousekeepingJobTest::TestJob.new
29
- Time.stub(:now, 2.days.ago) do
30
- job.perform_now
31
- job.save!
32
- end
33
- job = HousekeepingJobTest::TestJob.new
34
- Time.stub(:now, 8.days.ago) do
35
- job.perform_now
36
- job.save!
37
- end
38
-
39
- job = HousekeepingJobTest::TestJob.new
40
- Time.stub(:now, 2.days.ago) do
41
- job.fail!
42
- end
43
- job = HousekeepingJobTest::TestJob.new
44
- Time.stub(:now, 15.days.ago) do
45
- job.fail!
46
- end
47
-
48
- job = HousekeepingJobTest::TestJob.new
49
- Time.stub(:now, 400.days.ago) do
50
- job.pause!
51
- end
52
- job = HousekeepingJobTest::TestJob.new
53
- job.pause!
54
-
55
- HousekeepingJobTest::TestJob.create!(created_at: 15.days.ago)
56
- HousekeepingJobTest::TestJob.create!
57
-
58
- assert_equal 10, HousekeepingJobTest::TestJob.count, -> { HousekeepingJobTest::TestJob.all.to_a.ai }
59
- end
60
-
61
- after do
62
- @job.destroy if @job && !@job.new_record?
63
- end
64
-
65
- describe 'perform' do
66
- it 'destroys jobs' do
67
- @job = RocketJob::Jobs::HousekeepingJob.new
68
- @job.perform_now
69
- assert_equal 1, HousekeepingJobTest::TestJob.aborted.count, -> { HousekeepingJobTest::TestJob.aborted.to_a.ai }
70
- assert_equal 1, HousekeepingJobTest::TestJob.completed.count, -> { HousekeepingJobTest::TestJob.completed.to_a.ai }
71
- assert_equal 1, HousekeepingJobTest::TestJob.failed.count, -> { HousekeepingJobTest::TestJob.failed.to_a.ai }
72
- assert_equal 2, HousekeepingJobTest::TestJob.paused.count, -> { HousekeepingJobTest::TestJob.paused.to_a.ai }
73
- assert_equal 2, HousekeepingJobTest::TestJob.queued.count, -> { HousekeepingJobTest::TestJob.queued.to_a.ai }
74
- end
75
- end
76
- end
77
-
78
- describe 'zombie cleanup' do
79
- before do
80
- @server = RocketJob::Server.new
81
- @server.started!
82
- assert @server.reload.zombie?
83
- assert_equal 1, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
84
- end
85
-
86
- it 'removes zombies' do
87
- @job = RocketJob::Jobs::HousekeepingJob.new
88
- assert @job.destroy_zombies
89
- @job.perform_now
90
- assert_equal 0, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
91
- end
92
-
93
- it 'leaves zombies' do
94
- @job = RocketJob::Jobs::HousekeepingJob.new(destroy_zombies: false)
95
- refute @job.destroy_zombies
96
- assert_equal 1, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
97
- @job.perform_now
98
- assert_equal 1, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
99
- end
100
- end
101
- end
102
- end
@@ -1,59 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- class OnDemandJobTest < Minitest::Test
4
- describe RocketJob::Jobs::OnDemandJob do
5
- before do
6
- RocketJob::Jobs::OnDemandJob.delete_all
7
- end
8
-
9
- describe '#perform' do
10
- it 'hello world' do
11
- code = <<~CODE
12
- logger.info 'Hello World'
13
- CODE
14
-
15
- job = RocketJob::Jobs::OnDemandJob.new(code: code)
16
- job.perform_now
17
- end
18
-
19
- it 'retain output' do
20
- code = <<~CODE
21
- {'value' => 'h' * 24}
22
- CODE
23
-
24
- job = RocketJob::Jobs::OnDemandJob.new(
25
- code: code,
26
- collect_output: true
27
- )
28
- job.perform_now
29
- assert_equal 'h' * 24, job.result['value']
30
- end
31
-
32
- it 'accepts input data' do
33
- code = <<~CODE
34
- {'value' => data['a'] * data['b']}
35
- CODE
36
-
37
- job = RocketJob::Jobs::OnDemandJob.new(
38
- code: code,
39
- collect_output: true,
40
- data: {'a' => 10, 'b' => 2}
41
- )
42
- job.perform_now
43
- assert_equal 20, job.result['value']
44
- end
45
-
46
- it 'validates code' do
47
- code = <<~CODE
48
- def bad code
49
- CODE
50
-
51
- job = RocketJob::Jobs::OnDemandJob.new(code: code)
52
- refute job.valid?
53
- assert_raises Mongoid::Errors::Validations do
54
- job.perform_now
55
- end
56
- end
57
- end
58
- end
59
- end
@@ -1,107 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- class UploadFileJobTest < Minitest::Test
4
- class TestJob < RocketJob::Job
5
- field :upload_file_name, type: String
6
-
7
- def upload(file_name)
8
- self.upload_file_name = file_name
9
- end
10
-
11
- def perform
12
- end
13
- end
14
-
15
- class BatchTestJob < RocketJob::Job
16
- field :upload_file_name, type: String
17
- field :saved_streams, type: Array
18
-
19
- def upload(file_name, streams: nil)
20
- self.upload_file_name = file_name
21
- self.saved_streams = streams
22
- end
23
-
24
- def perform
25
- end
26
- end
27
-
28
- class BadJob < RocketJob::Job
29
- def perform
30
- end
31
- end
32
-
33
- describe RocketJob::Jobs::UploadFileJob do
34
- before do
35
- RocketJob::Job.delete_all
36
- end
37
-
38
- describe '#valid?' do
39
- it 'validates upload_file_name' do
40
- job = RocketJob::Jobs::UploadFileJob.new(job_class_name: UploadFileJobTest::TestJob.to_s)
41
- refute job.valid?
42
- assert_equal ["can't be blank"], job.errors.messages[:upload_file_name]
43
- end
44
-
45
- it 'validates file does not exist' do
46
- job = RocketJob::Jobs::UploadFileJob.new(
47
- job_class_name: UploadFileJobTest::TestJob.to_s,
48
- upload_file_name: '/tmp/blah'
49
- )
50
- refute job.valid?
51
- assert_equal ['Upload file: /tmp/blah does not exist.'], job.errors.messages[:upload_file_name]
52
- end
53
-
54
- it 'validates job_class_name' do
55
- job = RocketJob::Jobs::UploadFileJob.new(job_class_name: UploadFileJobTest::BadJob.to_s)
56
- refute job.valid?
57
- assert_equal ['UploadFileJobTest::BadJob must implement any one of: :upload :upload_file_name= :full_file_name= instance methods'], job.errors.messages[:job_class_name]
58
- end
59
-
60
- it 'with valid job and upload_file_name' do
61
- job = RocketJob::Jobs::UploadFileJob.new(
62
- job_class_name: UploadFileJobTest::TestJob.to_s,
63
- upload_file_name: __FILE__
64
- )
65
- assert job.valid?
66
- end
67
- end
68
-
69
- describe '#perform' do
70
- let :job do
71
- RocketJob::Jobs::UploadFileJob.new(
72
- job_class_name: UploadFileJobTest::TestJob.name,
73
- upload_file_name: __FILE__
74
- )
75
- end
76
-
77
- it 'creates the job' do
78
- job.perform_now
79
- assert_equal 1, UploadFileJobTest::TestJob.count
80
- assert job = UploadFileJobTest::TestJob.first
81
- end
82
-
83
- it 'calls upload' do
84
- job.perform_now
85
- assert job = UploadFileJobTest::TestJob.first
86
- assert_equal __FILE__, job.upload_file_name
87
- end
88
-
89
- it 'calls upload with original_file_name' do
90
- job.job_class_name = BatchTestJob.name
91
- job.perform_now
92
- assert created_job = UploadFileJobTest::BatchTestJob.first
93
- assert_equal __FILE__, created_job.upload_file_name
94
- assert_nil created_job.saved_streams
95
- end
96
-
97
- it 'job retains the original_file_name when present' do
98
- job.job_class_name = BatchTestJob.name
99
- job.original_file_name = 'file.zip'
100
- job.perform_now
101
- assert created_job = UploadFileJobTest::BatchTestJob.first
102
- assert_equal 'file.zip', created_job.upload_file_name
103
- assert_equal %i[file zip], created_job.saved_streams
104
- end
105
- end
106
- end
107
- end
@@ -1,166 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- module Plugins
5
- class CronTest < Minitest::Test
6
- class CronJob < RocketJob::Job
7
- include RocketJob::Plugins::Cron
8
-
9
- def perform
10
- 'DONE'
11
- end
12
- end
13
-
14
- class FailOnceCronJob < RocketJob::Job
15
- include RocketJob::Plugins::Cron
16
-
17
- def perform
18
- raise 'oh no' if failure_count.zero?
19
- end
20
- end
21
-
22
- describe RocketJob::Plugins::Cron do
23
- before do
24
- # destroy_all could create new instances
25
- CronJob.delete_all
26
- FailOnceCronJob.delete_all
27
- assert_equal 0, CronJob.count
28
- end
29
-
30
- after do
31
- @job.destroy if @job && !@job.new_record?
32
- CronJob.delete_all
33
- end
34
-
35
- describe '#create!' do
36
- it 'queues a new job' do
37
- @job = CronJob.create!(cron_schedule: '* 1 * * *')
38
- assert @job.valid?
39
- refute @job.new_record?
40
- end
41
-
42
- describe 'timezones are supported' do
43
- it 'handles UTC' do
44
- time = Time.parse('2015-12-09 17:50:05 UTC')
45
- Time.stub(:now, time) do
46
- @job = CronJob.create!(cron_schedule: '* 1 * * * UTC')
47
- end
48
- assert @job.valid?
49
- refute @job.new_record?
50
- assert_equal Time.parse('2015-12-10 01:00:00 UTC'), @job.run_at
51
- end
52
-
53
- it 'handles Eastern' do
54
- time = Time.parse('2015-12-09 17:50:05 UTC')
55
- Time.stub(:now, time) do
56
- @job = CronJob.create!(cron_schedule: '* 1 * * * America/New_York')
57
- end
58
- assert @job.valid?
59
- refute @job.new_record?
60
- assert_equal Time.parse('2015-12-10 06:00:00 UTC'), @job.run_at
61
- end
62
- end
63
- end
64
-
65
- describe '#save' do
66
- it 'updates run_at for a new record' do
67
- @job = CronJob.create!(cron_schedule: '* 1 * * *')
68
- assert @job.run_at
69
- end
70
-
71
- it 'updates run_at for a modified record' do
72
- @job = CronJob.create!(cron_schedule: '* 1 * * * UTC')
73
- assert run_at = @job.run_at
74
- @job.cron_schedule = '* 2 * * * UTC'
75
- assert_equal run_at, @job.run_at
76
- @job.save!
77
- assert run_at < @job.run_at
78
- end
79
- end
80
-
81
- describe '#valid?' do
82
- it 'allows missing cron schedule' do
83
- @job = CronJob.new
84
- assert @job.valid?
85
- end
86
-
87
- it 'fails on bad cron schedule' do
88
- @job = CronJob.new(cron_schedule: 'blah')
89
- refute @job.valid?
90
- assert_equal "not a valid cronline : 'blah'", @job.errors.messages[:cron_schedule].first
91
- end
92
-
93
- it 'passes on valid cron schedule' do
94
- @job = CronJob.new(cron_schedule: '* 1 * * *')
95
- assert @job.valid?
96
- end
97
- end
98
-
99
- describe '#fail' do
100
- describe 'with cron_schedule' do
101
- let :job do
102
- job = FailOnceCronJob.create!(cron_schedule: '* 1 * * *')
103
- job.start
104
- assert_raises RuntimeError do
105
- job.perform_now
106
- end
107
- job.reload
108
- job
109
- end
110
-
111
- it 'allows current cron job instance to fail' do
112
- assert job.failed?
113
- end
114
-
115
- it 'clears out cron_schedule' do
116
- refute job.cron_schedule
117
- end
118
-
119
- it 'retains run_at' do
120
- assert job.run_at
121
- end
122
-
123
- it 'schedules a new instance' do
124
- assert_equal 0, FailOnceCronJob.count
125
- job
126
- assert_equal 2, FailOnceCronJob.count
127
- assert scheduled_job = FailOnceCronJob.last
128
- assert scheduled_job.queued?
129
- assert_equal '* 1 * * *', scheduled_job.cron_schedule
130
- end
131
-
132
- it 'restarts on retry' do
133
- job.retry!
134
- job.perform_now
135
- assert job.completed?
136
- assert_equal 1, FailOnceCronJob.count, -> { FailOnceCronJob.all.to_a.collect(&:state).to_s }
137
- assert_equal 1, FailOnceCronJob.queued.count
138
- end
139
- end
140
-
141
- describe 'without cron_schedule' do
142
- let :job do
143
- job = CronJob.create!
144
- job.start
145
- job.fail
146
- job
147
- end
148
-
149
- it 'allows current cron job instance to fail' do
150
- assert job.failed?
151
- end
152
-
153
- it 'has no cron_schedule' do
154
- refute job.cron_schedule
155
- end
156
-
157
- it 'does not schedule a new instance' do
158
- assert_equal 0, CronJob.count
159
- job
160
- assert_equal 1, CronJob.count
161
- end
162
- end
163
- end
164
- end
165
- end
166
- end