rocketjob 3.5.2 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/bin/rocketjob +1 -0
  4. data/bin/rocketjob_batch_perf +11 -0
  5. data/lib/rocket_job/batch.rb +32 -0
  6. data/lib/rocket_job/batch/callbacks.rb +40 -0
  7. data/lib/rocket_job/batch/io.rb +154 -0
  8. data/lib/rocket_job/batch/logger.rb +57 -0
  9. data/lib/rocket_job/batch/lower_priority.rb +54 -0
  10. data/lib/rocket_job/batch/model.rb +157 -0
  11. data/lib/rocket_job/batch/performance.rb +99 -0
  12. data/lib/rocket_job/batch/result.rb +8 -0
  13. data/lib/rocket_job/batch/results.rb +9 -0
  14. data/lib/rocket_job/batch/state_machine.rb +102 -0
  15. data/lib/rocket_job/batch/statistics.rb +88 -0
  16. data/lib/rocket_job/batch/tabular.rb +56 -0
  17. data/lib/rocket_job/batch/tabular/input.rb +123 -0
  18. data/lib/rocket_job/batch/tabular/output.rb +59 -0
  19. data/lib/rocket_job/batch/throttle.rb +91 -0
  20. data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
  21. data/lib/rocket_job/batch/worker.rb +288 -0
  22. data/lib/rocket_job/cli.rb +29 -7
  23. data/lib/rocket_job/config.rb +1 -1
  24. data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
  25. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
  26. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
  27. data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
  28. data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
  29. data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
  31. data/lib/rocket_job/jobs/performance_job.rb +18 -0
  32. data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
  33. data/lib/rocket_job/plugins/document.rb +2 -8
  34. data/lib/rocket_job/plugins/job/persistence.rb +6 -4
  35. data/lib/rocket_job/plugins/job/throttle.rb +3 -6
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -2
  37. data/lib/rocket_job/server.rb +14 -3
  38. data/lib/rocket_job/sliced/input.rb +336 -0
  39. data/lib/rocket_job/sliced/output.rb +99 -0
  40. data/lib/rocket_job/sliced/slice.rb +166 -0
  41. data/lib/rocket_job/sliced/slices.rb +166 -0
  42. data/lib/rocket_job/sliced/writer/input.rb +60 -0
  43. data/lib/rocket_job/sliced/writer/output.rb +82 -0
  44. data/lib/rocket_job/version.rb +1 -1
  45. data/lib/rocket_job/worker.rb +2 -2
  46. data/lib/rocketjob.rb +28 -0
  47. metadata +51 -62
  48. data/test/config/database.yml +0 -5
  49. data/test/config/mongoid.yml +0 -88
  50. data/test/config_test.rb +0 -10
  51. data/test/dirmon_entry_test.rb +0 -313
  52. data/test/dirmon_job_test.rb +0 -216
  53. data/test/files/text.txt +0 -3
  54. data/test/job_test.rb +0 -71
  55. data/test/jobs/housekeeping_job_test.rb +0 -102
  56. data/test/jobs/on_demand_job_test.rb +0 -59
  57. data/test/jobs/upload_file_job_test.rb +0 -107
  58. data/test/plugins/cron_test.rb +0 -166
  59. data/test/plugins/job/callbacks_test.rb +0 -166
  60. data/test/plugins/job/defaults_test.rb +0 -53
  61. data/test/plugins/job/logger_test.rb +0 -56
  62. data/test/plugins/job/model_test.rb +0 -94
  63. data/test/plugins/job/persistence_test.rb +0 -94
  64. data/test/plugins/job/state_machine_test.rb +0 -116
  65. data/test/plugins/job/throttle_test.rb +0 -111
  66. data/test/plugins/job/worker_test.rb +0 -199
  67. data/test/plugins/processing_window_test.rb +0 -109
  68. data/test/plugins/restart_test.rb +0 -193
  69. data/test/plugins/retry_test.rb +0 -88
  70. data/test/plugins/singleton_test.rb +0 -92
  71. data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
  72. data/test/plugins/state_machine_test.rb +0 -67
  73. data/test/plugins/transaction_test.rb +0 -84
  74. data/test/test_db.sqlite3 +0 -0
  75. data/test/test_helper.rb +0 -17
@@ -1,3 +0,0 @@
1
- Hello World
2
- Line2
3
- Line3
@@ -1,71 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- class JobTest < Minitest::Test
5
- class SimpleJob < RocketJob::Job
6
- def perform
7
- end
8
- end
9
-
10
- describe RocketJob::Job do
11
- before do
12
- @description = 'Hello World'
13
- @job = SimpleJob.new(description: @description)
14
- @job2 = SimpleJob.new(description: @description, priority: 52)
15
- end
16
-
17
- after do
18
- @job.destroy if @job && !@job.new_record?
19
- @job2.destroy if @job2 && !@job2.new_record?
20
- end
21
-
22
- describe '#status' do
23
- it 'return status for a queued job' do
24
- assert_equal true, @job.queued?
25
- h = @job.status
26
- assert_equal :queued, h['state']
27
- assert_equal @description, h['description']
28
- end
29
-
30
- it 'return status for a failed job' do
31
- @job.start!
32
- @job.fail!('worker:1234', 'oh no')
33
- assert_equal true, @job.failed?
34
- h = @job.status
35
- assert_equal :failed, h['state']
36
- assert_equal @description, h['description']
37
- assert_equal 'RocketJob::JobException', h['exception']['class_name'], h
38
- assert_equal 'oh no', h['exception']['message'], h
39
- end
40
- end
41
-
42
- describe '.requeue_dead_server' do
43
- it 'requeue jobs from dead workers' do
44
- assert_equal 52, @job2.priority
45
- worker_name = 'server:12345'
46
- @job.worker_name = worker_name
47
- @job.start!
48
- assert @job.running?, @job.state
49
-
50
- worker_name2 = 'server:76467'
51
- @job2.worker_name = worker_name2
52
- @job2.start!
53
- assert_equal true, @job2.valid?
54
- assert @job2.running?, @job2.state
55
- @job2.save!
56
-
57
- RocketJob::Job.requeue_dead_server(worker_name)
58
- @job.reload
59
-
60
- assert @job.queued?
61
- assert_nil @job.worker_name
62
-
63
- assert_equal worker_name2, @job2.worker_name
64
- @job2.reload
65
- assert_equal worker_name2, @job2.worker_name
66
- assert @job2.running?, @job2.state
67
- assert_equal worker_name2, @job2.worker_name
68
- end
69
- end
70
- end
71
- end
@@ -1,102 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- class HousekeepingJobTest < Minitest::Test
5
- class TestJob < RocketJob::Job
6
- def perform
7
- end
8
- end
9
-
10
- describe RocketJob::Jobs::HousekeepingJob do
11
- before do
12
- HousekeepingJobTest::TestJob.delete_all
13
- RocketJob::Jobs::HousekeepingJob.delete_all
14
- RocketJob::Server.delete_all
15
- end
16
-
17
- describe 'job retention' do
18
- before do
19
- job = HousekeepingJobTest::TestJob.new
20
- Time.stub(:now, 2.days.ago) do
21
- job.abort!
22
- end
23
- job = HousekeepingJobTest::TestJob.new
24
- Time.stub(:now, 8.days.ago) do
25
- job.abort!
26
- end
27
-
28
- job = HousekeepingJobTest::TestJob.new
29
- Time.stub(:now, 2.days.ago) do
30
- job.perform_now
31
- job.save!
32
- end
33
- job = HousekeepingJobTest::TestJob.new
34
- Time.stub(:now, 8.days.ago) do
35
- job.perform_now
36
- job.save!
37
- end
38
-
39
- job = HousekeepingJobTest::TestJob.new
40
- Time.stub(:now, 2.days.ago) do
41
- job.fail!
42
- end
43
- job = HousekeepingJobTest::TestJob.new
44
- Time.stub(:now, 15.days.ago) do
45
- job.fail!
46
- end
47
-
48
- job = HousekeepingJobTest::TestJob.new
49
- Time.stub(:now, 400.days.ago) do
50
- job.pause!
51
- end
52
- job = HousekeepingJobTest::TestJob.new
53
- job.pause!
54
-
55
- HousekeepingJobTest::TestJob.create!(created_at: 15.days.ago)
56
- HousekeepingJobTest::TestJob.create!
57
-
58
- assert_equal 10, HousekeepingJobTest::TestJob.count, -> { HousekeepingJobTest::TestJob.all.to_a.ai }
59
- end
60
-
61
- after do
62
- @job.destroy if @job && !@job.new_record?
63
- end
64
-
65
- describe 'perform' do
66
- it 'destroys jobs' do
67
- @job = RocketJob::Jobs::HousekeepingJob.new
68
- @job.perform_now
69
- assert_equal 1, HousekeepingJobTest::TestJob.aborted.count, -> { HousekeepingJobTest::TestJob.aborted.to_a.ai }
70
- assert_equal 1, HousekeepingJobTest::TestJob.completed.count, -> { HousekeepingJobTest::TestJob.completed.to_a.ai }
71
- assert_equal 1, HousekeepingJobTest::TestJob.failed.count, -> { HousekeepingJobTest::TestJob.failed.to_a.ai }
72
- assert_equal 2, HousekeepingJobTest::TestJob.paused.count, -> { HousekeepingJobTest::TestJob.paused.to_a.ai }
73
- assert_equal 2, HousekeepingJobTest::TestJob.queued.count, -> { HousekeepingJobTest::TestJob.queued.to_a.ai }
74
- end
75
- end
76
- end
77
-
78
- describe 'zombie cleanup' do
79
- before do
80
- @server = RocketJob::Server.new
81
- @server.started!
82
- assert @server.reload.zombie?
83
- assert_equal 1, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
84
- end
85
-
86
- it 'removes zombies' do
87
- @job = RocketJob::Jobs::HousekeepingJob.new
88
- assert @job.destroy_zombies
89
- @job.perform_now
90
- assert_equal 0, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
91
- end
92
-
93
- it 'leaves zombies' do
94
- @job = RocketJob::Jobs::HousekeepingJob.new(destroy_zombies: false)
95
- refute @job.destroy_zombies
96
- assert_equal 1, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
97
- @job.perform_now
98
- assert_equal 1, RocketJob::Server.count, -> { RocketJob::Server.all.to_a.ai }
99
- end
100
- end
101
- end
102
- end
@@ -1,59 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- class OnDemandJobTest < Minitest::Test
4
- describe RocketJob::Jobs::OnDemandJob do
5
- before do
6
- RocketJob::Jobs::OnDemandJob.delete_all
7
- end
8
-
9
- describe '#perform' do
10
- it 'hello world' do
11
- code = <<~CODE
12
- logger.info 'Hello World'
13
- CODE
14
-
15
- job = RocketJob::Jobs::OnDemandJob.new(code: code)
16
- job.perform_now
17
- end
18
-
19
- it 'retain output' do
20
- code = <<~CODE
21
- {'value' => 'h' * 24}
22
- CODE
23
-
24
- job = RocketJob::Jobs::OnDemandJob.new(
25
- code: code,
26
- collect_output: true
27
- )
28
- job.perform_now
29
- assert_equal 'h' * 24, job.result['value']
30
- end
31
-
32
- it 'accepts input data' do
33
- code = <<~CODE
34
- {'value' => data['a'] * data['b']}
35
- CODE
36
-
37
- job = RocketJob::Jobs::OnDemandJob.new(
38
- code: code,
39
- collect_output: true,
40
- data: {'a' => 10, 'b' => 2}
41
- )
42
- job.perform_now
43
- assert_equal 20, job.result['value']
44
- end
45
-
46
- it 'validates code' do
47
- code = <<~CODE
48
- def bad code
49
- CODE
50
-
51
- job = RocketJob::Jobs::OnDemandJob.new(code: code)
52
- refute job.valid?
53
- assert_raises Mongoid::Errors::Validations do
54
- job.perform_now
55
- end
56
- end
57
- end
58
- end
59
- end
@@ -1,107 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- class UploadFileJobTest < Minitest::Test
4
- class TestJob < RocketJob::Job
5
- field :upload_file_name, type: String
6
-
7
- def upload(file_name)
8
- self.upload_file_name = file_name
9
- end
10
-
11
- def perform
12
- end
13
- end
14
-
15
- class BatchTestJob < RocketJob::Job
16
- field :upload_file_name, type: String
17
- field :saved_streams, type: Array
18
-
19
- def upload(file_name, streams: nil)
20
- self.upload_file_name = file_name
21
- self.saved_streams = streams
22
- end
23
-
24
- def perform
25
- end
26
- end
27
-
28
- class BadJob < RocketJob::Job
29
- def perform
30
- end
31
- end
32
-
33
- describe RocketJob::Jobs::UploadFileJob do
34
- before do
35
- RocketJob::Job.delete_all
36
- end
37
-
38
- describe '#valid?' do
39
- it 'validates upload_file_name' do
40
- job = RocketJob::Jobs::UploadFileJob.new(job_class_name: UploadFileJobTest::TestJob.to_s)
41
- refute job.valid?
42
- assert_equal ["can't be blank"], job.errors.messages[:upload_file_name]
43
- end
44
-
45
- it 'validates file does not exist' do
46
- job = RocketJob::Jobs::UploadFileJob.new(
47
- job_class_name: UploadFileJobTest::TestJob.to_s,
48
- upload_file_name: '/tmp/blah'
49
- )
50
- refute job.valid?
51
- assert_equal ['Upload file: /tmp/blah does not exist.'], job.errors.messages[:upload_file_name]
52
- end
53
-
54
- it 'validates job_class_name' do
55
- job = RocketJob::Jobs::UploadFileJob.new(job_class_name: UploadFileJobTest::BadJob.to_s)
56
- refute job.valid?
57
- assert_equal ['UploadFileJobTest::BadJob must implement any one of: :upload :upload_file_name= :full_file_name= instance methods'], job.errors.messages[:job_class_name]
58
- end
59
-
60
- it 'with valid job and upload_file_name' do
61
- job = RocketJob::Jobs::UploadFileJob.new(
62
- job_class_name: UploadFileJobTest::TestJob.to_s,
63
- upload_file_name: __FILE__
64
- )
65
- assert job.valid?
66
- end
67
- end
68
-
69
- describe '#perform' do
70
- let :job do
71
- RocketJob::Jobs::UploadFileJob.new(
72
- job_class_name: UploadFileJobTest::TestJob.name,
73
- upload_file_name: __FILE__
74
- )
75
- end
76
-
77
- it 'creates the job' do
78
- job.perform_now
79
- assert_equal 1, UploadFileJobTest::TestJob.count
80
- assert job = UploadFileJobTest::TestJob.first
81
- end
82
-
83
- it 'calls upload' do
84
- job.perform_now
85
- assert job = UploadFileJobTest::TestJob.first
86
- assert_equal __FILE__, job.upload_file_name
87
- end
88
-
89
- it 'calls upload with original_file_name' do
90
- job.job_class_name = BatchTestJob.name
91
- job.perform_now
92
- assert created_job = UploadFileJobTest::BatchTestJob.first
93
- assert_equal __FILE__, created_job.upload_file_name
94
- assert_nil created_job.saved_streams
95
- end
96
-
97
- it 'job retains the original_file_name when present' do
98
- job.job_class_name = BatchTestJob.name
99
- job.original_file_name = 'file.zip'
100
- job.perform_now
101
- assert created_job = UploadFileJobTest::BatchTestJob.first
102
- assert_equal 'file.zip', created_job.upload_file_name
103
- assert_equal %i[file zip], created_job.saved_streams
104
- end
105
- end
106
- end
107
- end
@@ -1,166 +0,0 @@
1
- require_relative '../test_helper'
2
-
3
- # Unit Test for RocketJob::Job
4
- module Plugins
5
- class CronTest < Minitest::Test
6
- class CronJob < RocketJob::Job
7
- include RocketJob::Plugins::Cron
8
-
9
- def perform
10
- 'DONE'
11
- end
12
- end
13
-
14
- class FailOnceCronJob < RocketJob::Job
15
- include RocketJob::Plugins::Cron
16
-
17
- def perform
18
- raise 'oh no' if failure_count.zero?
19
- end
20
- end
21
-
22
- describe RocketJob::Plugins::Cron do
23
- before do
24
- # destroy_all could create new instances
25
- CronJob.delete_all
26
- FailOnceCronJob.delete_all
27
- assert_equal 0, CronJob.count
28
- end
29
-
30
- after do
31
- @job.destroy if @job && !@job.new_record?
32
- CronJob.delete_all
33
- end
34
-
35
- describe '#create!' do
36
- it 'queues a new job' do
37
- @job = CronJob.create!(cron_schedule: '* 1 * * *')
38
- assert @job.valid?
39
- refute @job.new_record?
40
- end
41
-
42
- describe 'timezones are supported' do
43
- it 'handles UTC' do
44
- time = Time.parse('2015-12-09 17:50:05 UTC')
45
- Time.stub(:now, time) do
46
- @job = CronJob.create!(cron_schedule: '* 1 * * * UTC')
47
- end
48
- assert @job.valid?
49
- refute @job.new_record?
50
- assert_equal Time.parse('2015-12-10 01:00:00 UTC'), @job.run_at
51
- end
52
-
53
- it 'handles Eastern' do
54
- time = Time.parse('2015-12-09 17:50:05 UTC')
55
- Time.stub(:now, time) do
56
- @job = CronJob.create!(cron_schedule: '* 1 * * * America/New_York')
57
- end
58
- assert @job.valid?
59
- refute @job.new_record?
60
- assert_equal Time.parse('2015-12-10 06:00:00 UTC'), @job.run_at
61
- end
62
- end
63
- end
64
-
65
- describe '#save' do
66
- it 'updates run_at for a new record' do
67
- @job = CronJob.create!(cron_schedule: '* 1 * * *')
68
- assert @job.run_at
69
- end
70
-
71
- it 'updates run_at for a modified record' do
72
- @job = CronJob.create!(cron_schedule: '* 1 * * * UTC')
73
- assert run_at = @job.run_at
74
- @job.cron_schedule = '* 2 * * * UTC'
75
- assert_equal run_at, @job.run_at
76
- @job.save!
77
- assert run_at < @job.run_at
78
- end
79
- end
80
-
81
- describe '#valid?' do
82
- it 'allows missing cron schedule' do
83
- @job = CronJob.new
84
- assert @job.valid?
85
- end
86
-
87
- it 'fails on bad cron schedule' do
88
- @job = CronJob.new(cron_schedule: 'blah')
89
- refute @job.valid?
90
- assert_equal "not a valid cronline : 'blah'", @job.errors.messages[:cron_schedule].first
91
- end
92
-
93
- it 'passes on valid cron schedule' do
94
- @job = CronJob.new(cron_schedule: '* 1 * * *')
95
- assert @job.valid?
96
- end
97
- end
98
-
99
- describe '#fail' do
100
- describe 'with cron_schedule' do
101
- let :job do
102
- job = FailOnceCronJob.create!(cron_schedule: '* 1 * * *')
103
- job.start
104
- assert_raises RuntimeError do
105
- job.perform_now
106
- end
107
- job.reload
108
- job
109
- end
110
-
111
- it 'allows current cron job instance to fail' do
112
- assert job.failed?
113
- end
114
-
115
- it 'clears out cron_schedule' do
116
- refute job.cron_schedule
117
- end
118
-
119
- it 'retains run_at' do
120
- assert job.run_at
121
- end
122
-
123
- it 'schedules a new instance' do
124
- assert_equal 0, FailOnceCronJob.count
125
- job
126
- assert_equal 2, FailOnceCronJob.count
127
- assert scheduled_job = FailOnceCronJob.last
128
- assert scheduled_job.queued?
129
- assert_equal '* 1 * * *', scheduled_job.cron_schedule
130
- end
131
-
132
- it 'restarts on retry' do
133
- job.retry!
134
- job.perform_now
135
- assert job.completed?
136
- assert_equal 1, FailOnceCronJob.count, -> { FailOnceCronJob.all.to_a.collect(&:state).to_s }
137
- assert_equal 1, FailOnceCronJob.queued.count
138
- end
139
- end
140
-
141
- describe 'without cron_schedule' do
142
- let :job do
143
- job = CronJob.create!
144
- job.start
145
- job.fail
146
- job
147
- end
148
-
149
- it 'allows current cron job instance to fail' do
150
- assert job.failed?
151
- end
152
-
153
- it 'has no cron_schedule' do
154
- refute job.cron_schedule
155
- end
156
-
157
- it 'does not schedule a new instance' do
158
- assert_equal 0, CronJob.count
159
- job
160
- assert_equal 1, CronJob.count
161
- end
162
- end
163
- end
164
- end
165
- end
166
- end